]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | """ |
2 | rgw routines | |
3 | """ | |
4 | import argparse | |
5 | import contextlib | |
6 | import json | |
7 | import logging | |
8 | import os | |
9 | import errno | |
10 | import util.rgw as rgw_utils | |
11 | ||
7c673cae FG |
12 | from teuthology.orchestra import run |
13 | from teuthology import misc as teuthology | |
14 | from teuthology import contextutil | |
15 | from teuthology.orchestra.run import CommandFailedError | |
31f18b77 | 16 | from util.rgw import rgwadmin, wait_for_radosgw |
7c673cae FG |
17 | from util.rados import (rados, create_ec_pool, |
18 | create_replicated_pool, | |
19 | create_cache_pool) | |
20 | ||
21 | log = logging.getLogger(__name__) | |
22 | ||
23 | @contextlib.contextmanager | |
31f18b77 | 24 | def start_rgw(ctx, config): |
7c673cae FG |
25 | """ |
26 | Start rgw on remote sites. | |
27 | """ | |
28 | log.info('Starting rgw...') | |
7c673cae | 29 | testdir = teuthology.get_testdir(ctx) |
31f18b77 | 30 | for client in config.keys(): |
7c673cae FG |
31 | (remote,) = ctx.cluster.only(client).remotes.iterkeys() |
32 | cluster_name, daemon_type, client_id = teuthology.split_role(client) | |
33 | client_with_id = daemon_type + '.' + client_id | |
34 | client_with_cluster = cluster_name + '.' + client_with_id | |
7c673cae FG |
35 | |
36 | client_config = config.get(client) | |
37 | if client_config is None: | |
38 | client_config = {} | |
39 | log.info("rgw %s config is %s", client, client_config) | |
7c673cae FG |
40 | cmd_prefix = [ |
41 | 'sudo', | |
42 | 'adjust-ulimits', | |
43 | 'ceph-coverage', | |
44 | '{tdir}/archive/coverage'.format(tdir=testdir), | |
45 | 'daemon-helper', | |
46 | 'term', | |
47 | ] | |
48 | ||
49 | rgw_cmd = ['radosgw'] | |
50 | ||
31f18b77 | 51 | log.info("Using %s as radosgw frontend", ctx.rgw.frontend) |
7c673cae | 52 | |
31f18b77 | 53 | host, port = ctx.rgw.role_endpoints[client] |
7c673cae | 54 | rgw_cmd.extend([ |
31f18b77 FG |
55 | '--rgw-frontends', |
56 | '{frontend} port={port}'.format(frontend=ctx.rgw.frontend, port=port), | |
7c673cae FG |
57 | '-n', client_with_id, |
58 | '--cluster', cluster_name, | |
59 | '-k', '/etc/ceph/{client_with_cluster}.keyring'.format(client_with_cluster=client_with_cluster), | |
60 | '--log-file', | |
61 | '/var/log/ceph/rgw.{client_with_cluster}.log'.format(client_with_cluster=client_with_cluster), | |
62 | '--rgw_ops_log_socket_path', | |
63 | '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(tdir=testdir, | |
64 | client_with_cluster=client_with_cluster), | |
65 | '--foreground', | |
66 | run.Raw('|'), | |
67 | 'sudo', | |
68 | 'tee', | |
69 | '/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(tdir=testdir, | |
70 | client_with_cluster=client_with_cluster), | |
71 | run.Raw('2>&1'), | |
72 | ]) | |
73 | ||
74 | if client_config.get('valgrind'): | |
75 | cmd_prefix = teuthology.get_valgrind_args( | |
76 | testdir, | |
31f18b77 | 77 | client_with_cluster, |
7c673cae FG |
78 | cmd_prefix, |
79 | client_config.get('valgrind') | |
80 | ) | |
81 | ||
82 | run_cmd = list(cmd_prefix) | |
83 | run_cmd.extend(rgw_cmd) | |
84 | ||
85 | ctx.daemons.add_daemon( | |
31f18b77 | 86 | remote, 'rgw', client_with_id, |
7c673cae FG |
87 | cluster=cluster_name, |
88 | args=run_cmd, | |
89 | logger=log.getChild(client), | |
90 | stdin=run.PIPE, | |
91 | wait=False, | |
92 | ) | |
93 | ||
94 | # XXX: add_daemon() doesn't let us wait until radosgw finishes startup | |
31f18b77 | 95 | for client in config.keys(): |
7c673cae FG |
96 | host, port = ctx.rgw.role_endpoints[client] |
97 | endpoint = 'http://{host}:{port}/'.format(host=host, port=port) | |
98 | log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint)) | |
31f18b77 | 99 | wait_for_radosgw(endpoint) |
7c673cae FG |
100 | |
101 | try: | |
102 | yield | |
103 | finally: | |
7c673cae | 104 | for client in config.iterkeys(): |
31f18b77 FG |
105 | cluster_name, daemon_type, client_id = teuthology.split_role(client) |
106 | client_with_id = daemon_type + '.' + client_id | |
107 | client_with_cluster = cluster_name + '.' + client_with_id | |
108 | ctx.daemons.get_daemon('rgw', client_with_id, cluster_name).stop() | |
7c673cae FG |
109 | ctx.cluster.only(client).run( |
110 | args=[ | |
111 | 'rm', | |
112 | '-f', | |
31f18b77 FG |
113 | '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir, |
114 | client=client_with_cluster), | |
7c673cae FG |
115 | ], |
116 | ) | |
117 | ||
7c673cae FG |
118 | def assign_ports(ctx, config): |
119 | """ | |
120 | Assign port numberst starting with port 7280. | |
121 | """ | |
122 | port = 7280 | |
123 | role_endpoints = {} | |
124 | for remote, roles_for_host in ctx.cluster.remotes.iteritems(): | |
125 | for role in roles_for_host: | |
126 | if role in config: | |
127 | role_endpoints[role] = (remote.name.split('@')[1], port) | |
128 | port += 1 | |
129 | ||
130 | return role_endpoints | |
131 | ||
7c673cae | 132 | @contextlib.contextmanager |
31f18b77 | 133 | def create_pools(ctx, config): |
7c673cae | 134 | """Create replicated or erasure coded data pools for rgw.""" |
7c673cae | 135 | |
31f18b77 | 136 | log.info('Creating data pools') |
7c673cae FG |
137 | for client in config.keys(): |
138 | (remote,) = ctx.cluster.only(client).remotes.iterkeys() | |
139 | data_pool = '.rgw.buckets' | |
140 | cluster_name, daemon_type, client_id = teuthology.split_role(client) | |
141 | ||
142 | if ctx.rgw.ec_data_pool: | |
143 | create_ec_pool(remote, data_pool, client, 64, | |
144 | ctx.rgw.erasure_code_profile, cluster_name) | |
145 | else: | |
146 | create_replicated_pool(remote, data_pool, 64, cluster_name) | |
147 | if ctx.rgw.cache_pools: | |
148 | create_cache_pool(remote, data_pool, data_pool + '.cache', 64, | |
149 | 64*1024*1024, cluster_name) | |
31f18b77 | 150 | log.debug('Pools created') |
7c673cae FG |
151 | yield |
152 | ||
153 | @contextlib.contextmanager | |
31f18b77 FG |
154 | def configure_compression(ctx, config, compression): |
155 | """ set a compression type in the default zone placement """ | |
156 | log.info('Configuring compression type = %s', compression) | |
7c673cae | 157 | for client, c_config in config.iteritems(): |
7c673cae FG |
158 | # XXX: the 'default' zone and zonegroup aren't created until we run RGWRados::init_complete(). |
159 | # issue a 'radosgw-admin user list' command to trigger this | |
160 | rgwadmin(ctx, client, cmd=['user', 'list'], check_status=True) | |
161 | ||
162 | rgwadmin(ctx, client, | |
163 | cmd=['zone', 'placement', 'modify', '--rgw-zone', 'default', | |
31f18b77 FG |
164 | '--placement-id', 'default-placement', |
165 | '--compression', compression], | |
7c673cae | 166 | check_status=True) |
7c673cae FG |
167 | yield |
168 | ||
169 | @contextlib.contextmanager | |
170 | def task(ctx, config): | |
171 | """ | |
7c673cae FG |
172 | For example, to run rgw on all clients:: |
173 | ||
174 | tasks: | |
175 | - ceph: | |
176 | - rgw: | |
177 | ||
178 | To only run on certain clients:: | |
179 | ||
180 | tasks: | |
181 | - ceph: | |
182 | - rgw: [client.0, client.3] | |
183 | ||
184 | or | |
185 | ||
186 | tasks: | |
187 | - ceph: | |
188 | - rgw: | |
189 | client.0: | |
190 | client.3: | |
191 | ||
7c673cae FG |
192 | To run radosgw through valgrind: |
193 | ||
194 | tasks: | |
195 | - ceph: | |
196 | - rgw: | |
197 | client.0: | |
198 | valgrind: [--tool=memcheck] | |
199 | client.3: | |
200 | valgrind: [--tool=memcheck] | |
7c673cae FG |
201 | """ |
202 | if config is None: | |
203 | config = dict(('client.{id}'.format(id=id_), None) | |
204 | for id_ in teuthology.all_roles_of_type( | |
205 | ctx.cluster, 'client')) | |
206 | elif isinstance(config, list): | |
207 | config = dict((name, None) for name in config) | |
208 | ||
209 | overrides = ctx.config.get('overrides', {}) | |
210 | teuthology.deep_merge(config, overrides.get('rgw', {})) | |
211 | ||
7c673cae FG |
212 | role_endpoints = assign_ports(ctx, config) |
213 | ctx.rgw = argparse.Namespace() | |
214 | ctx.rgw.role_endpoints = role_endpoints | |
7c673cae | 215 | |
31f18b77 FG |
216 | ctx.rgw.ec_data_pool = bool(config.pop('ec-data-pool', False)) |
217 | ctx.rgw.erasure_code_profile = config.pop('erasure_code_profile', {}) | |
218 | ctx.rgw.cache_pools = bool(config.pop('cache-pools', False)) | |
219 | ctx.rgw.frontend = config.pop('frontend', 'civetweb') | |
220 | ctx.rgw.compression_type = config.pop('compression type', None) | |
7c673cae | 221 | ctx.rgw.config = config |
7c673cae | 222 | |
31f18b77 FG |
223 | subtasks = [ |
224 | lambda: create_pools(ctx=ctx, config=config), | |
225 | ] | |
226 | if ctx.rgw.compression_type: | |
7c673cae | 227 | subtasks.extend([ |
31f18b77 FG |
228 | lambda: configure_compression(ctx=ctx, config=config, |
229 | compression=ctx.rgw.compression_type), | |
7c673cae | 230 | ]) |
31f18b77 FG |
231 | subtasks.extend([ |
232 | lambda: start_rgw(ctx=ctx, config=config), | |
233 | ]) | |
7c673cae | 234 | |
7c673cae FG |
235 | with contextutil.nested(*subtasks): |
236 | yield |