]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | """ |
2 | Deploy and configure Tempest for Teuthology | |
3 | """ | |
f67539c2 | 4 | import configparser |
11fdf7f2 TL |
5 | import contextlib |
6 | import logging | |
7 | ||
8 | from teuthology import misc as teuthology | |
9 | from teuthology import contextutil | |
9f95a23c | 10 | from teuthology.exceptions import ConfigError |
11fdf7f2 TL |
11 | from teuthology.orchestra import run |
12 | ||
13 | log = logging.getLogger(__name__) | |
14 | ||
15 | ||
16 | def get_tempest_dir(ctx): | |
17 | return '{tdir}/tempest'.format(tdir=teuthology.get_testdir(ctx)) | |
18 | ||
19 | def run_in_tempest_dir(ctx, client, cmdargs, **kwargs): | |
20 | ctx.cluster.only(client).run( | |
21 | args=[ 'cd', get_tempest_dir(ctx), run.Raw('&&'), ] + cmdargs, | |
22 | **kwargs | |
23 | ) | |
24 | ||
25 | def run_in_tempest_rgw_dir(ctx, client, cmdargs, **kwargs): | |
26 | ctx.cluster.only(client).run( | |
27 | args=[ 'cd', get_tempest_dir(ctx) + '/rgw', run.Raw('&&'), ] + cmdargs, | |
28 | **kwargs | |
29 | ) | |
30 | ||
31 | def run_in_tempest_venv(ctx, client, cmdargs, **kwargs): | |
32 | run_in_tempest_dir(ctx, client, | |
33 | [ 'source', | |
34 | '.tox/venv/bin/activate', | |
35 | run.Raw('&&') | |
36 | ] + cmdargs, **kwargs) | |
37 | ||
38 | @contextlib.contextmanager | |
39 | def download(ctx, config): | |
40 | """ | |
41 | Download the Tempest from github. | |
42 | Remove downloaded file upon exit. | |
43 | ||
44 | The context passed in should be identical to the context | |
45 | passed in to the main task. | |
46 | """ | |
47 | assert isinstance(config, dict) | |
48 | log.info('Downloading Tempest...') | |
49 | for (client, cconf) in config.items(): | |
50 | ctx.cluster.only(client).run( | |
51 | args=[ | |
52 | 'git', 'clone', | |
53 | '-b', cconf.get('force-branch', 'master'), | |
54 | 'https://github.com/openstack/tempest.git', | |
55 | get_tempest_dir(ctx) | |
56 | ], | |
57 | ) | |
58 | ||
59 | sha1 = cconf.get('sha1') | |
60 | if sha1 is not None: | |
61 | run_in_tempest_dir(ctx, client, [ 'git', 'reset', '--hard', sha1 ]) | |
62 | try: | |
63 | yield | |
64 | finally: | |
65 | log.info('Removing Tempest...') | |
66 | for client in config: | |
67 | ctx.cluster.only(client).run( | |
68 | args=[ 'rm', '-rf', get_tempest_dir(ctx) ], | |
69 | ) | |
70 | ||
71 | def get_toxvenv_dir(ctx): | |
72 | return ctx.tox.venv_path | |
73 | ||
74 | @contextlib.contextmanager | |
75 | def setup_venv(ctx, config): | |
76 | """ | |
77 | Setup the virtualenv for Tempest using tox. | |
78 | """ | |
79 | assert isinstance(config, dict) | |
80 | log.info('Setting up virtualenv for Tempest') | |
81 | for (client, _) in config.items(): | |
82 | run_in_tempest_dir(ctx, client, | |
83 | [ '{tvdir}/bin/tox'.format(tvdir=get_toxvenv_dir(ctx)), | |
84 | '-e', 'venv', '--notest' | |
85 | ]) | |
86 | yield | |
87 | ||
88 | def setup_logging(ctx, cpar): | |
89 | cpar.set('DEFAULT', 'log_dir', teuthology.get_archive_dir(ctx)) | |
90 | cpar.set('DEFAULT', 'log_file', 'tempest.log') | |
91 | ||
92 | def to_config(config, params, section, cpar): | |
93 | for (k, v) in config[section].items(): | |
e306af50 | 94 | if isinstance(v, str): |
11fdf7f2 | 95 | v = v.format(**params) |
e306af50 TL |
96 | elif isinstance(v, bool): |
97 | v = 'true' if v else 'false' | |
98 | else: | |
99 | v = str(v) | |
11fdf7f2 TL |
100 | cpar.set(section, k, v) |
101 | ||
102 | @contextlib.contextmanager | |
103 | def configure_instance(ctx, config): | |
104 | assert isinstance(config, dict) | |
105 | log.info('Configuring Tempest') | |
106 | ||
11fdf7f2 TL |
107 | for (client, cconfig) in config.items(): |
108 | run_in_tempest_venv(ctx, client, | |
109 | [ | |
110 | 'tempest', | |
111 | 'init', | |
112 | '--workspace-path', | |
113 | get_tempest_dir(ctx) + '/workspace.yaml', | |
114 | 'rgw' | |
115 | ]) | |
116 | ||
117 | # prepare the config file | |
118 | tetcdir = '{tdir}/rgw/etc'.format(tdir=get_tempest_dir(ctx)) | |
119 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
120 | local_conf = remote.get_file(tetcdir + '/tempest.conf.sample') | |
121 | ||
122 | # fill the params dictionary which allows to use templatized configs | |
123 | keystone_role = cconfig.get('use-keystone-role', None) | |
124 | if keystone_role is None \ | |
125 | or keystone_role not in ctx.keystone.public_endpoints: | |
126 | raise ConfigError('the use-keystone-role is misconfigured') | |
127 | public_host, public_port = ctx.keystone.public_endpoints[keystone_role] | |
128 | params = { | |
129 | 'keystone_public_host': public_host, | |
130 | 'keystone_public_port': str(public_port), | |
131 | } | |
132 | ||
e306af50 | 133 | cpar = configparser.ConfigParser() |
11fdf7f2 TL |
134 | cpar.read(local_conf) |
135 | setup_logging(ctx, cpar) | |
136 | to_config(cconfig, params, 'auth', cpar) | |
137 | to_config(cconfig, params, 'identity', cpar) | |
138 | to_config(cconfig, params, 'object-storage', cpar) | |
139 | to_config(cconfig, params, 'object-storage-feature-enabled', cpar) | |
9f95a23c | 140 | cpar.write(open(local_conf, 'w+')) |
11fdf7f2 TL |
141 | |
142 | remote.put_file(local_conf, tetcdir + '/tempest.conf') | |
143 | yield | |
144 | ||
145 | @contextlib.contextmanager | |
146 | def run_tempest(ctx, config): | |
147 | assert isinstance(config, dict) | |
148 | log.info('Configuring Tempest') | |
149 | ||
150 | for (client, cconf) in config.items(): | |
f67539c2 TL |
151 | blocklist = cconf.get('blocklist', []) |
152 | assert isinstance(blocklist, list) | |
11fdf7f2 TL |
153 | run_in_tempest_venv(ctx, client, |
154 | [ | |
155 | 'tempest', | |
156 | 'run', | |
157 | '--workspace-path', | |
158 | get_tempest_dir(ctx) + '/workspace.yaml', | |
159 | '--workspace', | |
160 | 'rgw', | |
e306af50 | 161 | '--regex', '^tempest.api.object_storage', |
f67539c2 | 162 | '--black-regex', '|'.join(blocklist) |
11fdf7f2 TL |
163 | ]) |
164 | try: | |
165 | yield | |
166 | finally: | |
167 | pass | |
168 | ||
169 | ||
170 | @contextlib.contextmanager | |
171 | def task(ctx, config): | |
172 | """ | |
173 | Deploy and run Tempest's object storage campaign | |
174 | ||
175 | Example of configuration: | |
176 | ||
177 | overrides: | |
178 | ceph: | |
179 | conf: | |
180 | client: | |
e306af50 | 181 | rgw keystone api version: 3 |
11fdf7f2 TL |
182 | rgw keystone accepted roles: admin,Member |
183 | rgw keystone implicit tenants: true | |
184 | rgw keystone accepted admin roles: admin | |
185 | rgw swift enforce content length: true | |
186 | rgw swift account in url: true | |
187 | rgw swift versioning enabled: true | |
e306af50 TL |
188 | rgw keystone admin domain: Default |
189 | rgw keystone admin user: admin | |
190 | rgw keystone admin password: ADMIN | |
191 | rgw keystone admin project: admin | |
11fdf7f2 TL |
192 | tasks: |
193 | # typically, the task should be preceded with install, ceph, tox, | |
194 | # keystone and rgw. Tox and Keystone are specific requirements | |
195 | # of tempest.py. | |
196 | - rgw: | |
197 | # it's important to match the prefix with the endpoint's URL | |
198 | # in Keystone. Additionally, if we want to test /info and its | |
199 | # accompanying stuff, the whole Swift API must be put in root | |
200 | # of the whole URL hierarchy (read: frontend_prefix == /swift). | |
201 | frontend_prefix: /swift | |
202 | client.0: | |
203 | use-keystone-role: client.0 | |
204 | - tempest: | |
205 | client.0: | |
206 | force-branch: master | |
207 | use-keystone-role: client.0 | |
208 | auth: | |
209 | admin_username: admin | |
210 | admin_project_name: admin | |
211 | admin_password: ADMIN | |
212 | admin_domain_name: Default | |
213 | identity: | |
214 | uri: http://{keystone_public_host}:{keystone_public_port}/v2.0/ | |
215 | uri_v3: http://{keystone_public_host}:{keystone_public_port}/v3/ | |
216 | admin_role: admin | |
217 | object-storage: | |
218 | reseller_admin_role: admin | |
219 | object-storage-feature-enabled: | |
220 | container_sync: false | |
221 | discoverability: false | |
f67539c2 | 222 | blocklist: |
11fdf7f2 TL |
223 | # please strip half of these items after merging PRs #15369 |
224 | # and #12704 | |
225 | - .*test_list_containers_reverse_order.* | |
226 | - .*test_list_container_contents_with_end_marker.* | |
227 | - .*test_delete_non_empty_container.* | |
228 | - .*test_container_synchronization.* | |
229 | - .*test_get_object_after_expiration_time.* | |
230 | - .*test_create_object_with_transfer_encoding.* | |
231 | """ | |
232 | assert config is None or isinstance(config, list) \ | |
233 | or isinstance(config, dict), \ | |
234 | 'task tempest only supports a list or dictionary for configuration' | |
235 | ||
236 | if not ctx.tox: | |
237 | raise ConfigError('tempest must run after the tox task') | |
238 | if not ctx.keystone: | |
239 | raise ConfigError('tempest must run after the keystone task') | |
240 | ||
241 | all_clients = ['client.{id}'.format(id=id_) | |
242 | for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] | |
243 | if config is None: | |
244 | config = all_clients | |
245 | if isinstance(config, list): | |
246 | config = dict.fromkeys(config) | |
11fdf7f2 TL |
247 | |
248 | overrides = ctx.config.get('overrides', {}) | |
249 | # merge each client section, not the top level. | |
9f95a23c | 250 | for client in config.keys(): |
11fdf7f2 TL |
251 | if not config[client]: |
252 | config[client] = {} | |
253 | teuthology.deep_merge(config[client], overrides.get('keystone', {})) | |
254 | ||
255 | log.debug('Tempest config is %s', config) | |
256 | ||
257 | with contextutil.nested( | |
258 | lambda: download(ctx=ctx, config=config), | |
259 | lambda: setup_venv(ctx=ctx, config=config), | |
260 | lambda: configure_instance(ctx=ctx, config=config), | |
261 | lambda: run_tempest(ctx=ctx, config=config), | |
262 | ): | |
263 | yield |