]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | """ |
2 | Deploy and configure Tempest for Teuthology | |
3 | """ | |
4 | import contextlib | |
5 | import logging | |
6 | ||
e306af50 TL |
7 | from six.moves import configparser |
8 | ||
11fdf7f2 TL |
9 | from teuthology import misc as teuthology |
10 | from teuthology import contextutil | |
9f95a23c | 11 | from teuthology.exceptions import ConfigError |
11fdf7f2 TL |
12 | from teuthology.orchestra import run |
13 | ||
14 | log = logging.getLogger(__name__) | |
15 | ||
16 | ||
17 | def get_tempest_dir(ctx): | |
18 | return '{tdir}/tempest'.format(tdir=teuthology.get_testdir(ctx)) | |
19 | ||
20 | def run_in_tempest_dir(ctx, client, cmdargs, **kwargs): | |
21 | ctx.cluster.only(client).run( | |
22 | args=[ 'cd', get_tempest_dir(ctx), run.Raw('&&'), ] + cmdargs, | |
23 | **kwargs | |
24 | ) | |
25 | ||
26 | def run_in_tempest_rgw_dir(ctx, client, cmdargs, **kwargs): | |
27 | ctx.cluster.only(client).run( | |
28 | args=[ 'cd', get_tempest_dir(ctx) + '/rgw', run.Raw('&&'), ] + cmdargs, | |
29 | **kwargs | |
30 | ) | |
31 | ||
32 | def run_in_tempest_venv(ctx, client, cmdargs, **kwargs): | |
33 | run_in_tempest_dir(ctx, client, | |
34 | [ 'source', | |
35 | '.tox/venv/bin/activate', | |
36 | run.Raw('&&') | |
37 | ] + cmdargs, **kwargs) | |
38 | ||
39 | @contextlib.contextmanager | |
40 | def download(ctx, config): | |
41 | """ | |
42 | Download the Tempest from github. | |
43 | Remove downloaded file upon exit. | |
44 | ||
45 | The context passed in should be identical to the context | |
46 | passed in to the main task. | |
47 | """ | |
48 | assert isinstance(config, dict) | |
49 | log.info('Downloading Tempest...') | |
50 | for (client, cconf) in config.items(): | |
51 | ctx.cluster.only(client).run( | |
52 | args=[ | |
53 | 'git', 'clone', | |
54 | '-b', cconf.get('force-branch', 'master'), | |
55 | 'https://github.com/openstack/tempest.git', | |
56 | get_tempest_dir(ctx) | |
57 | ], | |
58 | ) | |
59 | ||
60 | sha1 = cconf.get('sha1') | |
61 | if sha1 is not None: | |
62 | run_in_tempest_dir(ctx, client, [ 'git', 'reset', '--hard', sha1 ]) | |
63 | try: | |
64 | yield | |
65 | finally: | |
66 | log.info('Removing Tempest...') | |
67 | for client in config: | |
68 | ctx.cluster.only(client).run( | |
69 | args=[ 'rm', '-rf', get_tempest_dir(ctx) ], | |
70 | ) | |
71 | ||
72 | def get_toxvenv_dir(ctx): | |
73 | return ctx.tox.venv_path | |
74 | ||
75 | @contextlib.contextmanager | |
76 | def setup_venv(ctx, config): | |
77 | """ | |
78 | Setup the virtualenv for Tempest using tox. | |
79 | """ | |
80 | assert isinstance(config, dict) | |
81 | log.info('Setting up virtualenv for Tempest') | |
82 | for (client, _) in config.items(): | |
83 | run_in_tempest_dir(ctx, client, | |
84 | [ '{tvdir}/bin/tox'.format(tvdir=get_toxvenv_dir(ctx)), | |
85 | '-e', 'venv', '--notest' | |
86 | ]) | |
87 | yield | |
88 | ||
89 | def setup_logging(ctx, cpar): | |
90 | cpar.set('DEFAULT', 'log_dir', teuthology.get_archive_dir(ctx)) | |
91 | cpar.set('DEFAULT', 'log_file', 'tempest.log') | |
92 | ||
93 | def to_config(config, params, section, cpar): | |
94 | for (k, v) in config[section].items(): | |
e306af50 | 95 | if isinstance(v, str): |
11fdf7f2 | 96 | v = v.format(**params) |
e306af50 TL |
97 | elif isinstance(v, bool): |
98 | v = 'true' if v else 'false' | |
99 | else: | |
100 | v = str(v) | |
11fdf7f2 TL |
101 | cpar.set(section, k, v) |
102 | ||
103 | @contextlib.contextmanager | |
104 | def configure_instance(ctx, config): | |
105 | assert isinstance(config, dict) | |
106 | log.info('Configuring Tempest') | |
107 | ||
11fdf7f2 TL |
108 | for (client, cconfig) in config.items(): |
109 | run_in_tempest_venv(ctx, client, | |
110 | [ | |
111 | 'tempest', | |
112 | 'init', | |
113 | '--workspace-path', | |
114 | get_tempest_dir(ctx) + '/workspace.yaml', | |
115 | 'rgw' | |
116 | ]) | |
117 | ||
118 | # prepare the config file | |
119 | tetcdir = '{tdir}/rgw/etc'.format(tdir=get_tempest_dir(ctx)) | |
120 | (remote,) = ctx.cluster.only(client).remotes.keys() | |
121 | local_conf = remote.get_file(tetcdir + '/tempest.conf.sample') | |
122 | ||
123 | # fill the params dictionary which allows to use templatized configs | |
124 | keystone_role = cconfig.get('use-keystone-role', None) | |
125 | if keystone_role is None \ | |
126 | or keystone_role not in ctx.keystone.public_endpoints: | |
127 | raise ConfigError('the use-keystone-role is misconfigured') | |
128 | public_host, public_port = ctx.keystone.public_endpoints[keystone_role] | |
129 | params = { | |
130 | 'keystone_public_host': public_host, | |
131 | 'keystone_public_port': str(public_port), | |
132 | } | |
133 | ||
e306af50 | 134 | cpar = configparser.ConfigParser() |
11fdf7f2 TL |
135 | cpar.read(local_conf) |
136 | setup_logging(ctx, cpar) | |
137 | to_config(cconfig, params, 'auth', cpar) | |
138 | to_config(cconfig, params, 'identity', cpar) | |
139 | to_config(cconfig, params, 'object-storage', cpar) | |
140 | to_config(cconfig, params, 'object-storage-feature-enabled', cpar) | |
9f95a23c | 141 | cpar.write(open(local_conf, 'w+')) |
11fdf7f2 TL |
142 | |
143 | remote.put_file(local_conf, tetcdir + '/tempest.conf') | |
144 | yield | |
145 | ||
146 | @contextlib.contextmanager | |
147 | def run_tempest(ctx, config): | |
148 | assert isinstance(config, dict) | |
149 | log.info('Configuring Tempest') | |
150 | ||
151 | for (client, cconf) in config.items(): | |
152 | blacklist = cconf.get('blacklist', []) | |
153 | assert isinstance(blacklist, list) | |
154 | run_in_tempest_venv(ctx, client, | |
155 | [ | |
156 | 'tempest', | |
157 | 'run', | |
158 | '--workspace-path', | |
159 | get_tempest_dir(ctx) + '/workspace.yaml', | |
160 | '--workspace', | |
161 | 'rgw', | |
e306af50 TL |
162 | '--regex', '^tempest.api.object_storage', |
163 | '--black-regex', '|'.join(blacklist) | |
11fdf7f2 TL |
164 | ]) |
165 | try: | |
166 | yield | |
167 | finally: | |
168 | pass | |
169 | ||
170 | ||
171 | @contextlib.contextmanager | |
172 | def task(ctx, config): | |
173 | """ | |
174 | Deploy and run Tempest's object storage campaign | |
175 | ||
176 | Example of configuration: | |
177 | ||
178 | overrides: | |
179 | ceph: | |
180 | conf: | |
181 | client: | |
e306af50 | 182 | rgw keystone api version: 3 |
11fdf7f2 TL |
183 | rgw keystone accepted roles: admin,Member |
184 | rgw keystone implicit tenants: true | |
185 | rgw keystone accepted admin roles: admin | |
186 | rgw swift enforce content length: true | |
187 | rgw swift account in url: true | |
188 | rgw swift versioning enabled: true | |
e306af50 TL |
189 | rgw keystone admin domain: Default |
190 | rgw keystone admin user: admin | |
191 | rgw keystone admin password: ADMIN | |
192 | rgw keystone admin project: admin | |
11fdf7f2 TL |
193 | tasks: |
194 | # typically, the task should be preceded with install, ceph, tox, | |
195 | # keystone and rgw. Tox and Keystone are specific requirements | |
196 | # of tempest.py. | |
197 | - rgw: | |
198 | # it's important to match the prefix with the endpoint's URL | |
199 | # in Keystone. Additionally, if we want to test /info and its | |
200 | # accompanying stuff, the whole Swift API must be put in root | |
201 | # of the whole URL hierarchy (read: frontend_prefix == /swift). | |
202 | frontend_prefix: /swift | |
203 | client.0: | |
204 | use-keystone-role: client.0 | |
205 | - tempest: | |
206 | client.0: | |
207 | force-branch: master | |
208 | use-keystone-role: client.0 | |
209 | auth: | |
210 | admin_username: admin | |
211 | admin_project_name: admin | |
212 | admin_password: ADMIN | |
213 | admin_domain_name: Default | |
214 | identity: | |
215 | uri: http://{keystone_public_host}:{keystone_public_port}/v2.0/ | |
216 | uri_v3: http://{keystone_public_host}:{keystone_public_port}/v3/ | |
217 | admin_role: admin | |
218 | object-storage: | |
219 | reseller_admin_role: admin | |
220 | object-storage-feature-enabled: | |
221 | container_sync: false | |
222 | discoverability: false | |
223 | blacklist: | |
224 | # please strip half of these items after merging PRs #15369 | |
225 | # and #12704 | |
226 | - .*test_list_containers_reverse_order.* | |
227 | - .*test_list_container_contents_with_end_marker.* | |
228 | - .*test_delete_non_empty_container.* | |
229 | - .*test_container_synchronization.* | |
230 | - .*test_get_object_after_expiration_time.* | |
231 | - .*test_create_object_with_transfer_encoding.* | |
232 | """ | |
233 | assert config is None or isinstance(config, list) \ | |
234 | or isinstance(config, dict), \ | |
235 | 'task tempest only supports a list or dictionary for configuration' | |
236 | ||
237 | if not ctx.tox: | |
238 | raise ConfigError('tempest must run after the tox task') | |
239 | if not ctx.keystone: | |
240 | raise ConfigError('tempest must run after the keystone task') | |
241 | ||
242 | all_clients = ['client.{id}'.format(id=id_) | |
243 | for id_ in teuthology.all_roles_of_type(ctx.cluster, 'client')] | |
244 | if config is None: | |
245 | config = all_clients | |
246 | if isinstance(config, list): | |
247 | config = dict.fromkeys(config) | |
11fdf7f2 TL |
248 | |
249 | overrides = ctx.config.get('overrides', {}) | |
250 | # merge each client section, not the top level. | |
9f95a23c | 251 | for client in config.keys(): |
11fdf7f2 TL |
252 | if not config[client]: |
253 | config[client] = {} | |
254 | teuthology.deep_merge(config[client], overrides.get('keystone', {})) | |
255 | ||
256 | log.debug('Tempest config is %s', config) | |
257 | ||
258 | with contextutil.nested( | |
259 | lambda: download(ctx=ctx, config=config), | |
260 | lambda: setup_venv(ctx=ctx, config=config), | |
261 | lambda: configure_instance(ctx=ctx, config=config), | |
262 | lambda: run_tempest(ctx=ctx, config=config), | |
263 | ): | |
264 | yield |