]>
git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/calamari_nosetests.py
7 from cStringIO
import StringIO
8 from teuthology
import contextutil
9 from teuthology
import misc
10 from teuthology
import packaging
11 from teuthology
.orchestra
import run
13 log
= logging
.getLogger(__name__
)
15 # extra stuff we need to do our job here
20 # stuff that would be in a devmode install, but should be
21 # installed in the system for running nosetests against
22 # a production install.
23 EXTRA_NOSETEST_PKGS
= [
29 def find_client0(cluster
):
30 ''' Find remote that has client.0 role, or None '''
31 for rem
, roles
in cluster
.remotes
.iteritems():
32 if 'client.0' in roles
:
37 def pip(remote
, package
, venv
=None, uninstall
=False, force
=False):
38 ''' {un}install a package with pip, possibly in a virtualenv '''
40 pip
= os
.path
.join(venv
, 'bin', 'pip')
43 args
= ['sudo', 'pip']
46 args
.extend(['uninstall', '-y'])
48 args
.append('install')
56 @contextlib.contextmanager
57 def install_epel(remote
):
58 ''' install a disabled-by-default epel repo config file '''
61 if remote
.os
.package_type
== 'deb':
65 distromajor
= remote
.os
.version
.split('.')[0]
67 repofiledata
= textwrap
.dedent('''
70 metalink=http://mirrors.fedoraproject.org/metalink?repo=epel-{version}&arch=$basearch
73 ''').format(version
=distromajor
)
75 misc
.create_file(remote
, '/etc/yum.repos.d/epel.repo',
76 data
=repofiledata
, sudo
=True)
77 remote
.run(args
='sudo yum clean all')
82 misc
.delete_file(remote
, '/etc/yum.repos.d/epel.repo', sudo
=True)
85 def enable_epel(remote
, enable
=True):
86 ''' enable/disable the epel repo '''
87 args
= 'sudo sed -i'.split()
89 args
.extend(['s/enabled=0/enabled=1/'])
91 args
.extend(['s/enabled=1/enabled=0/'])
92 args
.extend(['/etc/yum.repos.d/epel.repo'])
95 remote
.run(args
='sudo yum clean all')
98 @contextlib.contextmanager
99 def install_extra_pkgs(client
):
100 ''' Install EXTRA_PKGS '''
102 for pkg
in EXTRA_PKGS
:
103 packaging
.install_package(pkg
, client
)
107 for pkg
in EXTRA_PKGS
:
108 packaging
.remove_package(pkg
, client
)
111 @contextlib.contextmanager
112 def clone_calamari(config
, client
):
113 ''' clone calamari source into current directory on remote '''
114 branch
= config
.get('calamari_branch', 'master')
115 url
= config
.get('calamari_giturl', 'git://github.com/ceph/calamari')
118 # ensure branch is present (clone -b will succeed even if
119 # the branch doesn't exist, falling back to master)
121 args
='git ls-remote %s %s' % (url
, branch
),
123 label
='check for calamari branch %s existence' % branch
125 if len(out
.getvalue()) == 0:
126 raise RuntimeError("Calamari branch %s doesn't exist" % branch
)
127 client
.run(args
='git clone -b %s %s' % (branch
, url
))
130 # sudo python setup.py develop may have left some root files around
131 client
.run(args
='sudo rm -rf calamari')
134 @contextlib.contextmanager
135 def write_info_yaml(cluster
, client
):
136 ''' write info.yaml to client for nosetests '''
140 rem
.name
: {'roles': roles
}
141 for rem
, roles
in cluster
.remotes
.iteritems()
144 misc
.create_file(client
, 'calamari/info.yaml',
145 data
=yaml
.safe_dump(info
, default_flow_style
=False))
148 misc
.delete_file(client
, 'calamari/info.yaml')
151 @contextlib.contextmanager
152 def write_test_conf(client
):
153 ''' write calamari/tests/test.conf to client for nosetests '''
155 testconf
= textwrap
.dedent('''
158 calamari_control = external
159 ceph_control = external
163 embedded_timeout_factor = 1
164 external_timeout_factor = 3
165 external_cluster_path = info.yaml
167 misc
.create_file(client
, 'calamari/tests/test.conf', data
=testconf
)
171 misc
.delete_file(client
, 'calamari/tests/test.conf')
174 @contextlib.contextmanager
175 def prepare_nosetest_env(client
):
177 # extra dependencies that would be in the devmode venv
178 if client
.os
.package_type
== 'rpm':
179 enable_epel(client
, enable
=True)
180 for package
in EXTRA_NOSETEST_PKGS
:
181 packaging
.install_package(package
, client
)
182 if client
.os
.package_type
== 'rpm':
183 enable_epel(client
, enable
=False)
185 # install nose itself into the calamari venv, force it in case it's
186 # already installed in the system, so we can invoke it by path without
187 # fear that it's not present
188 pip(client
, 'nose', venv
='/opt/calamari/venv', force
=True)
190 # install a later version of requests into the venv as well
192 pip(client
, 'requests', venv
='/opt/calamari/venv', force
=True)
194 # link (setup.py develop) calamari/rest-api into the production venv
195 # because production does not include calamari_rest.management, needed
196 # for test_rest_api.py's ApiIntrospection
197 args
= 'cd calamari/rest-api'.split() + [run
.Raw(';')] + \
198 'sudo /opt/calamari/venv/bin/python setup.py develop'.split()
199 client
.run(args
=args
)
201 # because, at least in Python 2.6/Centos, site.py uses
202 # 'os.path.exists()' to process .pth file entries, and exists() uses
203 # access(2) to check for existence, all the paths leading up to
204 # $HOME/calamari/rest-api need to be searchable by all users of
205 # the package, which will include the WSGI/Django app, running
206 # as the Apache user. So make them all world-read-and-execute.
207 args
= 'sudo chmod a+x'.split() + \
208 ['.', './calamari', './calamari/rest-api']
209 client
.run(args
=args
)
211 # make one dummy request just to get the WSGI app to do
212 # all its log creation here, before the chmod below (I'm
213 # looking at you, graphite -- /var/log/calamari/info.log and
214 # /var/log/calamari/exception.log)
215 client
.run(args
='wget -q -O /dev/null http://localhost')
217 # /var/log/calamari/* is root-or-apache write-only
218 client
.run(args
='sudo chmod a+w /var/log/calamari/*')
223 args
= 'cd calamari/rest-api'.split() + [run
.Raw(';')] + \
224 'sudo /opt/calamari/venv/bin/python setup.py develop -u'.split()
225 client
.run(args
=args
)
226 for pkg
in ('nose', 'requests'):
227 pip(client
, pkg
, venv
='/opt/calamari/venv', uninstall
=True)
228 for package
in EXTRA_NOSETEST_PKGS
:
229 packaging
.remove_package(package
, client
)
232 @contextlib.contextmanager
233 def run_nosetests(client
):
234 ''' Actually run the tests '''
239 'CALAMARI_CONFIG=/etc/calamari/calamari.conf',
240 '/opt/calamari/venv/bin/nosetests',
244 client
.run(args
=args
)
248 @contextlib.contextmanager
249 def task(ctx
, config
):
251 Run Calamari tests against an instance set up by 'calamari_server'.
253 -- clone the Calamari source into $HOME (see options)
254 -- write calamari/info.yaml describing the cluster
255 -- write calamari/tests/test.conf containing
256 'external' for calamari_control and ceph_control
257 'bootstrap = False' to disable test bootstrapping (installing minions)
258 no api_url necessary (inferred from client.0)
259 'external_cluster_path = info.yaml'
260 -- modify the production Calamari install to allow test runs:
261 install nose in the venv
262 install EXTRA_NOSETEST_PKGS
263 link in, with setup.py develop, calamari_rest (for ApiIntrospection)
264 -- set CALAMARI_CONFIG to point to /etc/calamari/calamari.conf
265 -- nosetests -v tests/
268 calamari_giturl: url from which to git clone calamari
269 (default: git://github.com/ceph/calamari)
270 calamari_branch: git branch of calamari to check out
273 Note: the tests must find a clean cluster, so don't forget to
274 set the crush default type appropriately, or install min_size OSD hosts
276 client0
= find_client0(ctx
.cluster
)
278 raise RuntimeError("must have client.0 role")
280 with contextutil
.nested(
281 lambda: install_epel(client0
),
282 lambda: install_extra_pkgs(client0
),
283 lambda: clone_calamari(config
, client0
),
284 lambda: write_info_yaml(ctx
.cluster
, client0
),
285 lambda: write_test_conf(client0
),
286 lambda: prepare_nosetest_env(client0
),
287 lambda: run_nosetests(client0
),