]> git.proxmox.com Git - ceph.git/blob - ceph/qa/tasks/cephfs/test_nfs.py
30d9d6df47b655c64b1c71b7b37baa2921bb8b24
[ceph.git] / ceph / qa / tasks / cephfs / test_nfs.py
1 # NOTE: these tests are not yet compatible with vstart_runner.py.
2 import errno
3 import json
4 import time
5 import logging
6 from io import BytesIO
7
8 from tasks.mgr.mgr_test_case import MgrTestCase
9 from teuthology.exceptions import CommandFailedError
10
11 log = logging.getLogger(__name__)
12
13
14 # TODO Add test for cluster update when ganesha can be deployed on multiple ports.
15 class TestNFS(MgrTestCase):
16 def _cmd(self, *args):
17 return self.mgr_cluster.mon_manager.raw_cluster_cmd(*args)
18
19 def _nfs_cmd(self, *args):
20 return self._cmd("nfs", *args)
21
22 def _orch_cmd(self, *args):
23 return self._cmd("orch", *args)
24
25 def _sys_cmd(self, cmd):
26 cmd[0:0] = ['sudo']
27 ret = self.ctx.cluster.run(args=cmd, check_status=False, stdout=BytesIO(), stderr=BytesIO())
28 stdout = ret[0].stdout
29 if stdout:
30 return stdout.getvalue()
31
32 def setUp(self):
33 super(TestNFS, self).setUp()
34 self.cluster_id = "test"
35 self.export_type = "cephfs"
36 self.pseudo_path = "/cephfs"
37 self.path = "/"
38 self.fs_name = "nfs-cephfs"
39 self.expected_name = "nfs.test"
40 self.sample_export = {
41 "export_id": 1,
42 "path": self.path,
43 "cluster_id": self.cluster_id,
44 "pseudo": self.pseudo_path,
45 "access_type": "RW",
46 "squash": "no_root_squash",
47 "security_label": True,
48 "protocols": [
49 4
50 ],
51 "transports": [
52 "TCP"
53 ],
54 "fsal": {
55 "name": "CEPH",
56 "user_id": "test1",
57 "fs_name": self.fs_name,
58 "sec_label_xattr": ''
59 },
60 "clients": []
61 }
62
63 def _check_nfs_server_status(self):
64 res = self._sys_cmd(['systemctl', 'status', 'nfs-server'])
65 if isinstance(res, bytes) and b'Active: active' in res:
66 self._disable_nfs()
67
68 def _disable_nfs(self):
69 log.info("Disabling NFS")
70 self._sys_cmd(['systemctl', 'disable', 'nfs-server', '--now'])
71
72 def _fetch_nfs_status(self):
73 return self._orch_cmd('ps', f'--service_name={self.expected_name}')
74
75 def _check_nfs_cluster_status(self, expected_status, fail_msg):
76 '''
77 Tests if nfs cluster created or deleted successfully
78 :param expected_status: Status to be verified
79 :param fail_msg: Message to be printed if test failed
80 '''
81 # Wait for few seconds as ganesha daemon takes few seconds to be deleted/created
82 wait_time = 10
83 while wait_time <= 60:
84 time.sleep(wait_time)
85 if expected_status in self._fetch_nfs_status():
86 return
87 wait_time += 10
88 self.fail(fail_msg)
89
90 def _check_auth_ls(self, export_id=1, check_in=False):
91 '''
92 Tests export user id creation or deletion.
93 :param export_id: Denotes export number
94 :param check_in: Check specified export id
95 '''
96 output = self._cmd('auth', 'ls')
97 if check_in:
98 self.assertIn(f'client.{self.cluster_id}{export_id}', output)
99 else:
100 self.assertNotIn(f'client-{self.cluster_id}', output)
101
102 def _test_idempotency(self, cmd_func, cmd_args):
103 '''
104 Test idempotency of commands. It first runs the TestNFS test method
105 for a command and then checks the result of command run again. TestNFS
106 test method has required checks to verify that command works.
107 :param cmd_func: TestNFS method
108 :param cmd_args: nfs command arguments to be run
109 '''
110 cmd_func()
111 ret = self.mgr_cluster.mon_manager.raw_cluster_cmd_result(*cmd_args)
112 if ret != 0:
113 self.fail("Idempotency test failed")
114
115 def _test_create_cluster(self):
116 '''
117 Test single nfs cluster deployment.
118 '''
119 # Disable any running nfs ganesha daemon
120 self._check_nfs_server_status()
121 self._nfs_cmd('cluster', 'create', self.export_type, self.cluster_id)
122 # Check for expected status and daemon name (nfs.<cluster_id>)
123 self._check_nfs_cluster_status('running', 'NFS Ganesha cluster deployment failed')
124
125 def _test_delete_cluster(self):
126 '''
127 Test deletion of a single nfs cluster.
128 '''
129 self._nfs_cmd('cluster', 'delete', self.cluster_id)
130 self._check_nfs_cluster_status('No daemons reported',
131 'NFS Ganesha cluster could not be deleted')
132
133 def _test_list_cluster(self, empty=False):
134 '''
135 Test listing of deployed nfs clusters. If nfs cluster is deployed then
136 it checks for expected cluster id. Otherwise checks nothing is listed.
137 :param empty: If true it denotes no cluster is deployed.
138 '''
139 if empty:
140 cluster_id = ''
141 else:
142 cluster_id = self.cluster_id
143 nfs_output = self._nfs_cmd('cluster', 'ls')
144 self.assertEqual(cluster_id, nfs_output.strip())
145
146 def _create_export(self, export_id, create_fs=False, extra_cmd=None):
147 '''
148 Test creation of a single export.
149 :param export_id: Denotes export number
150 :param create_fs: If false filesytem exists. Otherwise create it.
151 :param extra_cmd: List of extra arguments for creating export.
152 '''
153 if create_fs:
154 self._cmd('fs', 'volume', 'create', self.fs_name)
155 export_cmd = ['nfs', 'export', 'create', 'cephfs', self.fs_name, self.cluster_id]
156 if isinstance(extra_cmd, list):
157 export_cmd.extend(extra_cmd)
158 else:
159 export_cmd.append(self.pseudo_path)
160 # Runs the nfs export create command
161 self._cmd(*export_cmd)
162 # Check if user id for export is created
163 self._check_auth_ls(export_id, check_in=True)
164 res = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'get',
165 f'export-{export_id}', '-'])
166 # Check if export object is created
167 if res == b'':
168 self.fail("Export cannot be created")
169
170 def _create_default_export(self):
171 '''
172 Deploy a single nfs cluster and create export with default options.
173 '''
174 self._test_create_cluster()
175 self._create_export(export_id='1', create_fs=True)
176
177 def _delete_export(self):
178 '''
179 Delete an export.
180 '''
181 self._nfs_cmd('export', 'delete', self.cluster_id, self.pseudo_path)
182 self._check_auth_ls()
183
184 def _test_list_export(self):
185 '''
186 Test listing of created exports.
187 '''
188 nfs_output = json.loads(self._nfs_cmd('export', 'ls', self.cluster_id))
189 self.assertIn(self.pseudo_path, nfs_output)
190
191 def _test_list_detailed(self, sub_vol_path):
192 '''
193 Test listing of created exports with detailed option.
194 :param sub_vol_path: Denotes path of subvolume
195 '''
196 nfs_output = json.loads(self._nfs_cmd('export', 'ls', self.cluster_id, '--detailed'))
197 # Export-1 with default values (access type = rw and path = '\')
198 self.assertDictEqual(self.sample_export, nfs_output[0])
199 # Export-2 with r only
200 self.sample_export['export_id'] = 2
201 self.sample_export['pseudo'] = self.pseudo_path + '1'
202 self.sample_export['access_type'] = 'RO'
203 self.sample_export['fsal']['user_id'] = self.cluster_id + '2'
204 self.assertDictEqual(self.sample_export, nfs_output[1])
205 # Export-3 for subvolume with r only
206 self.sample_export['export_id'] = 3
207 self.sample_export['path'] = sub_vol_path
208 self.sample_export['pseudo'] = self.pseudo_path + '2'
209 self.sample_export['fsal']['user_id'] = self.cluster_id + '3'
210 self.assertDictEqual(self.sample_export, nfs_output[2])
211 # Export-4 for subvolume
212 self.sample_export['export_id'] = 4
213 self.sample_export['pseudo'] = self.pseudo_path + '3'
214 self.sample_export['access_type'] = 'RW'
215 self.sample_export['fsal']['user_id'] = self.cluster_id + '4'
216 self.assertDictEqual(self.sample_export, nfs_output[3])
217
218 def _get_export(self):
219 '''
220 Returns export block in json format
221 '''
222 return json.loads(self._nfs_cmd('export', 'get', self.cluster_id, self.pseudo_path))
223
224 def _test_get_export(self):
225 '''
226 Test fetching of created export.
227 '''
228 nfs_output = self._get_export()
229 self.assertDictEqual(self.sample_export, nfs_output)
230
231 def _check_export_obj_deleted(self, conf_obj=False):
232 '''
233 Test if export or config object are deleted successfully.
234 :param conf_obj: It denotes config object needs to be checked
235 '''
236 rados_obj_ls = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'ls'])
237
238 if b'export-' in rados_obj_ls or (conf_obj and b'conf-nfs' in rados_obj_ls):
239 self.fail("Delete export failed")
240
241 def _get_port_ip_info(self):
242 '''
243 Return port and ip for a cluster
244 '''
245 #{'test': [{'hostname': 'smithi068', 'ip': ['172.21.15.68'], 'port': 2049}]}
246 info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id))['test'][0]
247 return info_output["port"], info_output["ip"][0]
248
249 def _test_mnt(self, pseudo_path, port, ip, check=True):
250 '''
251 Test mounting of created exports
252 :param pseudo_path: It is the pseudo root name
253 :param port: Port of deployed nfs cluster
254 :param ip: IP of deployed nfs cluster
255 :param check: It denotes if i/o testing needs to be done
256 '''
257 try:
258 self.ctx.cluster.run(args=['sudo', 'mount', '-t', 'nfs', '-o', f'port={port}',
259 f'{ip}:{pseudo_path}', '/mnt'])
260 except CommandFailedError as e:
261 # Check if mount failed only when non existing pseudo path is passed
262 if not check and e.exitstatus == 32:
263 return
264 raise
265
266 try:
267 self.ctx.cluster.run(args=['sudo', 'touch', '/mnt/test'])
268 out_mnt = self._sys_cmd(['sudo', 'ls', '/mnt'])
269 self.assertEqual(out_mnt, b'test\n')
270 finally:
271 self.ctx.cluster.run(args=['sudo', 'umount', '/mnt'])
272
273 def _write_to_read_only_export(self, pseudo_path, port, ip):
274 '''
275 Check if write to read only export fails
276 '''
277 try:
278 self._test_mnt(pseudo_path, port, ip)
279 except CommandFailedError as e:
280 # Write to cephfs export should fail for test to pass
281 if e.exitstatus != errno.EPERM:
282 raise
283
284 def test_create_and_delete_cluster(self):
285 '''
286 Test successful creation and deletion of the nfs cluster.
287 '''
288 self._test_create_cluster()
289 self._test_list_cluster()
290 self._test_delete_cluster()
291 # List clusters again to ensure no cluster is shown
292 self._test_list_cluster(empty=True)
293
294 def test_create_delete_cluster_idempotency(self):
295 '''
296 Test idempotency of cluster create and delete commands.
297 '''
298 self._test_idempotency(self._test_create_cluster, ['nfs', 'cluster', 'create', self.export_type,
299 self.cluster_id])
300 self._test_idempotency(self._test_delete_cluster, ['nfs', 'cluster', 'delete', self.cluster_id])
301
302 def test_create_cluster_with_invalid_cluster_id(self):
303 '''
304 Test nfs cluster deployment failure with invalid cluster id.
305 '''
306 try:
307 invalid_cluster_id = '/cluster_test' # Only [A-Za-z0-9-_.] chars are valid
308 self._nfs_cmd('cluster', 'create', self.export_type, invalid_cluster_id)
309 self.fail(f"Cluster successfully created with invalid cluster id {invalid_cluster_id}")
310 except CommandFailedError as e:
311 # Command should fail for test to pass
312 if e.exitstatus != errno.EINVAL:
313 raise
314
315 def test_create_cluster_with_invalid_export_type(self):
316 '''
317 Test nfs cluster deployment failure with invalid export type.
318 '''
319 try:
320 invalid_export_type = 'rgw' # Only cephfs is valid
321 self._nfs_cmd('cluster', 'create', invalid_export_type, self.cluster_id)
322 self.fail(f"Cluster successfully created with invalid export type {invalid_export_type}")
323 except CommandFailedError as e:
324 # Command should fail for test to pass
325 if e.exitstatus != errno.EINVAL:
326 raise
327
328 def test_create_and_delete_export(self):
329 '''
330 Test successful creation and deletion of the cephfs export.
331 '''
332 self._create_default_export()
333 self._test_get_export()
334 port, ip = self._get_port_ip_info()
335 self._test_mnt(self.pseudo_path, port, ip)
336 self._delete_export()
337 # Check if rados export object is deleted
338 self._check_export_obj_deleted()
339 self._test_mnt(self.pseudo_path, port, ip, False)
340 self._test_delete_cluster()
341
342 def test_create_delete_export_idempotency(self):
343 '''
344 Test idempotency of export create and delete commands.
345 '''
346 self._test_idempotency(self._create_default_export, ['nfs', 'export', 'create', 'cephfs',
347 self.fs_name, self.cluster_id,
348 self.pseudo_path])
349 self._test_idempotency(self._delete_export, ['nfs', 'export', 'delete', self.cluster_id,
350 self.pseudo_path])
351 self._test_delete_cluster()
352
353 def test_create_multiple_exports(self):
354 '''
355 Test creating multiple exports with different access type and path.
356 '''
357 # Export-1 with default values (access type = rw and path = '\')
358 self._create_default_export()
359 # Export-2 with r only
360 self._create_export(export_id='2', extra_cmd=[self.pseudo_path+'1', '--readonly'])
361 # Export-3 for subvolume with r only
362 self._cmd('fs', 'subvolume', 'create', self.fs_name, 'sub_vol')
363 fs_path = self._cmd('fs', 'subvolume', 'getpath', self.fs_name, 'sub_vol').strip()
364 self._create_export(export_id='3', extra_cmd=[self.pseudo_path+'2', '--readonly', fs_path])
365 # Export-4 for subvolume
366 self._create_export(export_id='4', extra_cmd=[self.pseudo_path+'3', fs_path])
367 # Check if exports gets listed
368 self._test_list_detailed(fs_path)
369 self._test_delete_cluster()
370 # Check if rados ganesha conf object is deleted
371 self._check_export_obj_deleted(conf_obj=True)
372 self._check_auth_ls()
373
374 def test_exports_on_mgr_restart(self):
375 '''
376 Test export availability on restarting mgr.
377 '''
378 self._create_default_export()
379 # unload and load module will restart the mgr
380 self._unload_module("cephadm")
381 self._load_module("cephadm")
382 self._orch_cmd("set", "backend", "cephadm")
383 # Check if ganesha daemon is running
384 self._check_nfs_cluster_status('running', 'Failed to redeploy NFS Ganesha cluster')
385 # Checks if created export is listed
386 self._test_list_export()
387 port, ip = self._get_port_ip_info()
388 self._test_mnt(self.pseudo_path, port, ip)
389 self._delete_export()
390 self._test_delete_cluster()
391
392 def test_export_create_with_non_existing_fsname(self):
393 '''
394 Test creating export with non-existing filesystem.
395 '''
396 try:
397 fs_name = 'nfs-test'
398 self._test_create_cluster()
399 self._nfs_cmd('export', 'create', 'cephfs', fs_name, self.cluster_id, self.pseudo_path)
400 self.fail(f"Export created with non-existing filesystem {fs_name}")
401 except CommandFailedError as e:
402 # Command should fail for test to pass
403 if e.exitstatus != errno.ENOENT:
404 raise
405 finally:
406 self._test_delete_cluster()
407
408 def test_export_create_with_non_existing_clusterid(self):
409 '''
410 Test creating cephfs export with non-existing nfs cluster.
411 '''
412 try:
413 cluster_id = 'invalidtest'
414 self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, cluster_id, self.pseudo_path)
415 self.fail(f"Export created with non-existing cluster id {cluster_id}")
416 except CommandFailedError as e:
417 # Command should fail for test to pass
418 if e.exitstatus != errno.ENOENT:
419 raise
420
421 def test_export_create_with_relative_pseudo_path_and_root_directory(self):
422 '''
423 Test creating cephfs export with relative or '/' pseudo path.
424 '''
425 def check_pseudo_path(pseudo_path):
426 try:
427 self._nfs_cmd('export', 'create', 'cephfs', self.fs_name, self.cluster_id,
428 pseudo_path)
429 self.fail(f"Export created for {pseudo_path}")
430 except CommandFailedError as e:
431 # Command should fail for test to pass
432 if e.exitstatus != errno.EINVAL:
433 raise
434
435 self._test_create_cluster()
436 self._cmd('fs', 'volume', 'create', self.fs_name)
437 check_pseudo_path('invalidpath')
438 check_pseudo_path('/')
439 check_pseudo_path('//')
440 self._cmd('fs', 'volume', 'rm', self.fs_name, '--yes-i-really-mean-it')
441 self._test_delete_cluster()
442
443 def test_write_to_read_only_export(self):
444 '''
445 Test write to readonly export.
446 '''
447 self._test_create_cluster()
448 self._create_export(export_id='1', create_fs=True, extra_cmd=[self.pseudo_path, '--readonly'])
449 port, ip = self._get_port_ip_info()
450 self._write_to_read_only_export(self.pseudo_path, port, ip)
451 self._test_delete_cluster()
452
453 def test_cluster_info(self):
454 '''
455 Test cluster info outputs correct ip and hostname
456 '''
457 self._test_create_cluster()
458 info_output = json.loads(self._nfs_cmd('cluster', 'info', self.cluster_id))
459 info_ip = info_output[self.cluster_id][0].pop("ip")
460 host_details = {self.cluster_id: [{
461 "hostname": self._sys_cmd(['hostname']).decode("utf-8").strip(),
462 "port": 2049
463 }]}
464 host_ip = self._sys_cmd(['hostname', '-I']).decode("utf-8").split()
465 self.assertDictEqual(info_output, host_details)
466 self.assertTrue(any([ip in info_ip for ip in host_ip]))
467 self._test_delete_cluster()
468
469 def test_cluster_set_reset_user_config(self):
470 '''
471 Test cluster is created using user config and reverts back to default
472 config on reset.
473 '''
474 self._test_create_cluster()
475
476 pool = 'nfs-ganesha'
477 user_id = 'test'
478 fs_name = 'user_test_fs'
479 pseudo_path = '/ceph'
480 self._cmd('fs', 'volume', 'create', fs_name)
481 time.sleep(20)
482 key = self._cmd('auth', 'get-or-create-key', f'client.{user_id}', 'mon',
483 'allow r', 'osd',
484 f'allow rw pool={pool} namespace={self.cluster_id}, allow rw tag cephfs data={fs_name}',
485 'mds', f'allow rw path={self.path}').strip()
486 config = f""" LOG {{
487 Default_log_level = FULL_DEBUG;
488 }}
489
490 EXPORT {{
491 Export_Id = 100;
492 Transports = TCP;
493 Path = /;
494 Pseudo = {pseudo_path};
495 Protocols = 4;
496 Access_Type = RW;
497 Attr_Expiration_Time = 0;
498 Squash = None;
499 FSAL {{
500 Name = CEPH;
501 Filesystem = {fs_name};
502 User_Id = {user_id};
503 Secret_Access_Key = '{key}';
504 }}
505 }}"""
506 port, ip = self._get_port_ip_info()
507 self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'cluster', 'config',
508 'set', self.cluster_id, '-i', '-'], stdin=config)
509 time.sleep(30)
510 res = self._sys_cmd(['rados', '-p', pool, '-N', self.cluster_id, 'get',
511 f'userconf-nfs.{user_id}', '-'])
512 self.assertEqual(config, res.decode('utf-8'))
513 self._test_mnt(pseudo_path, port, ip)
514 self._nfs_cmd('cluster', 'config', 'reset', self.cluster_id)
515 rados_obj_ls = self._sys_cmd(['rados', '-p', 'nfs-ganesha', '-N', self.cluster_id, 'ls'])
516 if b'conf-nfs' not in rados_obj_ls and b'userconf-nfs' in rados_obj_ls:
517 self.fail("User config not deleted")
518 time.sleep(30)
519 self._test_mnt(pseudo_path, port, ip, False)
520 self._cmd('fs', 'volume', 'rm', fs_name, '--yes-i-really-mean-it')
521 self._test_delete_cluster()
522
523 def test_cluster_set_user_config_with_non_existing_clusterid(self):
524 '''
525 Test setting user config for non-existing nfs cluster.
526 '''
527 try:
528 cluster_id = 'invalidtest'
529 self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'cluster',
530 'config', 'set', self.cluster_id, '-i', '-'], stdin='testing')
531 self.fail(f"User config set for non-existing cluster {cluster_id}")
532 except CommandFailedError as e:
533 # Command should fail for test to pass
534 if e.exitstatus != errno.ENOENT:
535 raise
536
537 def test_cluster_reset_user_config_with_non_existing_clusterid(self):
538 '''
539 Test resetting user config for non-existing nfs cluster.
540 '''
541 try:
542 cluster_id = 'invalidtest'
543 self._nfs_cmd('cluster', 'config', 'reset', cluster_id)
544 self.fail(f"User config reset for non-existing cluster {cluster_id}")
545 except CommandFailedError as e:
546 # Command should fail for test to pass
547 if e.exitstatus != errno.ENOENT:
548 raise
549
550 def test_update_export(self):
551 '''
552 Test update of exports
553 '''
554 self._create_default_export()
555 port, ip = self._get_port_ip_info()
556 self._test_mnt(self.pseudo_path, port, ip)
557 export_block = self._get_export()
558 new_pseudo_path = '/testing'
559 export_block['pseudo'] = new_pseudo_path
560 export_block['access_type'] = 'RO'
561 self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'export', 'update', '-i', '-'],
562 stdin=json.dumps(export_block))
563 self._check_nfs_cluster_status('running', 'NFS Ganesha cluster restart failed')
564 self._write_to_read_only_export(new_pseudo_path, port, ip)
565 self._test_delete_cluster()
566
567 def test_update_export_with_invalid_values(self):
568 '''
569 Test update of export with invalid values
570 '''
571 self._create_default_export()
572 export_block = self._get_export()
573
574 def update_with_invalid_values(key, value, fsal=False):
575 export_block_new = dict(export_block)
576 if fsal:
577 export_block_new['fsal'] = dict(export_block['fsal'])
578 export_block_new['fsal'][key] = value
579 else:
580 export_block_new[key] = value
581 try:
582 self.ctx.cluster.run(args=['sudo', 'ceph', 'nfs', 'export', 'update', '-i', '-'],
583 stdin=json.dumps(export_block_new))
584 except CommandFailedError:
585 pass
586
587 update_with_invalid_values('export_id', 9)
588 update_with_invalid_values('cluster_id', 'testing_new')
589 update_with_invalid_values('pseudo', 'test_relpath')
590 update_with_invalid_values('access_type', 'W')
591 update_with_invalid_values('squash', 'no_squash')
592 update_with_invalid_values('security_label', 'invalid')
593 update_with_invalid_values('protocols', [2])
594 update_with_invalid_values('transports', ['UD'])
595 update_with_invalid_values('name', 'RGW', True)
596 update_with_invalid_values('user_id', 'testing_export', True)
597 update_with_invalid_values('fs_name', 'b', True)
598 self._test_delete_cluster()