ceph/qa/tasks/mgr/dashboard/test_osd.py

   1 # -*- coding: utf-8 -*-
   2
   3 from __future__ import absolute_import
   4
   5 import json
   6
   7 from .helper import DashboardTestCase, JObj, JAny, JList, JLeaf, JTuple
   8
   9
  10 class OsdTest(DashboardTestCase):
  11
  12     AUTH_ROLES = ['cluster-manager']
  13
  14     @classmethod
  15     def setUpClass(cls):
  16         super(OsdTest, cls).setUpClass()
  17         cls._load_module('test_orchestrator')
  18         cmd = ['orch', 'set', 'backend', 'test_orchestrator']
  19         cls.mgr_cluster.mon_manager.raw_cluster_cmd(*cmd)
  20
  21     def tearDown(self):
  22         self._post('/api/osd/0/mark_in')
  23
  24     @DashboardTestCase.RunAs('test', 'test', ['block-manager'])
  25     def test_access_permissions(self):
  26         self._get('/api/osd')
  27         self.assertStatus(403)
  28         self._get('/api/osd/0')
  29         self.assertStatus(403)
  30
  31     def assert_in_and_not_none(self, data, properties):
  32         self.assertSchema(data, JObj({p: JAny(none=False) for p in properties}, allow_unknown=True))
  33
  34     def test_list(self):
  35         data = self._get('/api/osd')
  36         self.assertStatus(200)
  37
  38         self.assertGreaterEqual(len(data), 1)
  39         data = data[0]
  40         self.assert_in_and_not_none(data, ['host', 'tree', 'state', 'stats', 'stats_history'])
  41         self.assert_in_and_not_none(data['host'], ['name'])
  42         self.assert_in_and_not_none(data['tree'], ['id'])
  43         self.assert_in_and_not_none(data['stats'], ['numpg', 'stat_bytes_used', 'stat_bytes',
  44                                                     'op_r', 'op_w'])
  45         self.assert_in_and_not_none(data['stats_history'], ['op_out_bytes', 'op_in_bytes'])
  46         self.assertSchema(data['stats_history']['op_out_bytes'],
  47                           JList(JTuple([JLeaf(float), JLeaf(float)])))
  48
  49     def test_details(self):
  50         data = self._get('/api/osd/0')
  51         self.assertStatus(200)
  52         self.assert_in_and_not_none(data, ['osd_metadata', 'histogram'])
  53         self.assert_in_and_not_none(data['histogram'], ['osd'])
  54         self.assert_in_and_not_none(data['histogram']['osd'], ['op_w_latency_in_bytes_histogram',
  55                                                                'op_r_latency_out_bytes_histogram'])
  56
  57     def test_histogram(self):
  58         data = self._get('/api/osd/0/histogram')
  59         self.assertStatus(200)
  60         self.assert_in_and_not_none(data['osd'], ['op_w_latency_in_bytes_histogram',
  61                                                                'op_r_latency_out_bytes_histogram'])
  62
  63     def test_scrub(self):
  64         self._post('/api/osd/0/scrub?deep=False')
  65         self.assertStatus(200)
  66
  67         self._post('/api/osd/0/scrub?deep=True')
  68         self.assertStatus(200)
  69
  70     def test_safe_to_delete(self):
  71         data = self._get('/api/osd/safe_to_delete?svc_ids=0')
  72         self.assertStatus(200)
  73         self.assertSchema(data, JObj({
  74              'is_safe_to_delete': JAny(none=True),
  75              'message': str
  76              }))
  77         self.assertTrue(data['is_safe_to_delete'])
  78
  79     def test_osd_smart(self):
  80         self._get('/api/osd/0/smart')
  81         self.assertStatus(200)
  82
  83     def test_mark_out_and_in(self):
  84         self._post('/api/osd/0/mark_out')
  85         self.assertStatus(200)
  86
  87         self._post('/api/osd/0/mark_in')
  88         self.assertStatus(200)
  89
  90     def test_mark_down(self):
  91         self._post('/api/osd/0/mark_down')
  92         self.assertStatus(200)
  93
  94     def test_reweight(self):
  95         self._post('/api/osd/0/reweight', {'weight': 0.4})
  96         self.assertStatus(200)
  97
  98         def get_reweight_value():
  99             self._get('/api/osd/0')
 100             response = self.jsonBody()
 101             if 'osd_map' in response and 'weight' in response['osd_map']:
 102                 return round(response['osd_map']['weight'], 1)
 103         self.wait_until_equal(get_reweight_value, 0.4, 10)
 104         self.assertStatus(200)
 105
 106         # Undo
 107         self._post('/api/osd/0/reweight', {'weight': 1})
 108
 109     def test_create_lost_destroy_remove(self):
 110         # Create
 111         self._task_post('/api/osd', {
 112             'method': 'bare',
 113             'data': {
 114                 'uuid': 'f860ca2e-757d-48ce-b74a-87052cad563f',
 115                 'svc_id': 5
 116             },
 117             'tracking_id': 'bare-5'
 118         })
 119         self.assertStatus(201)
 120
 121         # invalid method
 122         self._task_post('/api/osd', {
 123             'method': 'xyz',
 124             'data': {
 125                 'uuid': 'f860ca2e-757d-48ce-b74a-87052cad563f',
 126                 'svc_id': 5
 127             },
 128             'tracking_id': 'bare-5'
 129         })
 130         self.assertStatus(400)
 131
 132         # Lost
 133         self._post('/api/osd/5/mark_lost')
 134         self.assertStatus(200)
 135         # Destroy
 136         self._post('/api/osd/5/destroy')
 137         self.assertStatus(200)
 138         # Purge
 139         self._post('/api/osd/5/purge')
 140         self.assertStatus(200)
 141
 142     def test_create_with_drive_group(self):
 143         data = {
 144             'method': 'drive_groups',
 145             'data': [
 146                 {
 147                     'service_type': 'osd',
 148                     'service_id': 'test',
 149                     'host_pattern': '*',
 150                     'data_devices': {
 151                         'vendor': 'abc',
 152                         'model': 'cba',
 153                         'rotational': True,
 154                         'size': '4 TB'
 155                     },
 156                     'wal_devices': {
 157                         'vendor': 'def',
 158                         'model': 'fed',
 159                         'rotational': False,
 160                         'size': '1 TB'
 161                     },
 162                     'db_devices': {
 163                         'vendor': 'ghi',
 164                         'model': 'ihg',
 165                         'rotational': False,
 166                         'size': '512 GB'
 167                     },
 168                     'wal_slots': 5,
 169                     'db_slots': 5,
 170                     'encrypted': True
 171                 }
 172             ],
 173             'tracking_id': 'test'
 174         }
 175         self._post('/api/osd', data)
 176         self.assertStatus(201)
 177
 178     def test_safe_to_destroy(self):
 179         osd_dump = json.loads(self._ceph_cmd(['osd', 'dump', '-f', 'json']))
 180         max_id = max(map(lambda e: e['osd'], osd_dump['osds']))
 181
 182         def get_pg_status_equal_unknown(osd_ids):
 183             self._get('/api/osd/safe_to_destroy?ids={}'.format(osd_ids))
 184             if 'message' in self.jsonBody():
 185                 return 'pgs have unknown state' in self.jsonBody()['message']
 186             return False
 187
 188         # 1 OSD safe to destroy
 189         unused_osd_id = max_id + 10
 190         self.wait_until_equal(
 191             lambda: get_pg_status_equal_unknown(unused_osd_id), False, 30)
 192         self.assertStatus(200)
 193         self.assertJsonBody({
 194             'is_safe_to_destroy': True,
 195             'active': [],
 196             'missing_stats': [],
 197             'safe_to_destroy': [unused_osd_id],
 198             'stored_pgs': [],
 199         })
 200
 201         # multiple OSDs safe to destroy
 202         unused_osd_ids = [max_id + 11, max_id + 12]
 203         self.wait_until_equal(
 204             lambda: get_pg_status_equal_unknown(str(unused_osd_ids)), False, 30)
 205         self.assertStatus(200)
 206         self.assertJsonBody({
 207             'is_safe_to_destroy': True,
 208             'active': [],
 209             'missing_stats': [],
 210             'safe_to_destroy': unused_osd_ids,
 211             'stored_pgs': [],
 212         })
 213
 214         # 1 OSD unsafe to destroy
 215         def get_destroy_status():
 216             self._get('/api/osd/safe_to_destroy?ids=0')
 217             if 'is_safe_to_destroy' in self.jsonBody():
 218                 return self.jsonBody()['is_safe_to_destroy']
 219             return None
 220         self.wait_until_equal(get_destroy_status, False, 10)
 221         self.assertStatus(200)
 222
 223     def test_osd_devices(self):
 224         data = self._get('/api/osd/0/devices')
 225         self.assertStatus(200)
 226         self.assertSchema(data, JList(JObj({
 227             'daemons': JList(str),
 228             'devid': str,
 229             'location': JList(JObj({
 230                 'host': str,
 231                 'dev': str,
 232                 'path': str
 233             }))
 234         })))
 235
 236
 237 class OsdFlagsTest(DashboardTestCase):
 238     def __init__(self, *args, **kwargs):
 239         super(OsdFlagsTest, self).__init__(*args, **kwargs)
 240         self._initial_flags = ['sortbitwise', 'recovery_deletes', 'purged_snapdirs',
 241                                'pglog_hardlimit']  # These flags cannot be unset
 242
 243     @classmethod
 244     def _put_flags(cls, flags, ids=None):
 245         url = '/api/osd/flags'
 246         data = {'flags': flags}
 247
 248         if ids:
 249             url = url + '/individual'
 250             data['ids'] = ids
 251
 252         cls._put(url, data=data)
 253         return cls._resp.json()
 254
 255     def test_list_osd_flags(self):
 256         flags = self._get('/api/osd/flags')
 257         self.assertStatus(200)
 258         self.assertEqual(len(flags), 4)
 259         self.assertCountEqual(flags, self._initial_flags)
 260
 261     def test_add_osd_flag(self):
 262         flags = self._put_flags([
 263             'sortbitwise', 'recovery_deletes', 'purged_snapdirs', 'noout',
 264             'pause', 'pglog_hardlimit'
 265         ])
 266         self.assertCountEqual(flags, [
 267             'sortbitwise', 'recovery_deletes', 'purged_snapdirs', 'noout',
 268             'pause', 'pglog_hardlimit'
 269         ])
 270
 271         # Restore flags
 272         self._put_flags(self._initial_flags)
 273
 274     def test_get_indiv_flag(self):
 275         initial = self._get('/api/osd/flags/individual')
 276         self.assertStatus(200)
 277         self.assertSchema(initial, JList(JObj({
 278             'osd': int,
 279             'flags': JList(str)
 280         })))
 281
 282         self._ceph_cmd(['osd', 'set-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
 283         flags_added = self._get('/api/osd/flags/individual')
 284         self.assertStatus(200)
 285         for osd in flags_added:
 286             if osd['osd'] in [0, 1, 2]:
 287                 self.assertIn('noout', osd['flags'])
 288                 self.assertIn('noin', osd['flags'])
 289                 for osd_initial in initial:
 290                     if osd['osd'] == osd_initial['osd']:
 291                         self.assertGreater(len(osd['flags']), len(osd_initial['flags']))
 292
 293         self._ceph_cmd(['osd', 'unset-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
 294         flags_removed = self._get('/api/osd/flags/individual')
 295         self.assertStatus(200)
 296         for osd in flags_removed:
 297             if osd['osd'] in [0, 1, 2]:
 298                 self.assertNotIn('noout', osd['flags'])
 299                 self.assertNotIn('noin', osd['flags'])
 300
 301     def test_add_indiv_flag(self):
 302         flags_update = {'noup': None, 'nodown': None, 'noin': None, 'noout': True}
 303         svc_id = 0
 304
 305         resp = self._put_flags(flags_update, [svc_id])
 306         self._check_indiv_flags_resp(resp, [svc_id], ['noout'], [], ['noup', 'nodown', 'noin'])
 307         self._check_indiv_flags_osd([svc_id], ['noout'], ['noup', 'nodown', 'noin'])
 308
 309         self._ceph_cmd(['osd', 'unset-group', 'noout', 'osd.{}'.format(svc_id)])
 310
 311     def test_add_multiple_indiv_flags(self):
 312         flags_update = {'noup': None, 'nodown': None, 'noin': True, 'noout': True}
 313         svc_id = 0
 314
 315         resp = self._put_flags(flags_update, [svc_id])
 316         self._check_indiv_flags_resp(resp, [svc_id], ['noout', 'noin'], [], ['noup', 'nodown'])
 317         self._check_indiv_flags_osd([svc_id], ['noout', 'noin'], ['noup', 'nodown'])
 318
 319         self._ceph_cmd(['osd', 'unset-group', 'noout,noin', 'osd.{}'.format(svc_id)])
 320
 321     def test_add_multiple_indiv_flags_multiple_osds(self):
 322         flags_update = {'noup': None, 'nodown': None, 'noin': True, 'noout': True}
 323         svc_id = [0, 1, 2]
 324
 325         resp = self._put_flags(flags_update, svc_id)
 326         self._check_indiv_flags_resp(resp, svc_id, ['noout', 'noin'], [], ['noup', 'nodown'])
 327         self._check_indiv_flags_osd([svc_id], ['noout', 'noin'], ['noup', 'nodown'])
 328
 329         self._ceph_cmd(['osd', 'unset-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
 330
 331     def test_remove_indiv_flag(self):
 332         flags_update = {'noup': None, 'nodown': None, 'noin': None, 'noout': False}
 333         svc_id = 0
 334         self._ceph_cmd(['osd', 'set-group', 'noout', 'osd.{}'.format(svc_id)])
 335
 336         resp = self._put_flags(flags_update, [svc_id])
 337         self._check_indiv_flags_resp(resp, [svc_id], [], ['noout'], ['noup', 'nodown', 'noin'])
 338         self._check_indiv_flags_osd([svc_id], [], ['noup', 'nodown', 'noin', 'noout'])
 339
 340     def test_remove_multiple_indiv_flags(self):
 341         flags_update = {'noup': None, 'nodown': None, 'noin': False, 'noout': False}
 342         svc_id = 0
 343         self._ceph_cmd(['osd', 'set-group', 'noout,noin', 'osd.{}'.format(svc_id)])
 344
 345         resp = self._put_flags(flags_update, [svc_id])
 346         self._check_indiv_flags_resp(resp, [svc_id], [], ['noout', 'noin'], ['noup', 'nodown'])
 347         self._check_indiv_flags_osd([svc_id], [], ['noout', 'noin', 'noup', 'nodown'])
 348
 349     def test_remove_multiple_indiv_flags_multiple_osds(self):
 350         flags_update = {'noup': None, 'nodown': None, 'noin': False, 'noout': False}
 351         svc_id = [0, 1, 2]
 352         self._ceph_cmd(['osd', 'unset-group', 'noout,noin', 'osd.0', 'osd.1', 'osd.2'])
 353
 354         resp = self._put_flags(flags_update, svc_id)
 355         self._check_indiv_flags_resp(resp, svc_id, [], ['noout', 'noin'], ['noup', 'nodown'])
 356         self._check_indiv_flags_osd([svc_id], [], ['noout', 'noin', 'noup', 'nodown'])
 357
 358     def _check_indiv_flags_resp(self, resp, ids, added, removed, ignored):
 359         self.assertStatus(200)
 360         self.assertCountEqual(resp['ids'], ids)
 361         self.assertCountEqual(resp['added'], added)
 362         self.assertCountEqual(resp['removed'], removed)
 363
 364         for flag in ignored:
 365             self.assertNotIn(flag, resp['added'])
 366             self.assertNotIn(flag, resp['removed'])
 367
 368     def _check_indiv_flags_osd(self, ids, activated_flags, deactivated_flags):
 369         osds = json.loads(self._ceph_cmd(['osd', 'dump', '--format=json']))['osds']
 370         for osd in osds:
 371             if osd['osd'] in ids:
 372                 for flag in activated_flags:
 373                     self.assertIn(flag, osd['state'])
 374                 for flag in deactivated_flags:
 375                     self.assertNotIn(flag, osd['state'])