ceph/src/test/rgw/rgw_multi/multisite.py

   1 from abc import ABCMeta, abstractmethod
   2 from six import StringIO
   3
   4 import json
   5
   6 from .conn import get_gateway_connection
   7
   8 class Cluster:
   9     """ interface to run commands against a distinct ceph cluster """
  10     __metaclass__ = ABCMeta
  11
  12     @abstractmethod
  13     def admin(self, args = None, **kwargs):
  14         """ execute a radosgw-admin command """
  15         pass
  16
  17 class Gateway:
  18     """ interface to control a single radosgw instance """
  19     __metaclass__ = ABCMeta
  20
  21     def __init__(self, host = None, port = None, cluster = None, zone = None, proto = 'http', connection = None):
  22         self.host = host
  23         self.port = port
  24         self.cluster = cluster
  25         self.zone = zone
  26         self.proto = proto
  27         self.connection = connection
  28
  29     @abstractmethod
  30     def start(self, args = []):
  31         """ start the gateway with the given args """
  32         pass
  33
  34     @abstractmethod
  35     def stop(self):
  36         """ stop the gateway """
  37         pass
  38
  39     def endpoint(self):
  40         return '%s://%s:%d' % (self.proto, self.host, self.port)
  41
  42 class SystemObject:
  43     """ interface for system objects, represented in json format and
  44     manipulated with radosgw-admin commands """
  45     __metaclass__ = ABCMeta
  46
  47     def __init__(self, data = None, uuid = None):
  48         self.data = data
  49         self.id = uuid
  50         if data:
  51             self.load_from_json(data)
  52
  53     @abstractmethod
  54     def build_command(self, command):
  55         """ return the command line for the given command, including arguments
  56         to specify this object """
  57         pass
  58
  59     @abstractmethod
  60     def load_from_json(self, data):
  61         """ update internal state based on json data """
  62         pass
  63
  64     def command(self, cluster, cmd, args = None, **kwargs):
  65         """ run the given command and return the output and retcode """
  66         args = self.build_command(cmd) + (args or [])
  67         return cluster.admin(args, **kwargs)
  68
  69     def json_command(self, cluster, cmd, args = None, **kwargs):
  70         """ run the given command, parse the output and return the resulting
  71         data and retcode """
  72         s, r = self.command(cluster, cmd, args or [], **kwargs)
  73         if r == 0:
  74             output = s.decode('utf-8')
  75             output = output[output.find('{'):] # trim extra output before json
  76             data = json.loads(output)
  77             self.load_from_json(data)
  78             self.data = data
  79         return self.data, r
  80
  81     # mixins for supported commands
  82     class Create(object):
  83         def create(self, cluster, args = None, **kwargs):
  84             """ create the object with the given arguments """
  85             return self.json_command(cluster, 'create', args, **kwargs)
  86
  87     class Delete(object):
  88         def delete(self, cluster, args = None, **kwargs):
  89             """ delete the object """
  90             # not json_command() because delete has no output
  91             _, r = self.command(cluster, 'delete', args, **kwargs)
  92             if r == 0:
  93                 self.data = None
  94             return r
  95
  96     class Get(object):
  97         def get(self, cluster, args = None, **kwargs):
  98             """ read the object from storage """
  99             kwargs['read_only'] = True
 100             return self.json_command(cluster, 'get', args, **kwargs)
 101
 102     class Set(object):
 103         def set(self, cluster, data, args = None, **kwargs):
 104             """ set the object by json """
 105             kwargs['stdin'] = StringIO(json.dumps(data))
 106             return self.json_command(cluster, 'set', args, **kwargs)
 107
 108     class Modify(object):
 109         def modify(self, cluster, args = None, **kwargs):
 110             """ modify the object with the given arguments """
 111             return self.json_command(cluster, 'modify', args, **kwargs)
 112
 113     class CreateDelete(Create, Delete): pass
 114     class GetSet(Get, Set): pass
 115
 116 class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 117     def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
 118         self.name = name
 119         self.zonegroup = zonegroup
 120         self.cluster = cluster
 121         self.gateways = gateways or []
 122         super(Zone, self).__init__(data, zone_id)
 123
 124     def zone_arg(self):
 125         """ command-line argument to specify this zone """
 126         return ['--rgw-zone', self.name]
 127
 128     def zone_args(self):
 129         """ command-line arguments to specify this zone/zonegroup/realm """
 130         args = self.zone_arg()
 131         if self.zonegroup:
 132             args += self.zonegroup.zonegroup_args()
 133         return args
 134
 135     def build_command(self, command):
 136         """ build a command line for the given command and args """
 137         return ['zone', command] + self.zone_args()
 138
 139     def load_from_json(self, data):
 140         """ load the zone from json """
 141         self.id = data['id']
 142         self.name = data['name']
 143
 144     def start(self, args = None):
 145         """ start all gateways """
 146         for g in self.gateways:
 147             g.start(args)
 148
 149     def stop(self):
 150         """ stop all gateways """
 151         for g in self.gateways:
 152             g.stop()
 153
 154     def period(self):
 155         return self.zonegroup.period if self.zonegroup else None
 156
 157     def realm(self):
 158         return self.zonegroup.realm() if self.zonegroup else None
 159
 160     def is_read_only(self):
 161         return False
 162
 163     def tier_type(self):
 164         raise NotImplementedError
 165
 166     def has_buckets(self):
 167         return True
 168
 169     def get_conn(self, credentials):
 170         return ZoneConn(self, credentials) # not implemented, but can be used
 171
 172 class ZoneConn(object):
 173     def __init__(self, zone, credentials):
 174         self.zone = zone
 175         self.name = zone.name
 176         """ connect to the zone's first gateway """
 177         if isinstance(credentials, list):
 178             self.credentials = credentials[0]
 179         else:
 180             self.credentials = credentials
 181
 182         if self.zone.gateways is not None:
 183             self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
 184
 185     def get_connection(self):
 186         return self.conn
 187
 188     def get_bucket(self, bucket_name, credentials):
 189         raise NotImplementedError
 190
 191     def check_bucket_eq(self, zone, bucket_name):
 192         raise NotImplementedError
 193
 194 class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 195     def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone  = None):
 196         self.name = name
 197         self.period = period
 198         self.zones = zones or []
 199         self.master_zone = master_zone
 200         super(ZoneGroup, self).__init__(data, zonegroup_id)
 201         self.rw_zones = []
 202         self.ro_zones = []
 203         self.zones_by_type = {}
 204         for z in self.zones:
 205             if z.is_read_only():
 206                 self.ro_zones.append(z)
 207             else:
 208                 self.rw_zones.append(z)
 209
 210     def zonegroup_arg(self):
 211         """ command-line argument to specify this zonegroup """
 212         return ['--rgw-zonegroup', self.name]
 213
 214     def zonegroup_args(self):
 215         """ command-line arguments to specify this zonegroup/realm """
 216         args = self.zonegroup_arg()
 217         realm = self.realm()
 218         if realm:
 219             args += realm.realm_arg()
 220         return args
 221
 222     def build_command(self, command):
 223         """ build a command line for the given command and args """
 224         return ['zonegroup', command] + self.zonegroup_args()
 225
 226     def zone_by_id(self, zone_id):
 227         """ return the matching zone by id """
 228         for zone in self.zones:
 229             if zone.id == zone_id:
 230                 return zone
 231         return None
 232
 233     def load_from_json(self, data):
 234         """ load the zonegroup from json """
 235         self.id = data['id']
 236         self.name = data['name']
 237         master_id = data['master_zone']
 238         if not self.master_zone or master_id != self.master_zone.id:
 239             self.master_zone = self.zone_by_id(master_id)
 240
 241     def add(self, cluster, zone, args = None, **kwargs):
 242         """ add an existing zone to the zonegroup """
 243         args = zone.zone_arg() + (args or [])
 244         data, r = self.json_command(cluster, 'add', args, **kwargs)
 245         if r == 0:
 246             zone.zonegroup = self
 247             self.zones.append(zone)
 248         return data, r
 249
 250     def remove(self, cluster, zone, args = None, **kwargs):
 251         """ remove an existing zone from the zonegroup """
 252         args = zone.zone_arg() + (args or [])
 253         data, r = self.json_command(cluster, 'remove', args, **kwargs)
 254         if r == 0:
 255             zone.zonegroup = None
 256             self.zones.remove(zone)
 257         return data, r
 258
 259     def realm(self):
 260         return self.period.realm if self.period else None
 261
 262 class Period(SystemObject, SystemObject.Get):
 263     def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
 264         self.realm = realm
 265         self.zonegroups = zonegroups or []
 266         self.master_zonegroup = master_zonegroup
 267         super(Period, self).__init__(data, period_id)
 268
 269     def zonegroup_by_id(self, zonegroup_id):
 270         """ return the matching zonegroup by id """
 271         for zonegroup in self.zonegroups:
 272             if zonegroup.id == zonegroup_id:
 273                 return zonegroup
 274         return None
 275
 276     def build_command(self, command):
 277         """ build a command line for the given command and args """
 278         return ['period', command]
 279
 280     def load_from_json(self, data):
 281         """ load the period from json """
 282         self.id = data['id']
 283         master_id = data['master_zonegroup']
 284         if not self.master_zonegroup or master_id != self.master_zonegroup.id:
 285             self.master_zonegroup = self.zonegroup_by_id(master_id)
 286
 287     def update(self, zone, args = None, **kwargs):
 288         """ run 'radosgw-admin period update' on the given zone """
 289         assert(zone.cluster)
 290         args = zone.zone_args() + (args or [])
 291         if kwargs.pop('commit', False):
 292             args.append('--commit')
 293         return self.json_command(zone.cluster, 'update', args, **kwargs)
 294
 295     def commit(self, zone, args = None, **kwargs):
 296         """ run 'radosgw-admin period commit' on the given zone """
 297         assert(zone.cluster)
 298         args = zone.zone_args() + (args or [])
 299         return self.json_command(zone.cluster, 'commit', args, **kwargs)
 300
 301 class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
 302     def __init__(self, name, period = None, data = None, realm_id = None):
 303         self.name = name
 304         self.current_period = period
 305         super(Realm, self).__init__(data, realm_id)
 306
 307     def realm_arg(self):
 308         """ return the command-line arguments that specify this realm """
 309         return ['--rgw-realm', self.name]
 310
 311     def build_command(self, command):
 312         """ build a command line for the given command and args """
 313         return ['realm', command] + self.realm_arg()
 314
 315     def load_from_json(self, data):
 316         """ load the realm from json """
 317         self.id = data['id']
 318
 319     def pull(self, cluster, gateway, credentials, args = [], **kwargs):
 320         """ pull an existing realm from the given gateway """
 321         args += ['--url', gateway.endpoint()]
 322         args += credentials.credential_args()
 323         return self.json_command(cluster, 'pull', args, **kwargs)
 324
 325     def master_zonegroup(self):
 326         """ return the current period's master zonegroup """
 327         if self.current_period is None:
 328             return None
 329         return self.current_period.master_zonegroup
 330
 331     def meta_master_zone(self):
 332         """ return the current period's metadata master zone """
 333         zonegroup = self.master_zonegroup()
 334         if zonegroup is None:
 335             return None
 336         return zonegroup.master_zone
 337
 338 class Credentials:
 339     def __init__(self, access_key, secret):
 340         self.access_key = access_key
 341         self.secret = secret
 342
 343     def credential_args(self):
 344         return ['--access-key', self.access_key, '--secret', self.secret]
 345
 346 class User(SystemObject):
 347     def __init__(self, uid, data = None, name = None, credentials = None, tenant = None):
 348         self.name = name
 349         self.credentials = credentials or []
 350         self.tenant = tenant
 351         super(User, self).__init__(data, uid)
 352
 353     def user_arg(self):
 354         """ command-line argument to specify this user """
 355         args = ['--uid', self.id]
 356         if self.tenant:
 357             args += ['--tenant', self.tenant]
 358         return args
 359
 360     def build_command(self, command):
 361         """ build a command line for the given command and args """
 362         return ['user', command] + self.user_arg()
 363
 364     def load_from_json(self, data):
 365         """ load the user from json """
 366         self.id = data['user_id']
 367         self.name = data['display_name']
 368         self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
 369
 370     def create(self, zone, args = None, **kwargs):
 371         """ create the user with the given arguments """
 372         assert(zone.cluster)
 373         args = zone.zone_args() + (args or [])
 374         return self.json_command(zone.cluster, 'create', args, **kwargs)
 375
 376     def info(self, zone, args = None, **kwargs):
 377         """ read the user from storage """
 378         assert(zone.cluster)
 379         args = zone.zone_args() + (args or [])
 380         kwargs['read_only'] = True
 381         return self.json_command(zone.cluster, 'info', args, **kwargs)
 382
 383     def delete(self, zone, args = None, **kwargs):
 384         """ delete the user """
 385         assert(zone.cluster)
 386         args = zone.zone_args() + (args or [])
 387         return self.command(zone.cluster, 'delete', args, **kwargs)