ceph/src/test/rgw/rgw_multi/multisite.py

   1 from abc import ABCMeta, abstractmethod
   2 from io import StringIO
   3
   4 import json
   5
   6 from .conn import get_gateway_connection, get_gateway_iam_connection, get_gateway_secure_connection
   7
   8 class Cluster:
   9     """ interface to run commands against a distinct ceph cluster """
  10     __metaclass__ = ABCMeta
  11
  12     @abstractmethod
  13     def admin(self, args = None, **kwargs):
  14         """ execute a radosgw-admin command """
  15         pass
  16
  17 class Gateway:
  18     """ interface to control a single radosgw instance """
  19     __metaclass__ = ABCMeta
  20
  21     def __init__(self, host = None, port = None, cluster = None, zone = None, ssl_port = 0):
  22         self.host = host
  23         self.port = port
  24         self.cluster = cluster
  25         self.zone = zone
  26         self.connection = None
  27         self.secure_connection = None
  28         self.ssl_port = ssl_port
  29         self.iam_connection = None
  30
  31     @abstractmethod
  32     def start(self, args = []):
  33         """ start the gateway with the given args """
  34         pass
  35
  36     @abstractmethod
  37     def stop(self):
  38         """ stop the gateway """
  39         pass
  40
  41     def endpoint(self):
  42         return 'http://%s:%d' % (self.host, self.port)
  43
  44 class SystemObject:
  45     """ interface for system objects, represented in json format and
  46     manipulated with radosgw-admin commands """
  47     __metaclass__ = ABCMeta
  48
  49     def __init__(self, data = None, uuid = None):
  50         self.data = data
  51         self.id = uuid
  52         if data:
  53             self.load_from_json(data)
  54
  55     @abstractmethod
  56     def build_command(self, command):
  57         """ return the command line for the given command, including arguments
  58         to specify this object """
  59         pass
  60
  61     @abstractmethod
  62     def load_from_json(self, data):
  63         """ update internal state based on json data """
  64         pass
  65
  66     def command(self, cluster, cmd, args = None, **kwargs):
  67         """ run the given command and return the output and retcode """
  68         args = self.build_command(cmd) + (args or [])
  69         return cluster.admin(args, **kwargs)
  70
  71     def json_command(self, cluster, cmd, args = None, **kwargs):
  72         """ run the given command, parse the output and return the resulting
  73         data and retcode """
  74         s, r = self.command(cluster, cmd, args or [], **kwargs)
  75         if r == 0:
  76             data = json.loads(s)
  77             self.load_from_json(data)
  78             self.data = data
  79         return self.data, r
  80
  81     # mixins for supported commands
  82     class Create(object):
  83         def create(self, cluster, args = None, **kwargs):
  84             """ create the object with the given arguments """
  85             return self.json_command(cluster, 'create', args, **kwargs)
  86
  87     class Delete(object):
  88         def delete(self, cluster, args = None, **kwargs):
  89             """ delete the object """
  90             # not json_command() because delete has no output
  91             _, r = self.command(cluster, 'delete', args, **kwargs)
  92             if r == 0:
  93                 self.data = None
  94             return r
  95
  96     class Get(object):
  97         def get(self, cluster, args = None, **kwargs):
  98             """ read the object from storage """
  99             kwargs['read_only'] = True
 100             return self.json_command(cluster, 'get', args, **kwargs)
 101
 102     class Set(object):
 103         def set(self, cluster, data, args = None, **kwargs):
 104             """ set the object by json """
 105             kwargs['stdin'] = StringIO(json.dumps(data))
 106             return self.json_command(cluster, 'set', args, **kwargs)
 107
 108     class Modify(object):
 109         def modify(self, cluster, args = None, **kwargs):
 110             """ modify the object with the given arguments """
 111             return self.json_command(cluster, 'modify', args, **kwargs)
 112
 113     class CreateDelete(Create, Delete): pass
 114     class GetSet(Get, Set): pass
 115
 116 class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 117     def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
 118         self.name = name
 119         self.zonegroup = zonegroup
 120         self.cluster = cluster
 121         self.gateways = gateways or []
 122         super(Zone, self).__init__(data, zone_id)
 123
 124     def zone_arg(self):
 125         """ command-line argument to specify this zone """
 126         return ['--rgw-zone', self.name]
 127
 128     def zone_args(self):
 129         """ command-line arguments to specify this zone/zonegroup/realm """
 130         args = self.zone_arg()
 131         if self.zonegroup:
 132             args += self.zonegroup.zonegroup_args()
 133         return args
 134
 135     def build_command(self, command):
 136         """ build a command line for the given command and args """
 137         return ['zone', command] + self.zone_args()
 138
 139     def load_from_json(self, data):
 140         """ load the zone from json """
 141         self.id = data['id']
 142         self.name = data['name']
 143
 144     def start(self, args = None):
 145         """ start all gateways """
 146         for g in self.gateways:
 147             g.start(args)
 148
 149     def stop(self):
 150         """ stop all gateways """
 151         for g in self.gateways:
 152             g.stop()
 153
 154     def period(self):
 155         return self.zonegroup.period if self.zonegroup else None
 156
 157     def realm(self):
 158         return self.zonegroup.realm() if self.zonegroup else None
 159
 160     def is_read_only(self):
 161         return False
 162
 163     def tier_type(self):
 164         raise NotImplementedError
 165
 166     def syncs_from(self, zone_name):
 167         return zone_name != self.name
 168
 169     def has_buckets(self):
 170         return True
 171
 172     def has_roles(self):
 173         return True
 174
 175     def get_conn(self, credentials):
 176         return ZoneConn(self, credentials) # not implemented, but can be used
 177
 178 class ZoneConn(object):
 179     def __init__(self, zone, credentials):
 180         self.zone = zone
 181         self.name = zone.name
 182         """ connect to the zone's first gateway """
 183         if isinstance(credentials, list):
 184             self.credentials = credentials[0]
 185         else:
 186             self.credentials = credentials
 187
 188         if self.zone.gateways is not None:
 189             self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
 190             self.secure_conn = get_gateway_secure_connection(self.zone.gateways[0], self.credentials)
 191
 192             self.iam_conn = get_gateway_iam_connection(self.zone.gateways[0], self.credentials)
 193
 194             # create connections for the rest of the gateways (if exist)
 195             for gw in list(self.zone.gateways):
 196                 get_gateway_connection(gw, self.credentials)
 197                 get_gateway_secure_connection(gw, self.credentials)
 198
 199                 get_gateway_iam_connection(gw, self.credentials)
 200
 201
 202     def get_connection(self):
 203         return self.conn
 204
 205     def get_iam_connection(self):
 206         return self.iam_conn
 207
 208     def get_bucket(self, bucket_name, credentials):
 209         raise NotImplementedError
 210
 211     def check_bucket_eq(self, zone, bucket_name):
 212         raise NotImplementedError
 213
 214 class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 215     def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone  = None):
 216         self.name = name
 217         self.period = period
 218         self.zones = zones or []
 219         self.master_zone = master_zone
 220         super(ZoneGroup, self).__init__(data, zonegroup_id)
 221         self.rw_zones = []
 222         self.ro_zones = []
 223         self.zones_by_type = {}
 224         for z in self.zones:
 225             if z.is_read_only():
 226                 self.ro_zones.append(z)
 227             else:
 228                 self.rw_zones.append(z)
 229
 230     def zonegroup_arg(self):
 231         """ command-line argument to specify this zonegroup """
 232         return ['--rgw-zonegroup', self.name]
 233
 234     def zonegroup_args(self):
 235         """ command-line arguments to specify this zonegroup/realm """
 236         args = self.zonegroup_arg()
 237         realm = self.realm()
 238         if realm:
 239             args += realm.realm_arg()
 240         return args
 241
 242     def build_command(self, command):
 243         """ build a command line for the given command and args """
 244         return ['zonegroup', command] + self.zonegroup_args()
 245
 246     def zone_by_id(self, zone_id):
 247         """ return the matching zone by id """
 248         for zone in self.zones:
 249             if zone.id == zone_id:
 250                 return zone
 251         return None
 252
 253     def load_from_json(self, data):
 254         """ load the zonegroup from json """
 255         self.id = data['id']
 256         self.name = data['name']
 257         master_id = data['master_zone']
 258         if not self.master_zone or master_id != self.master_zone.id:
 259             self.master_zone = self.zone_by_id(master_id)
 260
 261     def add(self, cluster, zone, args = None, **kwargs):
 262         """ add an existing zone to the zonegroup """
 263         args = zone.zone_arg() + (args or [])
 264         data, r = self.json_command(cluster, 'add', args, **kwargs)
 265         if r == 0:
 266             zone.zonegroup = self
 267             self.zones.append(zone)
 268         return data, r
 269
 270     def remove(self, cluster, zone, args = None, **kwargs):
 271         """ remove an existing zone from the zonegroup """
 272         args = zone.zone_arg() + (args or [])
 273         data, r = self.json_command(cluster, 'remove', args, **kwargs)
 274         if r == 0:
 275             zone.zonegroup = None
 276             self.zones.remove(zone)
 277         return data, r
 278
 279     def realm(self):
 280         return self.period.realm if self.period else None
 281
 282 class Period(SystemObject, SystemObject.Get):
 283     def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
 284         self.realm = realm
 285         self.zonegroups = zonegroups or []
 286         self.master_zonegroup = master_zonegroup
 287         super(Period, self).__init__(data, period_id)
 288
 289     def zonegroup_by_id(self, zonegroup_id):
 290         """ return the matching zonegroup by id """
 291         for zonegroup in self.zonegroups:
 292             if zonegroup.id == zonegroup_id:
 293                 return zonegroup
 294         return None
 295
 296     def build_command(self, command):
 297         """ build a command line for the given command and args """
 298         return ['period', command]
 299
 300     def load_from_json(self, data):
 301         """ load the period from json """
 302         self.id = data['id']
 303         master_id = data['master_zonegroup']
 304         if not self.master_zonegroup or master_id != self.master_zonegroup.id:
 305             self.master_zonegroup = self.zonegroup_by_id(master_id)
 306
 307     def update(self, zone, args = None, **kwargs):
 308         """ run 'radosgw-admin period update' on the given zone """
 309         assert(zone.cluster)
 310         args = zone.zone_args() + (args or [])
 311         if kwargs.pop('commit', False):
 312             args.append('--commit')
 313         return self.json_command(zone.cluster, 'update', args, **kwargs)
 314
 315     def commit(self, zone, args = None, **kwargs):
 316         """ run 'radosgw-admin period commit' on the given zone """
 317         assert(zone.cluster)
 318         args = zone.zone_args() + (args or [])
 319         return self.json_command(zone.cluster, 'commit', args, **kwargs)
 320
 321 class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
 322     def __init__(self, name, period = None, data = None, realm_id = None):
 323         self.name = name
 324         self.current_period = period
 325         super(Realm, self).__init__(data, realm_id)
 326
 327     def realm_arg(self):
 328         """ return the command-line arguments that specify this realm """
 329         return ['--rgw-realm', self.name]
 330
 331     def build_command(self, command):
 332         """ build a command line for the given command and args """
 333         return ['realm', command] + self.realm_arg()
 334
 335     def load_from_json(self, data):
 336         """ load the realm from json """
 337         self.id = data['id']
 338
 339     def pull(self, cluster, gateway, credentials, args = [], **kwargs):
 340         """ pull an existing realm from the given gateway """
 341         args += ['--url', gateway.endpoint()]
 342         args += credentials.credential_args()
 343         return self.json_command(cluster, 'pull', args, **kwargs)
 344
 345     def master_zonegroup(self):
 346         """ return the current period's master zonegroup """
 347         if self.current_period is None:
 348             return None
 349         return self.current_period.master_zonegroup
 350
 351     def meta_master_zone(self):
 352         """ return the current period's metadata master zone """
 353         zonegroup = self.master_zonegroup()
 354         if zonegroup is None:
 355             return None
 356         return zonegroup.master_zone
 357
 358 class Credentials:
 359     def __init__(self, access_key, secret):
 360         self.access_key = access_key
 361         self.secret = secret
 362
 363     def credential_args(self):
 364         return ['--access-key', self.access_key, '--secret', self.secret]
 365
 366 class User(SystemObject):
 367     def __init__(self, uid, data = None, name = None, credentials = None, tenant = None):
 368         self.name = name
 369         self.credentials = credentials or []
 370         self.tenant = tenant
 371         super(User, self).__init__(data, uid)
 372
 373     def user_arg(self):
 374         """ command-line argument to specify this user """
 375         args = ['--uid', self.id]
 376         if self.tenant:
 377             args += ['--tenant', self.tenant]
 378         return args
 379
 380     def build_command(self, command):
 381         """ build a command line for the given command and args """
 382         return ['user', command] + self.user_arg()
 383
 384     def load_from_json(self, data):
 385         """ load the user from json """
 386         self.id = data['user_id']
 387         self.name = data['display_name']
 388         self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
 389
 390     def create(self, zone, args = None, **kwargs):
 391         """ create the user with the given arguments """
 392         assert(zone.cluster)
 393         args = zone.zone_args() + (args or [])
 394         return self.json_command(zone.cluster, 'create', args, **kwargs)
 395
 396     def info(self, zone, args = None, **kwargs):
 397         """ read the user from storage """
 398         assert(zone.cluster)
 399         args = zone.zone_args() + (args or [])
 400         kwargs['read_only'] = True
 401         return self.json_command(zone.cluster, 'info', args, **kwargs)
 402
 403     def delete(self, zone, args = None, **kwargs):
 404         """ delete the user """
 405         assert(zone.cluster)
 406         args = zone.zone_args() + (args or [])
 407         return self.command(zone.cluster, 'delete', args, **kwargs)