ceph/src/test/rgw/rgw_multi/multisite.py

   1 from abc import ABCMeta, abstractmethod
   2 from io import StringIO
   3
   4 import json
   5
   6 from .conn import get_gateway_connection, get_gateway_secure_connection
   7
   8 class Cluster:
   9     """ interface to run commands against a distinct ceph cluster """
  10     __metaclass__ = ABCMeta
  11
  12     @abstractmethod
  13     def admin(self, args = None, **kwargs):
  14         """ execute a radosgw-admin command """
  15         pass
  16
  17 class Gateway:
  18     """ interface to control a single radosgw instance """
  19     __metaclass__ = ABCMeta
  20
  21     def __init__(self, host = None, port = None, cluster = None, zone = None, ssl_port = 0):
  22         self.host = host
  23         self.port = port
  24         self.cluster = cluster
  25         self.zone = zone
  26         self.connection = None
  27         self.secure_connection = None
  28         self.ssl_port = ssl_port
  29
  30     @abstractmethod
  31     def start(self, args = []):
  32         """ start the gateway with the given args """
  33         pass
  34
  35     @abstractmethod
  36     def stop(self):
  37         """ stop the gateway """
  38         pass
  39
  40     def endpoint(self):
  41         return 'http://%s:%d' % (self.host, self.port)
  42
  43 class SystemObject:
  44     """ interface for system objects, represented in json format and
  45     manipulated with radosgw-admin commands """
  46     __metaclass__ = ABCMeta
  47
  48     def __init__(self, data = None, uuid = None):
  49         self.data = data
  50         self.id = uuid
  51         if data:
  52             self.load_from_json(data)
  53
  54     @abstractmethod
  55     def build_command(self, command):
  56         """ return the command line for the given command, including arguments
  57         to specify this object """
  58         pass
  59
  60     @abstractmethod
  61     def load_from_json(self, data):
  62         """ update internal state based on json data """
  63         pass
  64
  65     def command(self, cluster, cmd, args = None, **kwargs):
  66         """ run the given command and return the output and retcode """
  67         args = self.build_command(cmd) + (args or [])
  68         return cluster.admin(args, **kwargs)
  69
  70     def json_command(self, cluster, cmd, args = None, **kwargs):
  71         """ run the given command, parse the output and return the resulting
  72         data and retcode """
  73         s, r = self.command(cluster, cmd, args or [], **kwargs)
  74         if r == 0:
  75             data = json.loads(s)
  76             self.load_from_json(data)
  77             self.data = data
  78         return self.data, r
  79
  80     # mixins for supported commands
  81     class Create(object):
  82         def create(self, cluster, args = None, **kwargs):
  83             """ create the object with the given arguments """
  84             return self.json_command(cluster, 'create', args, **kwargs)
  85
  86     class Delete(object):
  87         def delete(self, cluster, args = None, **kwargs):
  88             """ delete the object """
  89             # not json_command() because delete has no output
  90             _, r = self.command(cluster, 'delete', args, **kwargs)
  91             if r == 0:
  92                 self.data = None
  93             return r
  94
  95     class Get(object):
  96         def get(self, cluster, args = None, **kwargs):
  97             """ read the object from storage """
  98             kwargs['read_only'] = True
  99             return self.json_command(cluster, 'get', args, **kwargs)
 100
 101     class Set(object):
 102         def set(self, cluster, data, args = None, **kwargs):
 103             """ set the object by json """
 104             kwargs['stdin'] = StringIO(json.dumps(data))
 105             return self.json_command(cluster, 'set', args, **kwargs)
 106
 107     class Modify(object):
 108         def modify(self, cluster, args = None, **kwargs):
 109             """ modify the object with the given arguments """
 110             return self.json_command(cluster, 'modify', args, **kwargs)
 111
 112     class CreateDelete(Create, Delete): pass
 113     class GetSet(Get, Set): pass
 114
 115 class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 116     def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
 117         self.name = name
 118         self.zonegroup = zonegroup
 119         self.cluster = cluster
 120         self.gateways = gateways or []
 121         super(Zone, self).__init__(data, zone_id)
 122
 123     def zone_arg(self):
 124         """ command-line argument to specify this zone """
 125         return ['--rgw-zone', self.name]
 126
 127     def zone_args(self):
 128         """ command-line arguments to specify this zone/zonegroup/realm """
 129         args = self.zone_arg()
 130         if self.zonegroup:
 131             args += self.zonegroup.zonegroup_args()
 132         return args
 133
 134     def build_command(self, command):
 135         """ build a command line for the given command and args """
 136         return ['zone', command] + self.zone_args()
 137
 138     def load_from_json(self, data):
 139         """ load the zone from json """
 140         self.id = data['id']
 141         self.name = data['name']
 142
 143     def start(self, args = None):
 144         """ start all gateways """
 145         for g in self.gateways:
 146             g.start(args)
 147
 148     def stop(self):
 149         """ stop all gateways """
 150         for g in self.gateways:
 151             g.stop()
 152
 153     def period(self):
 154         return self.zonegroup.period if self.zonegroup else None
 155
 156     def realm(self):
 157         return self.zonegroup.realm() if self.zonegroup else None
 158
 159     def is_read_only(self):
 160         return False
 161
 162     def tier_type(self):
 163         raise NotImplementedError
 164
 165     def syncs_from(self, zone_name):
 166         return zone_name != self.name
 167
 168     def has_buckets(self):
 169         return True
 170
 171     def get_conn(self, credentials):
 172         return ZoneConn(self, credentials) # not implemented, but can be used
 173
 174 class ZoneConn(object):
 175     def __init__(self, zone, credentials):
 176         self.zone = zone
 177         self.name = zone.name
 178         """ connect to the zone's first gateway """
 179         if isinstance(credentials, list):
 180             self.credentials = credentials[0]
 181         else:
 182             self.credentials = credentials
 183
 184         if self.zone.gateways is not None:
 185             self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
 186             self.secure_conn = get_gateway_secure_connection(self.zone.gateways[0], self.credentials)
 187             # create connections for the rest of the gateways (if exist)
 188             for gw in list(self.zone.gateways):
 189                 get_gateway_connection(gw, self.credentials)
 190                 get_gateway_secure_connection(gw, self.credentials)
 191
 192
 193     def get_connection(self):
 194         return self.conn
 195
 196     def get_bucket(self, bucket_name, credentials):
 197         raise NotImplementedError
 198
 199     def check_bucket_eq(self, zone, bucket_name):
 200         raise NotImplementedError
 201
 202 class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 203     def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone  = None):
 204         self.name = name
 205         self.period = period
 206         self.zones = zones or []
 207         self.master_zone = master_zone
 208         super(ZoneGroup, self).__init__(data, zonegroup_id)
 209         self.rw_zones = []
 210         self.ro_zones = []
 211         self.zones_by_type = {}
 212         for z in self.zones:
 213             if z.is_read_only():
 214                 self.ro_zones.append(z)
 215             else:
 216                 self.rw_zones.append(z)
 217
 218     def zonegroup_arg(self):
 219         """ command-line argument to specify this zonegroup """
 220         return ['--rgw-zonegroup', self.name]
 221
 222     def zonegroup_args(self):
 223         """ command-line arguments to specify this zonegroup/realm """
 224         args = self.zonegroup_arg()
 225         realm = self.realm()
 226         if realm:
 227             args += realm.realm_arg()
 228         return args
 229
 230     def build_command(self, command):
 231         """ build a command line for the given command and args """
 232         return ['zonegroup', command] + self.zonegroup_args()
 233
 234     def zone_by_id(self, zone_id):
 235         """ return the matching zone by id """
 236         for zone in self.zones:
 237             if zone.id == zone_id:
 238                 return zone
 239         return None
 240
 241     def load_from_json(self, data):
 242         """ load the zonegroup from json """
 243         self.id = data['id']
 244         self.name = data['name']
 245         master_id = data['master_zone']
 246         if not self.master_zone or master_id != self.master_zone.id:
 247             self.master_zone = self.zone_by_id(master_id)
 248
 249     def add(self, cluster, zone, args = None, **kwargs):
 250         """ add an existing zone to the zonegroup """
 251         args = zone.zone_arg() + (args or [])
 252         data, r = self.json_command(cluster, 'add', args, **kwargs)
 253         if r == 0:
 254             zone.zonegroup = self
 255             self.zones.append(zone)
 256         return data, r
 257
 258     def remove(self, cluster, zone, args = None, **kwargs):
 259         """ remove an existing zone from the zonegroup """
 260         args = zone.zone_arg() + (args or [])
 261         data, r = self.json_command(cluster, 'remove', args, **kwargs)
 262         if r == 0:
 263             zone.zonegroup = None
 264             self.zones.remove(zone)
 265         return data, r
 266
 267     def realm(self):
 268         return self.period.realm if self.period else None
 269
 270 class Period(SystemObject, SystemObject.Get):
 271     def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
 272         self.realm = realm
 273         self.zonegroups = zonegroups or []
 274         self.master_zonegroup = master_zonegroup
 275         super(Period, self).__init__(data, period_id)
 276
 277     def zonegroup_by_id(self, zonegroup_id):
 278         """ return the matching zonegroup by id """
 279         for zonegroup in self.zonegroups:
 280             if zonegroup.id == zonegroup_id:
 281                 return zonegroup
 282         return None
 283
 284     def build_command(self, command):
 285         """ build a command line for the given command and args """
 286         return ['period', command]
 287
 288     def load_from_json(self, data):
 289         """ load the period from json """
 290         self.id = data['id']
 291         master_id = data['master_zonegroup']
 292         if not self.master_zonegroup or master_id != self.master_zonegroup.id:
 293             self.master_zonegroup = self.zonegroup_by_id(master_id)
 294
 295     def update(self, zone, args = None, **kwargs):
 296         """ run 'radosgw-admin period update' on the given zone """
 297         assert(zone.cluster)
 298         args = zone.zone_args() + (args or [])
 299         if kwargs.pop('commit', False):
 300             args.append('--commit')
 301         return self.json_command(zone.cluster, 'update', args, **kwargs)
 302
 303     def commit(self, zone, args = None, **kwargs):
 304         """ run 'radosgw-admin period commit' on the given zone """
 305         assert(zone.cluster)
 306         args = zone.zone_args() + (args or [])
 307         return self.json_command(zone.cluster, 'commit', args, **kwargs)
 308
 309 class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
 310     def __init__(self, name, period = None, data = None, realm_id = None):
 311         self.name = name
 312         self.current_period = period
 313         super(Realm, self).__init__(data, realm_id)
 314
 315     def realm_arg(self):
 316         """ return the command-line arguments that specify this realm """
 317         return ['--rgw-realm', self.name]
 318
 319     def build_command(self, command):
 320         """ build a command line for the given command and args """
 321         return ['realm', command] + self.realm_arg()
 322
 323     def load_from_json(self, data):
 324         """ load the realm from json """
 325         self.id = data['id']
 326
 327     def pull(self, cluster, gateway, credentials, args = [], **kwargs):
 328         """ pull an existing realm from the given gateway """
 329         args += ['--url', gateway.endpoint()]
 330         args += credentials.credential_args()
 331         return self.json_command(cluster, 'pull', args, **kwargs)
 332
 333     def master_zonegroup(self):
 334         """ return the current period's master zonegroup """
 335         if self.current_period is None:
 336             return None
 337         return self.current_period.master_zonegroup
 338
 339     def meta_master_zone(self):
 340         """ return the current period's metadata master zone """
 341         zonegroup = self.master_zonegroup()
 342         if zonegroup is None:
 343             return None
 344         return zonegroup.master_zone
 345
 346 class Credentials:
 347     def __init__(self, access_key, secret):
 348         self.access_key = access_key
 349         self.secret = secret
 350
 351     def credential_args(self):
 352         return ['--access-key', self.access_key, '--secret', self.secret]
 353
 354 class User(SystemObject):
 355     def __init__(self, uid, data = None, name = None, credentials = None, tenant = None):
 356         self.name = name
 357         self.credentials = credentials or []
 358         self.tenant = tenant
 359         super(User, self).__init__(data, uid)
 360
 361     def user_arg(self):
 362         """ command-line argument to specify this user """
 363         args = ['--uid', self.id]
 364         if self.tenant:
 365             args += ['--tenant', self.tenant]
 366         return args
 367
 368     def build_command(self, command):
 369         """ build a command line for the given command and args """
 370         return ['user', command] + self.user_arg()
 371
 372     def load_from_json(self, data):
 373         """ load the user from json """
 374         self.id = data['user_id']
 375         self.name = data['display_name']
 376         self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
 377
 378     def create(self, zone, args = None, **kwargs):
 379         """ create the user with the given arguments """
 380         assert(zone.cluster)
 381         args = zone.zone_args() + (args or [])
 382         return self.json_command(zone.cluster, 'create', args, **kwargs)
 383
 384     def info(self, zone, args = None, **kwargs):
 385         """ read the user from storage """
 386         assert(zone.cluster)
 387         args = zone.zone_args() + (args or [])
 388         kwargs['read_only'] = True
 389         return self.json_command(zone.cluster, 'info', args, **kwargs)
 390
 391     def delete(self, zone, args = None, **kwargs):
 392         """ delete the user """
 393         assert(zone.cluster)
 394         args = zone.zone_args() + (args or [])
 395         return self.command(zone.cluster, 'delete', args, **kwargs)