ceph/src/test/rgw/rgw_multi/multisite.py

   1 from abc import ABCMeta, abstractmethod
   2 from cStringIO import StringIO
   3 import json
   4
   5 from conn import get_gateway_connection
   6
   7 class Cluster:
   8     """ interface to run commands against a distinct ceph cluster """
   9     __metaclass__ = ABCMeta
  10
  11     @abstractmethod
  12     def admin(self, args = None, **kwargs):
  13         """ execute a radosgw-admin command """
  14         pass
  15
  16 class Gateway:
  17     """ interface to control a single radosgw instance """
  18     __metaclass__ = ABCMeta
  19
  20     def __init__(self, host = None, port = None, cluster = None, zone = None, proto = 'http', connection = None):
  21         self.host = host
  22         self.port = port
  23         self.cluster = cluster
  24         self.zone = zone
  25         self.proto = proto
  26         self.connection = connection
  27
  28     @abstractmethod
  29     def start(self, args = []):
  30         """ start the gateway with the given args """
  31         pass
  32
  33     @abstractmethod
  34     def stop(self):
  35         """ stop the gateway """
  36         pass
  37
  38     def endpoint(self):
  39         return '%s://%s:%d' % (self.proto, self.host, self.port)
  40
  41 class SystemObject:
  42     """ interface for system objects, represented in json format and
  43     manipulated with radosgw-admin commands """
  44     __metaclass__ = ABCMeta
  45
  46     def __init__(self, data = None, uuid = None):
  47         self.data = data
  48         self.id = uuid
  49         if data:
  50             self.load_from_json(data)
  51
  52     @abstractmethod
  53     def build_command(self, command):
  54         """ return the command line for the given command, including arguments
  55         to specify this object """
  56         pass
  57
  58     @abstractmethod
  59     def load_from_json(self, data):
  60         """ update internal state based on json data """
  61         pass
  62
  63     def command(self, cluster, cmd, args = None, **kwargs):
  64         """ run the given command and return the output and retcode """
  65         args = self.build_command(cmd) + (args or [])
  66         return cluster.admin(args, **kwargs)
  67
  68     def json_command(self, cluster, cmd, args = None, **kwargs):
  69         """ run the given command, parse the output and return the resulting
  70         data and retcode """
  71         s, r = self.command(cluster, cmd, args or [], **kwargs)
  72         if r == 0:
  73             output = s.decode('utf-8')
  74             output = output[output.find('{'):] # trim extra output before json
  75             data = json.loads(output)
  76             self.load_from_json(data)
  77             self.data = data
  78         return self.data, r
  79
  80     # mixins for supported commands
  81     class Create(object):
  82         def create(self, cluster, args = None, **kwargs):
  83             """ create the object with the given arguments """
  84             return self.json_command(cluster, 'create', args, **kwargs)
  85
  86     class Delete(object):
  87         def delete(self, cluster, args = None, **kwargs):
  88             """ delete the object """
  89             # not json_command() because delete has no output
  90             _, r = self.command(cluster, 'delete', args, **kwargs)
  91             if r == 0:
  92                 self.data = None
  93             return r
  94
  95     class Get(object):
  96         def get(self, cluster, args = None, **kwargs):
  97             """ read the object from storage """
  98             kwargs['read_only'] = True
  99             return self.json_command(cluster, 'get', args, **kwargs)
 100
 101     class Set(object):
 102         def set(self, cluster, data, args = None, **kwargs):
 103             """ set the object by json """
 104             kwargs['stdin'] = StringIO(json.dumps(data))
 105             return self.json_command(cluster, 'set', args, **kwargs)
 106
 107     class Modify(object):
 108         def modify(self, cluster, args = None, **kwargs):
 109             """ modify the object with the given arguments """
 110             return self.json_command(cluster, 'modify', args, **kwargs)
 111
 112     class CreateDelete(Create, Delete): pass
 113     class GetSet(Get, Set): pass
 114
 115 class Zone(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 116     def __init__(self, name, zonegroup = None, cluster = None, data = None, zone_id = None, gateways = None):
 117         self.name = name
 118         self.zonegroup = zonegroup
 119         self.cluster = cluster
 120         self.gateways = gateways or []
 121         super(Zone, self).__init__(data, zone_id)
 122
 123     def zone_arg(self):
 124         """ command-line argument to specify this zone """
 125         return ['--rgw-zone', self.name]
 126
 127     def zone_args(self):
 128         """ command-line arguments to specify this zone/zonegroup/realm """
 129         args = self.zone_arg()
 130         if self.zonegroup:
 131             args += self.zonegroup.zonegroup_args()
 132         return args
 133
 134     def build_command(self, command):
 135         """ build a command line for the given command and args """
 136         return ['zone', command] + self.zone_args()
 137
 138     def load_from_json(self, data):
 139         """ load the zone from json """
 140         self.id = data['id']
 141         self.name = data['name']
 142
 143     def start(self, args = None):
 144         """ start all gateways """
 145         for g in self.gateways:
 146             g.start(args)
 147
 148     def stop(self):
 149         """ stop all gateways """
 150         for g in self.gateways:
 151             g.stop()
 152
 153     def period(self):
 154         return self.zonegroup.period if self.zonegroup else None
 155
 156     def realm(self):
 157         return self.zonegroup.realm() if self.zonegroup else None
 158
 159     def is_read_only(self):
 160         return False
 161
 162     def tier_type(self):
 163         raise NotImplementedError
 164
 165     def has_buckets(self):
 166         return True
 167
 168     def get_conn(self, credentials):
 169         return ZoneConn(self, credentials) # not implemented, but can be used
 170
 171 class ZoneConn(object):
 172     def __init__(self, zone, credentials):
 173         self.zone = zone
 174         self.name = zone.name
 175         """ connect to the zone's first gateway """
 176         if isinstance(credentials, list):
 177             self.credentials = credentials[0]
 178         else:
 179             self.credentials = credentials
 180
 181         if self.zone.gateways is not None:
 182             self.conn = get_gateway_connection(self.zone.gateways[0], self.credentials)
 183
 184     def get_connection(self):
 185         return self.conn
 186
 187     def get_bucket(self, bucket_name, credentials):
 188         raise NotImplementedError
 189
 190     def check_bucket_eq(self, zone, bucket_name):
 191         raise NotImplementedError
 192
 193 class ZoneGroup(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet, SystemObject.Modify):
 194     def __init__(self, name, period = None, data = None, zonegroup_id = None, zones = None, master_zone  = None):
 195         self.name = name
 196         self.period = period
 197         self.zones = zones or []
 198         self.master_zone = master_zone
 199         super(ZoneGroup, self).__init__(data, zonegroup_id)
 200         self.rw_zones = []
 201         self.ro_zones = []
 202         self.zones_by_type = {}
 203         for z in self.zones:
 204             if z.is_read_only():
 205                 self.ro_zones.append(z)
 206             else:
 207                 self.rw_zones.append(z)
 208
 209     def zonegroup_arg(self):
 210         """ command-line argument to specify this zonegroup """
 211         return ['--rgw-zonegroup', self.name]
 212
 213     def zonegroup_args(self):
 214         """ command-line arguments to specify this zonegroup/realm """
 215         args = self.zonegroup_arg()
 216         realm = self.realm()
 217         if realm:
 218             args += realm.realm_arg()
 219         return args
 220
 221     def build_command(self, command):
 222         """ build a command line for the given command and args """
 223         return ['zonegroup', command] + self.zonegroup_args()
 224
 225     def zone_by_id(self, zone_id):
 226         """ return the matching zone by id """
 227         for zone in self.zones:
 228             if zone.id == zone_id:
 229                 return zone
 230         return None
 231
 232     def load_from_json(self, data):
 233         """ load the zonegroup from json """
 234         self.id = data['id']
 235         self.name = data['name']
 236         master_id = data['master_zone']
 237         if not self.master_zone or master_id != self.master_zone.id:
 238             self.master_zone = self.zone_by_id(master_id)
 239
 240     def add(self, cluster, zone, args = None, **kwargs):
 241         """ add an existing zone to the zonegroup """
 242         args = zone.zone_arg() + (args or [])
 243         data, r = self.json_command(cluster, 'add', args, **kwargs)
 244         if r == 0:
 245             zone.zonegroup = self
 246             self.zones.append(zone)
 247         return data, r
 248
 249     def remove(self, cluster, zone, args = None, **kwargs):
 250         """ remove an existing zone from the zonegroup """
 251         args = zone.zone_arg() + (args or [])
 252         data, r = self.json_command(cluster, 'remove', args, **kwargs)
 253         if r == 0:
 254             zone.zonegroup = None
 255             self.zones.remove(zone)
 256         return data, r
 257
 258     def realm(self):
 259         return self.period.realm if self.period else None
 260
 261 class Period(SystemObject, SystemObject.Get):
 262     def __init__(self, realm = None, data = None, period_id = None, zonegroups = None, master_zonegroup = None):
 263         self.realm = realm
 264         self.zonegroups = zonegroups or []
 265         self.master_zonegroup = master_zonegroup
 266         super(Period, self).__init__(data, period_id)
 267
 268     def zonegroup_by_id(self, zonegroup_id):
 269         """ return the matching zonegroup by id """
 270         for zonegroup in self.zonegroups:
 271             if zonegroup.id == zonegroup_id:
 272                 return zonegroup
 273         return None
 274
 275     def build_command(self, command):
 276         """ build a command line for the given command and args """
 277         return ['period', command]
 278
 279     def load_from_json(self, data):
 280         """ load the period from json """
 281         self.id = data['id']
 282         master_id = data['master_zonegroup']
 283         if not self.master_zonegroup or master_id != self.master_zonegroup.id:
 284             self.master_zonegroup = self.zonegroup_by_id(master_id)
 285
 286     def update(self, zone, args = None, **kwargs):
 287         """ run 'radosgw-admin period update' on the given zone """
 288         assert(zone.cluster)
 289         args = zone.zone_args() + (args or [])
 290         if kwargs.pop('commit', False):
 291             args.append('--commit')
 292         return self.json_command(zone.cluster, 'update', args, **kwargs)
 293
 294     def commit(self, zone, args = None, **kwargs):
 295         """ run 'radosgw-admin period commit' on the given zone """
 296         assert(zone.cluster)
 297         args = zone.zone_args() + (args or [])
 298         return self.json_command(zone.cluster, 'commit', args, **kwargs)
 299
 300 class Realm(SystemObject, SystemObject.CreateDelete, SystemObject.GetSet):
 301     def __init__(self, name, period = None, data = None, realm_id = None):
 302         self.name = name
 303         self.current_period = period
 304         super(Realm, self).__init__(data, realm_id)
 305
 306     def realm_arg(self):
 307         """ return the command-line arguments that specify this realm """
 308         return ['--rgw-realm', self.name]
 309
 310     def build_command(self, command):
 311         """ build a command line for the given command and args """
 312         return ['realm', command] + self.realm_arg()
 313
 314     def load_from_json(self, data):
 315         """ load the realm from json """
 316         self.id = data['id']
 317
 318     def pull(self, cluster, gateway, credentials, args = [], **kwargs):
 319         """ pull an existing realm from the given gateway """
 320         args += ['--url', gateway.endpoint()]
 321         args += credentials.credential_args()
 322         return self.json_command(cluster, 'pull', args, **kwargs)
 323
 324     def master_zonegroup(self):
 325         """ return the current period's master zonegroup """
 326         if self.current_period is None:
 327             return None
 328         return self.current_period.master_zonegroup
 329
 330     def meta_master_zone(self):
 331         """ return the current period's metadata master zone """
 332         zonegroup = self.master_zonegroup()
 333         if zonegroup is None:
 334             return None
 335         return zonegroup.master_zone
 336
 337 class Credentials:
 338     def __init__(self, access_key, secret):
 339         self.access_key = access_key
 340         self.secret = secret
 341
 342     def credential_args(self):
 343         return ['--access-key', self.access_key, '--secret', self.secret]
 344
 345 class User(SystemObject):
 346     def __init__(self, uid, data = None, name = None, credentials = None):
 347         self.name = name
 348         self.credentials = credentials or []
 349         super(User, self).__init__(data, uid)
 350
 351     def user_arg(self):
 352         """ command-line argument to specify this user """
 353         return ['--uid', self.id]
 354
 355     def build_command(self, command):
 356         """ build a command line for the given command and args """
 357         return ['user', command] + self.user_arg()
 358
 359     def load_from_json(self, data):
 360         """ load the user from json """
 361         self.id = data['user_id']
 362         self.name = data['display_name']
 363         self.credentials = [Credentials(k['access_key'], k['secret_key']) for k in data['keys']]
 364
 365     def create(self, zone, args = None, **kwargs):
 366         """ create the user with the given arguments """
 367         assert(zone.cluster)
 368         args = zone.zone_args() + (args or [])
 369         return self.json_command(zone.cluster, 'create', args, **kwargs)
 370
 371     def info(self, zone, args = None, **kwargs):
 372         """ read the user from storage """
 373         assert(zone.cluster)
 374         args = zone.zone_args() + (args or [])
 375         kwargs['read_only'] = True
 376         return self.json_command(zone.cluster, 'info', args, **kwargs)
 377
 378     def delete(self, zone, args = None, **kwargs):
 379         """ delete the user """
 380         assert(zone.cluster)
 381         args = zone.zone_args() + (args or [])
 382         return self.command(zone.cluster, 'delete', args, **kwargs)