ceph/qa/workunits/rgw/common.py

   1 #!/usr/bin/env python3
   2
   3 import errno
   4 import subprocess
   5 import logging as log
   6 import boto3
   7 import botocore.exceptions
   8 import random
   9 import json
  10 from time import sleep
  11
  12 log.basicConfig(format = '%(message)s', level=log.DEBUG)
  13 log.getLogger('botocore').setLevel(log.CRITICAL)
  14 log.getLogger('boto3').setLevel(log.CRITICAL)
  15 log.getLogger('urllib3').setLevel(log.CRITICAL)
  16
  17 def exec_cmd(cmd, wait = True, **kwargs):
  18     check_retcode = kwargs.pop('check_retcode', True)
  19     kwargs['shell'] = True
  20     kwargs['stdout'] = subprocess.PIPE
  21     proc = subprocess.Popen(cmd, **kwargs)
  22     log.info(proc.args)
  23     if wait:
  24         out, _ = proc.communicate()
  25         if check_retcode:
  26             assert(proc.returncode == 0)
  27             return out
  28         return (out, proc.returncode)
  29     return ''
  30
  31 def create_user(uid, display_name, access_key, secret_key):
  32     _, ret = exec_cmd(f'radosgw-admin user create --uid {uid} --display-name "{display_name}" --access-key {access_key} --secret {secret_key}', check_retcode=False)
  33     assert(ret == 0 or errno.EEXIST)
  34
  35 def boto_connect(access_key, secret_key, config=None):
  36     def try_connect(portnum, ssl, proto):
  37         endpoint = proto + '://localhost:' + portnum
  38         conn = boto3.resource('s3',
  39                               aws_access_key_id=access_key,
  40                               aws_secret_access_key=secret_key,
  41                               use_ssl=ssl,
  42                               endpoint_url=endpoint,
  43                               verify=False,
  44                               config=config,
  45                               )
  46         try:
  47             list(conn.buckets.limit(1)) # just verify we can list buckets
  48         except botocore.exceptions.ConnectionError as e:
  49             print(e)
  50             raise
  51         print('connected to', endpoint)
  52         return conn
  53     try:
  54         return try_connect('80', False, 'http')
  55     except botocore.exceptions.ConnectionError:
  56         try: # retry on non-privileged http port
  57             return try_connect('8000', False, 'http')
  58         except botocore.exceptions.ConnectionError:
  59             # retry with ssl
  60             return try_connect('443', True, 'https')
  61
  62 def put_objects(bucket, key_list):
  63     objs = []
  64     for key in key_list:
  65         o = bucket.put_object(Key=key, Body=b"some_data")
  66         objs.append((o.key, o.version_id))
  67     return objs
  68
  69 def create_unlinked_objects(conn, bucket, key_list):
  70     # creates an unlinked/unlistable object for each key in key_list
  71
  72     object_versions = []
  73     try:
  74         exec_cmd('ceph config set client rgw_debug_inject_set_olh_err 2')
  75         exec_cmd('ceph config set client rgw_debug_inject_olh_cancel_modification_err true')
  76         sleep(1)
  77         for key in key_list:
  78             tag = str(random.randint(0, 1_000_000))
  79             try:
  80                 bucket.put_object(Key=key, Body=b"some_data", Metadata = {
  81                     'tag': tag,
  82                 })
  83             except Exception as e:
  84                 log.debug(e)
  85             out = exec_cmd(f'radosgw-admin bi list --bucket {bucket.name} --object {key}')
  86             instance_entries = filter(
  87                 lambda x: x['type'] == 'instance',
  88                 json.loads(out.replace(b'\x80', b'0x80')))
  89             found = False
  90             for ie in instance_entries:
  91                 instance_id = ie['entry']['instance']
  92                 ov = conn.ObjectVersion(bucket.name, key, instance_id).head()
  93                 if ov['Metadata'] and ov['Metadata']['tag'] == tag:
  94                     object_versions.append((key, instance_id))
  95                     found = True
  96                     break
  97             if not found:
  98                 raise Exception(f'failed to create unlinked object for key={key}')
  99     finally:
 100         exec_cmd('ceph config rm client rgw_debug_inject_set_olh_err')
 101         exec_cmd('ceph config rm client rgw_debug_inject_olh_cancel_modification_err')
 102     return object_versions
 103