ceph/qa/tasks/nvme_loop.py

   1 import contextlib
   2 import logging
   3
   4 from io import StringIO
   5 from teuthology import misc as teuthology
   6 from teuthology import contextutil
   7 from teuthology.orchestra import run
   8
   9
  10 log = logging.getLogger(__name__)
  11
  12
  13 @contextlib.contextmanager
  14 def task(ctx, config):
  15     log.info('Setting up nvme_loop on scratch devices...')
  16     host = 'hostnqn'
  17     port = '1'
  18     devs_by_remote = {}
  19     old_scratch_by_remote = {}
  20     for remote, roles in ctx.cluster.remotes.items():
  21         devs = teuthology.get_scratch_devices(remote)
  22         devs_by_remote[remote] = devs
  23         base = '/sys/kernel/config/nvmet'
  24         remote.run(
  25             args=[
  26                 'sudo', 'modprobe', 'nvme_loop',
  27                 run.Raw('&&'),
  28                 'sudo', 'mkdir', '-p', f'{base}/hosts/{host}',
  29                 run.Raw('&&'),
  30                 'sudo', 'mkdir', '-p', f'{base}/ports/{port}',
  31                 run.Raw('&&'),
  32                 'echo', 'loop', run.Raw('|'),
  33                 'sudo', 'tee', f'{base}/ports/{port}/addr_trtype',
  34             ]
  35         )
  36         for dev in devs:
  37             short = dev.split('/')[-1]
  38             log.info(f'Connecting nvme_loop {remote.shortname}:{dev}...')
  39             remote.run(
  40                 args=[
  41                     'sudo', 'mkdir', '-p', f'{base}/subsystems/{short}',
  42                     run.Raw('&&'),
  43                     'echo', '1', run.Raw('|'),
  44                     'sudo', 'tee', f'{base}/subsystems/{short}/attr_allow_any_host',
  45                     run.Raw('&&'),
  46                     'sudo', 'mkdir', '-p', f'{base}/subsystems/{short}/namespaces/1',
  47                     run.Raw('&&'),
  48                     'echo', dev, run.Raw('|'),
  49                     'sudo', 'tee', f'{base}/subsystems/{short}/namespaces/1/device_path',
  50                     run.Raw('&&'),
  51                     'echo', '1', run.Raw('|'),
  52                     'sudo', 'tee', f'{base}/subsystems/{short}/namespaces/1/enable',
  53                     run.Raw('&&'),
  54                     'sudo', 'ln', '-s', f'{base}/subsystems/{short}',
  55                     f'{base}/ports/{port}/subsystems/{short}',
  56                     run.Raw('&&'),
  57                     'sudo', 'nvme', 'connect', '-t', 'loop', '-n', short, '-q', host,
  58                 ]
  59             )
  60
  61         # identify nvme_loops devices
  62         old_scratch_by_remote[remote] = remote.read_file('/scratch_devs')
  63
  64         with contextutil.safe_while(sleep=1, tries=15) as proceed:
  65             while proceed():
  66                 p = remote.run(args=['sudo', 'nvme', 'list'], stdout=StringIO())
  67                 new_devs = []
  68                 for line in p.stdout.getvalue().splitlines():
  69                     dev, _, vendor = line.split()[0:3]
  70                     if dev.startswith('/dev/') and vendor == 'Linux':
  71                         new_devs.append(dev)
  72                 log.info(f'new_devs {new_devs}')
  73                 assert len(new_devs) <= len(devs)
  74                 if len(new_devs) == len(devs):
  75                     break
  76
  77         remote.write_file(
  78             path='/scratch_devs',
  79             data='\n'.join(new_devs) + '\n',
  80             sudo=True
  81         )
  82
  83     try:
  84         yield
  85
  86     finally:
  87         for remote, devs in devs_by_remote.items():
  88             for dev in devs:
  89                 short = dev.split('/')[-1]
  90                 log.info(f'Disconnecting nvme_loop {remote.shortname}:{dev}...')
  91                 remote.run(
  92                     args=[
  93                         'sudo', 'nvme', 'disconnect', '-n', short
  94                     ],
  95                     check_status=False,
  96                 )
  97             remote.write_file(
  98                 path='/scratch_devs',
  99                 data=old_scratch_by_remote[remote],
 100                 sudo=True
 101             )