tools/cgroup/iocost_monitor.py

   1 #!/usr/bin/env drgn
   2 #
   3 # Copyright (C) 2019 Tejun Heo <tj@kernel.org>
   4 # Copyright (C) 2019 Facebook
   5
   6 desc = """
   7 This is a drgn script to monitor the blk-iocost cgroup controller.
   8 See the comment at the top of block/blk-iocost.c for more details.
   9 For drgn, visit https://github.com/osandov/drgn.
  10 """
  11
  12 import sys
  13 import re
  14 import time
  15 import json
  16 import math
  17
  18 import drgn
  19 from drgn import container_of
  20 from drgn.helpers.linux.list import list_for_each_entry,list_empty
  21 from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup
  22
  23 import argparse
  24 parser = argparse.ArgumentParser(description=desc,
  25                                  formatter_class=argparse.RawTextHelpFormatter)
  26 parser.add_argument('devname', metavar='DEV',
  27                     help='Target block device name (e.g. sda)')
  28 parser.add_argument('--cgroup', action='append', metavar='REGEX',
  29                     help='Regex for target cgroups, ')
  30 parser.add_argument('--interval', '-i', metavar='SECONDS', type=float, default=1,
  31                     help='Monitoring interval in seconds (0 exits immediately '
  32                     'after checking requirements)')
  33 parser.add_argument('--json', action='store_true',
  34                     help='Output in json')
  35 args = parser.parse_args()
  36
  37 def err(s):
  38     print(s, file=sys.stderr, flush=True)
  39     sys.exit(1)
  40
  41 try:
  42     blkcg_root = prog['blkcg_root']
  43     plid = prog['blkcg_policy_iocost'].plid.value_()
  44 except:
  45     err('The kernel does not have iocost enabled')
  46
  47 IOC_RUNNING     = prog['IOC_RUNNING'].value_()
  48 NR_USAGE_SLOTS  = prog['NR_USAGE_SLOTS'].value_()
  49 HWEIGHT_WHOLE   = prog['HWEIGHT_WHOLE'].value_()
  50 VTIME_PER_SEC   = prog['VTIME_PER_SEC'].value_()
  51 VTIME_PER_USEC  = prog['VTIME_PER_USEC'].value_()
  52 AUTOP_SSD_FAST  = prog['AUTOP_SSD_FAST'].value_()
  53 AUTOP_SSD_DFL   = prog['AUTOP_SSD_DFL'].value_()
  54 AUTOP_SSD_QD1   = prog['AUTOP_SSD_QD1'].value_()
  55 AUTOP_HDD       = prog['AUTOP_HDD'].value_()
  56
  57 autop_names = {
  58     AUTOP_SSD_FAST:        'ssd_fast',
  59     AUTOP_SSD_DFL:         'ssd_dfl',
  60     AUTOP_SSD_QD1:         'ssd_qd1',
  61     AUTOP_HDD:             'hdd',
  62 }
  63
  64 class BlkgIterator:
  65     def blkcg_name(blkcg):
  66         return blkcg.css.cgroup.kn.name.string_().decode('utf-8')
  67
  68     def walk(self, blkcg, q_id, parent_path):
  69         if not self.include_dying and \
  70            not (blkcg.css.flags.value_() & prog['CSS_ONLINE'].value_()):
  71             return
  72
  73         name = BlkgIterator.blkcg_name(blkcg)
  74         path = parent_path + '/' + name if parent_path else name
  75         blkg = drgn.Object(prog, 'struct blkcg_gq',
  76                            address=radix_tree_lookup(blkcg.blkg_tree.address_of_(), q_id))
  77         if not blkg.address_:
  78             return
  79
  80         self.blkgs.append((path if path else '/', blkg))
  81
  82         for c in list_for_each_entry('struct blkcg',
  83                                      blkcg.css.children.address_of_(), 'css.sibling'):
  84             self.walk(c, q_id, path)
  85
  86     def __init__(self, root_blkcg, q_id, include_dying=False):
  87         self.include_dying = include_dying
  88         self.blkgs = []
  89         self.walk(root_blkcg, q_id, '')
  90
  91     def __iter__(self):
  92         return iter(self.blkgs)
  93
  94 class IocStat:
  95     def __init__(self, ioc):
  96         global autop_names
  97
  98         self.enabled = ioc.enabled.value_()
  99         self.running = ioc.running.value_() == IOC_RUNNING
 100         self.period_ms = ioc.period_us.value_() / 1_000
 101         self.period_at = ioc.period_at.value_() / 1_000_000
 102         self.vperiod_at = ioc.period_at_vtime.value_() / VTIME_PER_SEC
 103         self.vrate_pct = ioc.vtime_rate.counter.value_() * 100 / VTIME_PER_USEC
 104         self.busy_level = ioc.busy_level.value_()
 105         self.autop_idx = ioc.autop_idx.value_()
 106         self.user_cost_model = ioc.user_cost_model.value_()
 107         self.user_qos_params = ioc.user_qos_params.value_()
 108
 109         if self.autop_idx in autop_names:
 110             self.autop_name = autop_names[self.autop_idx]
 111         else:
 112             self.autop_name = '?'
 113
 114     def dict(self, now):
 115         return { 'device'               : devname,
 116                  'timestamp'            : now,
 117                  'enabled'              : self.enabled,
 118                  'running'              : self.running,
 119                  'period_ms'            : self.period_ms,
 120                  'period_at'            : self.period_at,
 121                  'period_vtime_at'      : self.vperiod_at,
 122                  'busy_level'           : self.busy_level,
 123                  'vrate_pct'            : self.vrate_pct, }
 124
 125     def table_preamble_str(self):
 126         state = ('RUN' if self.running else 'IDLE') if self.enabled else 'OFF'
 127         output = f'{devname} {state:4} ' \
 128                  f'per={self.period_ms}ms ' \
 129                  f'cur_per={self.period_at:.3f}:v{self.vperiod_at:.3f} ' \
 130                  f'busy={self.busy_level:+3} ' \
 131                  f'vrate={self.vrate_pct:6.2f}% ' \
 132                  f'params={self.autop_name}'
 133         if self.user_cost_model or self.user_qos_params:
 134             output += f'({"C" if self.user_cost_model else ""}{"Q" if self.user_qos_params else ""})'
 135         return output
 136
 137     def table_header_str(self):
 138         return f'{"":25} active {"weight":>9} {"hweight%":>13} {"inflt%":>6} ' \
 139                f'{"dbt":>3} {"delay":>6} {"usages%"}'
 140
 141 class IocgStat:
 142     def __init__(self, iocg):
 143         ioc = iocg.ioc
 144         blkg = iocg.pd.blkg
 145
 146         self.is_active = not list_empty(iocg.active_list.address_of_())
 147         self.weight = iocg.weight.value_()
 148         self.active = iocg.active.value_()
 149         self.inuse = iocg.inuse.value_()
 150         self.hwa_pct = iocg.hweight_active.value_() * 100 / HWEIGHT_WHOLE
 151         self.hwi_pct = iocg.hweight_inuse.value_() * 100 / HWEIGHT_WHOLE
 152         self.address = iocg.value_()
 153
 154         vdone = iocg.done_vtime.counter.value_()
 155         vtime = iocg.vtime.counter.value_()
 156         vrate = ioc.vtime_rate.counter.value_()
 157         period_vtime = ioc.period_us.value_() * vrate
 158         if period_vtime:
 159             self.inflight_pct = (vtime - vdone) * 100 / period_vtime
 160         else:
 161             self.inflight_pct = 0
 162
 163         # vdebt used to be an atomic64_t and is now u64, support both
 164         try:
 165             self.debt_ms = iocg.abs_vdebt.counter.value_() / VTIME_PER_USEC / 1000
 166         except:
 167             self.debt_ms = iocg.abs_vdebt.value_() / VTIME_PER_USEC / 1000
 168
 169         self.use_delay = blkg.use_delay.counter.value_()
 170         self.delay_ms = blkg.delay_nsec.counter.value_() / 1_000_000
 171
 172         usage_idx = iocg.usage_idx.value_()
 173         self.usages = []
 174         self.usage = 0
 175         for i in range(NR_USAGE_SLOTS):
 176             usage = iocg.usages[(usage_idx + 1 + i) % NR_USAGE_SLOTS].value_()
 177             upct = usage * 100 / HWEIGHT_WHOLE
 178             self.usages.append(upct)
 179             self.usage = max(self.usage, upct)
 180
 181     def dict(self, now, path):
 182         out = { 'cgroup'                : path,
 183                 'timestamp'             : now,
 184                 'is_active'             : self.is_active,
 185                 'weight'                : self.weight,
 186                 'weight_active'         : self.active,
 187                 'weight_inuse'          : self.inuse,
 188                 'hweight_active_pct'    : self.hwa_pct,
 189                 'hweight_inuse_pct'     : self.hwi_pct,
 190                 'inflight_pct'          : self.inflight_pct,
 191                 'debt_ms'               : self.debt_ms,
 192                 'use_delay'             : self.use_delay,
 193                 'delay_ms'              : self.delay_ms,
 194                 'usage_pct'             : self.usage,
 195                 'address'               : self.address }
 196         for i in range(len(self.usages)):
 197             out[f'usage_pct_{i}'] = str(self.usages[i])
 198         return out
 199
 200     def table_row_str(self, path):
 201         out = f'{path[-28:]:28} ' \
 202               f'{"*" if self.is_active else " "} ' \
 203               f'{self.inuse:5}/{self.active:5} ' \
 204               f'{self.hwi_pct:6.2f}/{self.hwa_pct:6.2f} ' \
 205               f'{self.inflight_pct:6.2f} ' \
 206               f'{min(math.ceil(self.debt_ms), 999):3} ' \
 207               f'{min(self.use_delay, 99):2}*'\
 208               f'{min(math.ceil(self.delay_ms), 999):03} '
 209         for u in self.usages:
 210             out += f'{min(round(u), 999):03d}:'
 211         out = out.rstrip(':')
 212         return out
 213
 214 # handle args
 215 table_fmt = not args.json
 216 interval = args.interval
 217 devname = args.devname
 218
 219 if args.json:
 220     table_fmt = False
 221
 222 re_str = None
 223 if args.cgroup:
 224     for r in args.cgroup:
 225         if re_str is None:
 226             re_str = r
 227         else:
 228             re_str += '|' + r
 229
 230 filter_re = re.compile(re_str) if re_str else None
 231
 232 # Locate the roots
 233 q_id = None
 234 root_iocg = None
 235 ioc = None
 236
 237 for i, ptr in radix_tree_for_each(blkcg_root.blkg_tree.address_of_()):
 238     blkg = drgn.Object(prog, 'struct blkcg_gq', address=ptr)
 239     try:
 240         if devname == blkg.q.kobj.parent.name.string_().decode('utf-8'):
 241             q_id = blkg.q.id.value_()
 242             if blkg.pd[plid]:
 243                 root_iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 244                 ioc = root_iocg.ioc
 245             break
 246     except:
 247         pass
 248
 249 if ioc is None:
 250     err(f'Could not find ioc for {devname}');
 251
 252 if interval == 0:
 253     sys.exit(0)
 254
 255 # Keep printing
 256 while True:
 257     now = time.time()
 258     iocstat = IocStat(ioc)
 259     output = ''
 260
 261     if table_fmt:
 262         output += '\n' + iocstat.table_preamble_str()
 263         output += '\n' + iocstat.table_header_str()
 264     else:
 265         output += json.dumps(iocstat.dict(now))
 266
 267     for path, blkg in BlkgIterator(blkcg_root, q_id):
 268         if filter_re and not filter_re.match(path):
 269             continue
 270         if not blkg.pd[plid]:
 271             continue
 272
 273         iocg = container_of(blkg.pd[plid], 'struct ioc_gq', 'pd')
 274         iocg_stat = IocgStat(iocg)
 275
 276         if not filter_re and not iocg_stat.is_active:
 277             continue
 278
 279         if table_fmt:
 280             output += '\n' + iocg_stat.table_row_str(path)
 281         else:
 282             output += '\n' + json.dumps(iocg_stat.dict(now, path))
 283
 284     print(output)
 285     sys.stdout.flush()
 286     time.sleep(interval)