ceph/src/pmdk/src/tools/pmreorder/statemachine.py

   1 # SPDX-License-Identifier: BSD-3-Clause
   2 # Copyright 2018-2020, Intel Corporation
   3
   4 import memoryoperations as memops
   5 import reorderengines
   6 from reorderexceptions import InconsistentFileException
   7 from reorderexceptions import NotSupportedOperationException
   8
   9
  10 class State:
  11     """
  12     The base class of all states.
  13
  14     :ivar _context: The reordering context.
  15     :type _context: opscontext.OpsContext
  16     :ivar trans_stores: The list of unflushed stores.
  17     :type trans_stores: list of :class:`memoryoperations.Store`
  18     """
  19     trans_stores = []
  20
  21     def __init__(self, context):
  22         """
  23         Default state constructor.
  24
  25         :param context: The context of the reordering.
  26         :type context: opscontext.OpsContext
  27         """
  28         self._context = context
  29
  30     def next(self, in_op):
  31         """
  32         Go to the next state based on the input.
  33
  34         :Note:
  35             The next state might in fact be the same state.
  36
  37         :param in_op: The state switch trigger operation.
  38         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
  39         :return: The next state.
  40         :rtype: subclass of :class:`State`
  41         """
  42         raise NotImplementedError
  43
  44     def run(self, in_op):
  45         """
  46         Perform the required operation in this state.
  47
  48         :param in_op: The operation to be performed in this state.
  49         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
  50         :return: None
  51         """
  52         raise NotImplementedError
  53
  54
  55 class InitState(State):
  56     """
  57     The initial no-op state.
  58     """
  59     def __init__(self, context):
  60         """
  61         Saves the reordering context.
  62
  63         :param context: The reordering context.
  64         :type context: opscontext.OpsContext
  65         """
  66         super(InitState, self).__init__(context)
  67
  68     def next(self, in_op):
  69         """
  70         Switch to the next valid state.
  71
  72         :param in_op: Ignored.
  73         :return: The next valid state.
  74         :rtype: CollectingState
  75         """
  76         return CollectingState(self._context)
  77
  78     def run(self, in_op):
  79         """
  80         Does nothing.
  81
  82         :param in_op: Ignored.
  83         :return: always True
  84         """
  85         return True
  86
  87
  88 class CollectingState(State):
  89     """
  90     Collects appropriate operations.
  91
  92     This state mostly aggregates stores and flushes. It also
  93     validates which stores will be made persistent and passes
  94     them on to the next state.
  95
  96     :ivar _ops_list: The list of collected stores.
  97     :type _ops_list: list of :class:`memoryoperations.Store`
  98     :ivar _inner_state: The internal state of operations.
  99     :type _inner_state: str
 100     """
 101     def __init__(self, context):
 102         """
 103         Saves the reordering context.
 104
 105         :param context: The reordering context.
 106         :type context: opscontext.OpsContext
 107         """
 108         super(CollectingState, self).__init__(context)
 109         self._ops_list = []
 110         self._ops_list += State.trans_stores
 111         self._inner_state = "init"
 112
 113     def next(self, in_op):
 114         """
 115         Switch to the next valid state.
 116
 117         :param in_op: The state switch trigger operation.
 118         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
 119         :return: The next state.
 120         :rtype: subclass of :class:`State`
 121         """
 122         if isinstance(in_op, memops.Fence) and \
 123                 self._inner_state == "flush":
 124             return ReplayingState(self._ops_list, self._context)
 125         else:
 126             return self
 127
 128     def run(self, in_op):
 129         """
 130         Perform operations in this state.
 131
 132         Based on the type of operation, different handling is employed.
 133         The recognized and handled types of operations are:
 134
 135             * :class:`memoryoperations.ReorderBase`
 136             * :class:`memoryoperations.FlushBase`
 137             * :class:`memoryoperations.Store`
 138             * :class:`memoryoperations.Register_file`
 139
 140         :param in_op: The operation to be performed in this state.
 141         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
 142         :return: always True
 143         """
 144         self.move_inner_state(in_op)
 145         if isinstance(in_op, memops.ReorderBase):
 146             self.substitute_reorder(in_op)
 147         elif isinstance(in_op, memops.FlushBase):
 148             self.flush_stores(in_op)
 149         elif isinstance(in_op, memops.Store):
 150             self._ops_list.append(in_op)
 151         elif isinstance(in_op, memops.Register_file):
 152             self.reg_file(in_op)
 153
 154         return True
 155
 156     def substitute_reorder(self, order_ops):
 157         """
 158         Changes the reordering engine based on the log marker class.
 159
 160         :param order_ops: The reordering marker class.
 161         :type order_ops: subclass of :class:`memoryoperations.ReorderBase`
 162         :return: None
 163         """
 164         if isinstance(order_ops, memops.ReorderFull):
 165             self._context.reorder_engine = \
 166                 reorderengines.FullReorderEngine()
 167             self._context.test_on_barrier = \
 168                 self._context.reorder_engine.test_on_barrier
 169         elif isinstance(order_ops, memops.ReorderPartial):
 170             # TODO add macro in valgrind or
 171             # parameter inside the tool to support parameters?
 172             self._context.reorder_engine = \
 173                  reorderengines.RandomPartialReorderEngine(3)
 174             self._context.test_on_barrier = \
 175                 self._context.reorder_engine.test_on_barrier
 176         elif isinstance(order_ops, memops.ReorderAccumulative):
 177             self._context.reorder_engine = \
 178                 reorderengines.AccumulativeReorderEngine()
 179             self._context.test_on_barrier = \
 180                 self._context.reorder_engine.test_on_barrier
 181         elif isinstance(order_ops, memops.ReorderReverseAccumulative):
 182             self._context.reorder_engine = \
 183                 reorderengines.AccumulativeReverseReorderEngine()
 184             self._context.test_on_barrier = \
 185                 self._context.reorder_engine.test_on_barrier
 186         elif isinstance(order_ops, memops.NoReorderDoCheck):
 187             self._context.reorder_engine = reorderengines.NoReorderEngine()
 188             self._context.test_on_barrier = \
 189                 self._context.reorder_engine.test_on_barrier
 190         elif isinstance(order_ops, memops.NoReorderNoCheck):
 191             self._context.reorder_engine = reorderengines.NoCheckerEngine()
 192             self._context.test_on_barrier = \
 193                 self._context.reorder_engine.test_on_barrier
 194         elif isinstance(order_ops, memops.ReorderDefault):
 195             self._context.reorder_engine = self._context.default_engine
 196             self._context.test_on_barrier = self._context.default_barrier
 197         else:
 198             raise NotSupportedOperationException(
 199                                    "Not supported reorder engine: {}"
 200                                    .format(order_ops))
 201
 202     def flush_stores(self, flush_op):
 203         """
 204         Marks appropriate stores as flushed.
 205
 206         Does not align the flush, the log is expected to have the
 207         flushes properly aligned.
 208
 209         :param flush_op: The flush operation marker.
 210         :type flush_op: subclass of :class:`memoryoperations.FlushBase`
 211         :return: None
 212         """
 213         for st in self._ops_list:
 214             if flush_op.is_in_flush(st):
 215                 st.flushed = True
 216
 217     def reg_file(self, file_op):
 218         """
 219         Register a new file mapped into virtual memory.
 220
 221         :param file_op: File registration operation marker.
 222         :type file_op: memoryoperations.Register_file
 223         :return: None
 224         """
 225         self._context.file_handler.add_file(file_op.name,
 226                                             file_op.address,
 227                                             file_op.size)
 228
 229     def move_inner_state(self, in_op):
 230         """
 231         Tracks the internal state of the collection.
 232
 233         The collected stores need to be processed only at specific moments -
 234         after full persistent memory barriers (flush-fence).
 235
 236         :param in_op: The performed operation.
 237         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
 238         :return: None
 239         """
 240         if isinstance(in_op, memops.Store) and \
 241                 self._inner_state == "init":
 242             self._inner_state = "dirty"
 243         elif isinstance(in_op, memops.FlushBase) and \
 244                 self._inner_state == "dirty":
 245             self._inner_state = "flush"
 246         elif isinstance(in_op, memops.Fence) and \
 247                 self._inner_state == "flush":
 248             self._inner_state = "fence"
 249         elif isinstance(in_op, memops.Flush) and \
 250                 self._inner_state == "init":
 251             self._inner_state = "flush"
 252
 253
 254 class ReplayingState(State):
 255     """
 256     Replays all collected stores according to the reordering context.
 257
 258     :ivar _ops_list: The list of stores to be reordered and replayed.
 259     :type _ops_list: list of :class:`memoryoperations.Store`
 260     """
 261     def __init__(self, in_ops_list, context):
 262         """
 263
 264         :param in_ops_list:
 265         :param context:
 266         :return:
 267         """
 268         super(ReplayingState, self).__init__(context)
 269         self._ops_list = in_ops_list
 270
 271     def next(self, in_op):
 272         """
 273         Switches to the collecting state regardless of the input.
 274
 275         :param in_op: Ignored.
 276         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
 277         :return: The next state.
 278         :rtype: CollectingState
 279         """
 280         return CollectingState(self._context)
 281
 282     def run(self, in_op):
 283         """
 284         Perform operations in this state.
 285
 286         The replaying state performs reordering and if necessary checks
 287         the consistency of the registered files. The decisions and
 288         type of reordering to be used is defined by the context.
 289
 290         :param in_op: The operation to be performed in this state.
 291         :type in_op: subclass of :class:`memoryoperations.BaseOperation`
 292         :return: State of consistency check.
 293         """
 294         # specifies consistency state of sequence
 295         consistency = True
 296
 297         # consider only flushed stores
 298         flushed_stores = list(filter(lambda x: x.flushed, self._ops_list))
 299
 300         # not-flushed stores should be passed to next state
 301         State.trans_stores = list(filter(lambda x: x.flushed is False,
 302                                          self._ops_list))
 303
 304         if self._context.test_on_barrier:
 305             for seq in self._context.reorder_engine.generate_sequence(
 306                                                               flushed_stores):
 307                 for op in seq:
 308                     # do stores
 309                     self._context.file_handler.do_store(op)
 310                 # check consistency of all files
 311                 try:
 312                     self._context.file_handler.check_consistency()
 313                 except InconsistentFileException as e:
 314                     consistency = False
 315                     self._context.logger.warning(e)
 316                     stacktrace = "Call trace:\n"
 317                     for num, op in enumerate(seq):
 318                         stacktrace += "Store [{}]:\n".format(num)
 319                         stacktrace += str(op.trace)
 320                     self._context.logger.warning(stacktrace)
 321
 322                 for op in reversed(seq):
 323                     # revert the changes
 324                     self._context.file_handler.do_revert(op)
 325         # write all flushed stores
 326         for op in flushed_stores:
 327             self._context.file_handler.do_store(op)
 328
 329         return consistency
 330
 331
 332 class StateMachine:
 333     """
 334     The state machine driver.
 335
 336     :ivar _curr_state: The current state.
 337     :type _curr_state: subclass of :class:`State`
 338     """
 339     def __init__(self, init_state):
 340         """
 341         Initialize the state machine with a specified state.
 342
 343         :param init_state: The initial state to be used.
 344         :type init_state: subclass of :class:`State`
 345         """
 346         self._curr_state = init_state
 347
 348     def run_all(self, operations):
 349         """
 350         Starts the state machine.
 351
 352         :param operations: The operations to be performed by the state
 353             machine.
 354         :type operations: list of :class:`memoryoperations.BaseOperation`
 355         :return: None
 356         """
 357         all_consistent = True
 358         for ops in operations:
 359             self._curr_state = self._curr_state.next(ops)
 360             check = self._curr_state.run(ops)
 361             if check is False:
 362                 all_consistent = check
 363
 364         return all_consistent