src/test/test-recovery4/log.expect

   1 info      0     hardware: starting simulation
   2 info     20      cmdlist: execute power node1 on
   3 info     20    node1/crm: status change startup => wait_for_quorum
   4 info     20    node1/lrm: status change startup => wait_for_agent_lock
   5 info     20      cmdlist: execute power node2 on
   6 info     20    node2/crm: status change startup => wait_for_quorum
   7 info     20    node2/lrm: status change startup => wait_for_agent_lock
   8 info     20      cmdlist: execute power node3 on
   9 info     20    node3/crm: status change startup => wait_for_quorum
  10 info     20    node3/lrm: status change startup => wait_for_agent_lock
  11 info     20    node1/crm: got lock 'ha_manager_lock'
  12 info     20    node1/crm: status change wait_for_quorum => master
  13 info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
  14 info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
  15 info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
  16 info     20    node1/crm: adding new service 'vm:102' on node 'node2'
  17 info     22    node2/crm: status change wait_for_quorum => slave
  18 info     23    node2/lrm: got lock 'ha_agent_node2_lock'
  19 info     23    node2/lrm: status change wait_for_agent_lock => active
  20 info     23    node2/lrm: starting service vm:102
  21 info     23    node2/lrm: service status vm:102 started
  22 info     24    node3/crm: status change wait_for_quorum => slave
  23 info    120      cmdlist: execute network node2 off
  24 info    120    node1/crm: node 'node2': state changed from 'online' => 'unknown'
  25 info    122    node2/crm: status change slave => wait_for_quorum
  26 info    123    node2/lrm: status change active => lost_agent_lock
  27 info    160    node1/crm: service 'vm:102': state changed from 'started' to 'fence'
  28 info    160    node1/crm: node 'node2': state changed from 'unknown' => 'fence'
  29 emai    160    node1/crm: FENCE: Try to fence node 'node2'
  30 info    164     watchdog: execute power node2 off
  31 info    163    node2/crm: killed by poweroff
  32 info    164    node2/lrm: killed by poweroff
  33 info    164     hardware: server 'node2' stopped by poweroff (watchdog)
  34 info    220      cmdlist: execute delay 0
  35 info    240    node1/crm: got lock 'ha_agent_node2_lock'
  36 info    240    node1/crm: fencing: acknowledged - got agent lock for node 'node2'
  37 info    240    node1/crm: node 'node2': state changed from 'fence' => 'unknown'
  38 emai    240    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node2'
  39 info    240    node1/crm: service 'vm:102': state changed from 'fence' to 'recovery'
  40 err     240    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  41 err     260    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  42 err     280    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  43 err     300    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  44 info    320      cmdlist: execute service vm:102 migrate node3
  45 info    320    node1/crm: got crm command: migrate vm:102 node3
  46 err     320    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  47 err     340    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  48 err     360    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  49 err     380    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  50 err     400    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  51 info    420      cmdlist: execute service vm:102 stop 5
  52 info    420    node1/crm: got crm command: stop vm:102 5
  53 err     420    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  54 err     440    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  55 err     460    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  56 err     480    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  57 err     500    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  58 info    520      cmdlist: execute service vm:102 disabled
  59 err     520    node1/crm: recovering service 'vm:102' from fenced node 'node2' failed, no recovery node found
  60 info    520    node1/crm: service 'vm:102': state changed from 'recovery' to 'stopped'
  61 info   1120     hardware: exit simulation - done