src/test/test-locked-service2/log.expect

   1 info      0     hardware: starting simulation
   2 info     20      cmdlist: execute power node1 on
   3 info     20    node1/crm: status change startup => wait_for_quorum
   4 info     20    node1/lrm: status change startup => wait_for_agent_lock
   5 info     20      cmdlist: execute power node2 on
   6 info     20    node2/crm: status change startup => wait_for_quorum
   7 info     20    node2/lrm: status change startup => wait_for_agent_lock
   8 info     20      cmdlist: execute power node3 on
   9 info     20    node3/crm: status change startup => wait_for_quorum
  10 info     20    node3/lrm: status change startup => wait_for_agent_lock
  11 info     20    node1/crm: got lock 'ha_manager_lock'
  12 info     20    node1/crm: status change wait_for_quorum => master
  13 info     20    node1/crm: node 'node1': state changed from 'unknown' => 'online'
  14 info     20    node1/crm: node 'node2': state changed from 'unknown' => 'online'
  15 info     20    node1/crm: node 'node3': state changed from 'unknown' => 'online'
  16 info     20    node1/crm: adding new service 'vm:103' on node 'node3'
  17 info     20    node1/crm: service 'vm:103': state changed from 'request_start' to 'started'  (node = node3)
  18 info     22    node2/crm: status change wait_for_quorum => slave
  19 info     24    node3/crm: status change wait_for_quorum => slave
  20 info     25    node3/lrm: got lock 'ha_agent_node3_lock'
  21 info     25    node3/lrm: status change wait_for_agent_lock => active
  22 info     25    node3/lrm: starting service vm:103
  23 info     25    node3/lrm: service status vm:103 started
  24 info    120      cmdlist: execute service vm:103 lock create
  25 info    220      cmdlist: execute network node3 off
  26 info    220    node1/crm: node 'node3': state changed from 'online' => 'unknown'
  27 info    224    node3/crm: status change slave => wait_for_quorum
  28 info    225    node3/lrm: status change active => lost_agent_lock
  29 info    260    node1/crm: service 'vm:103': state changed from 'started' to 'fence'
  30 info    260    node1/crm: node 'node3': state changed from 'unknown' => 'fence'
  31 emai    260    node1/crm: FENCE: Try to fence node 'node3'
  32 info    266     watchdog: execute power node3 off
  33 info    265    node3/crm: killed by poweroff
  34 info    266    node3/lrm: killed by poweroff
  35 info    266     hardware: server 'node3' stopped by poweroff (watchdog)
  36 info    340    node1/crm: got lock 'ha_agent_node3_lock'
  37 info    340    node1/crm: fencing: acknowledged - got agent lock for node 'node3'
  38 info    340    node1/crm: node 'node3': state changed from 'fence' => 'unknown'
  39 emai    340    node1/crm: SUCCEED: fencing: acknowledged - got agent lock for node 'node3'
  40 info    340    node1/crm: service 'vm:103': state changed from 'fence' to 'recovery'
  41 info    340    node1/crm: recover service 'vm:103' from fenced node 'node3' to node 'node1'
  42 info    340    node1/crm: service 'vm:103': state changed from 'recovery' to 'started'  (node = node1)
  43 info    341    node1/lrm: got lock 'ha_agent_node1_lock'
  44 info    341    node1/lrm: status change wait_for_agent_lock => active
  45 info    341    node1/lrm: starting service vm:103
  46 err     341    node1/lrm: service 'vm:103' locked (create), unable to start!
  47 warn    341    node1/lrm: unable to start service vm:103
  48 warn    341    node1/lrm: restart policy: retry number 1 for service 'vm:103'
  49 info    361    node1/lrm: starting service vm:103
  50 err     361    node1/lrm: service 'vm:103' locked (create), unable to start!
  51 warn    361    node1/lrm: unable to start service vm:103
  52 err     361    node1/lrm: unable to start service vm:103 on local node after 1 retries
  53 warn    380    node1/crm: starting service vm:103 on node 'node1' failed, relocating service.
  54 info    380    node1/crm: migrate service 'vm:103' to node 'node2' (running)
  55 info    380    node1/crm: service 'vm:103': state changed from 'started' to 'migrate'  (node = node1, target = node2)
  56 info    381    node1/lrm: service vm:103 - start migrate to node 'node2'
  57 err     381    node1/lrm: service 'vm:103' locked (create), unable to migrate!
  58 err     381    node1/lrm: service vm:103 not moved (migration error)
  59 err     400    node1/crm: service 'vm:103' - migration failed (exit code 1)
  60 info    400    node1/crm: service 'vm:103': state changed from 'migrate' to 'started'  (node = node1)
  61 info    401    node1/lrm: starting service vm:103
  62 err     401    node1/lrm: service 'vm:103' locked (create), unable to start!
  63 warn    401    node1/lrm: unable to start service vm:103
  64 warn    401    node1/lrm: restart policy: retry number 1 for service 'vm:103'
  65 info    421    node1/lrm: starting service vm:103
  66 err     421    node1/lrm: service 'vm:103' locked (create), unable to start!
  67 warn    421    node1/lrm: unable to start service vm:103
  68 err     421    node1/lrm: unable to start service vm:103 on local node after 1 retries
  69 err     440    node1/crm: recovery policy for service vm:103 failed, entering error state. Failed nodes: node1, node1
  70 info    440    node1/crm: service 'vm:103': state changed from 'started' to 'error'
  71 err     441    node1/lrm: service vm:103 is in an error state and needs manual intervention. Look up 'ERROR RECOVERY' in the documentation.
  72 info    820     hardware: exit simulation - done