From 7e7fc73b49d9ea90bdb3f5821592f832056f336f Mon Sep 17 00:00:00 2001 From: Martin Winter Date: Fri, 19 May 2017 20:41:20 -0700 Subject: [PATCH] (all tests): Add extra check to make sure daemons are still running after each essential stage. If daemon crashed at a later stage (not startup), then the test scripts didn't properly detect it and report unpredictable errors. This will properly log the daemon crashes Signed-off-by: Martin Winter --- .../test_all_protocol_startup.py | 55 +++++++++++++++++++ .../test_bgp_multiview_topo1.py | 10 ++++ tests/topotests/ldp-topo1/test_ldp_topo1.py | 40 ++++++++++++++ .../topotests/ospf6-topo1/test_ospf6_topo1.py | 13 ++++- .../topotests/ripng-topo1/test_ripng_topo1.py | 20 +++++++ 5 files changed, 136 insertions(+), 2 deletions(-) diff --git a/tests/topotests/all-protocol-startup/test_all_protocol_startup.py b/tests/topotests/all-protocol-startup/test_all_protocol_startup.py index c02db441d..6e2f12ac5 100755 --- a/tests/topotests/all-protocol-startup/test_all_protocol_startup.py +++ b/tests/topotests/all-protocol-startup/test_all_protocol_startup.py @@ -205,6 +205,11 @@ def test_error_messages_vtysh(): assert vtystderr == '', "Vtysh StdErr Output check failed for router r%s:\n%s" % (i, vtystderr) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -291,6 +296,11 @@ def test_converge_protocols(): # Not really implemented yet - just sleep 60 secs for now sleep(60) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line ## CLI(net) @@ -339,6 +349,11 @@ def test_rip_status(): assert failures == 0, "IP RIP status failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -389,6 +404,11 @@ def test_ripng_status(): assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -443,6 +463,11 @@ def test_ospfv2_interfaces(): assert failures == 0, "SHOW IP OSPF INTERFACE failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -491,6 +516,11 @@ def test_isis_interfaces(): assert failures == 0, "SHOW ISIS INTERFACE DETAIL failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -567,6 +597,11 @@ def test_bgp_summary(): assert failures == 0, "SHOW IP BGP SUMMARY failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -644,6 +679,11 @@ def test_bgp_ipv6_summary(): assert failures == 0, "SHOW BGP IPv6 SUMMARY failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -692,6 +732,11 @@ def test_bgp_ipv4(): assert failures == 0, "SHOW BGP IPv4 failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -740,6 +785,11 @@ def test_bgp_ipv6(): assert failures == 0, "SHOW BGP IPv6 failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -794,6 +844,11 @@ def test_mpls_interfaces(): assert failures == 0, "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) diff --git a/tests/topotests/bgp_multiview_topo1/test_bgp_multiview_topo1.py b/tests/topotests/bgp_multiview_topo1/test_bgp_multiview_topo1.py index 8c33156aa..d7c350f5e 100755 --- a/tests/topotests/bgp_multiview_topo1/test_bgp_multiview_topo1.py +++ b/tests/topotests/bgp_multiview_topo1/test_bgp_multiview_topo1.py @@ -251,6 +251,11 @@ def test_bgp_converge(): # print("\nwaiting 15s for routes to populate") # sleep(15) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting Quagga/FRR daemons, uncomment the next line # CLI(net) @@ -307,6 +312,11 @@ def test_bgp_routingTable(): assert failures == 0, "Routing Table verification failed for router r%s, view %s:\n%s" % (i, view, diff) + # Make sure that all daemons are running + for i in range(1, 2): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) diff --git a/tests/topotests/ldp-topo1/test_ldp_topo1.py b/tests/topotests/ldp-topo1/test_ldp_topo1.py index 77857ad92..af2bae185 100755 --- a/tests/topotests/ldp-topo1/test_ldp_topo1.py +++ b/tests/topotests/ldp-topo1/test_ldp_topo1.py @@ -252,6 +252,11 @@ def test_mpls_interfaces(): assert failures == 0, "MPLS LDP Interface status failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -313,6 +318,11 @@ def test_mpls_ldp_neighbor_establish(): print("\nwaiting 15s for LDP sessions to establish") sleep(15) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + def test_mpls_ldp_discovery(): global fatal_error @@ -360,6 +370,11 @@ def test_mpls_ldp_discovery(): assert failures == 0, "MPLS LDP Interface discovery output for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -420,6 +435,11 @@ def test_mpls_ldp_neighbor(): assert failures == 0, "MPLS LDP Interface neighbor output for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line #CLI(net) @@ -496,6 +516,11 @@ def test_mpls_ldp_binding(): assert failures == 0, "MPLS LDP Interface binding output for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line #CLI(net) @@ -549,6 +574,11 @@ def test_zebra_ipv4_routingTable(): assert failures == 0, "IPv4 Zebra Routing Table verification failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -614,6 +644,11 @@ def test_mpls_table(): assert failures == 0, "MPLS table output for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -713,6 +748,11 @@ def test_linux_mpls_routes(): assert failures == 0, "Linux Kernel MPLS route output for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) diff --git a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py index 82f4d1c08..90489e4be 100755 --- a/tests/topotests/ospf6-topo1/test_ospf6_topo1.py +++ b/tests/topotests/ospf6-topo1/test_ospf6_topo1.py @@ -208,8 +208,7 @@ def test_router_running(): print("******************************************\n") sleep(5) - # CLI(net) - failedRunning = "" + # Make sure that all daemons are running for i in range(1, 5): fatal_error = net['r%s' % i].checkRouterRunning() assert fatal_error == "", fatal_error @@ -259,6 +258,11 @@ def test_ospf6_converged(): print("\nwaiting 15s for routes to populate") sleep(15) + # Make sure that all daemons are still running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + def test_ospfv3_routingTable(): global fatal_error global net @@ -304,6 +308,11 @@ def test_ospfv3_routingTable(): assert failures == 0, "OSPFv3 (IPv6) Routing Table verification failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are still running + for i in range(1, 5): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) diff --git a/tests/topotests/ripng-topo1/test_ripng_topo1.py b/tests/topotests/ripng-topo1/test_ripng_topo1.py index ed72a01bd..e790808e3 100755 --- a/tests/topotests/ripng-topo1/test_ripng_topo1.py +++ b/tests/topotests/ripng-topo1/test_ripng_topo1.py @@ -173,6 +173,11 @@ def test_converge_protocols(): # Not really implemented yet - just sleep 60 secs for now sleep(60) + # Make sure that all daemons are running + for i in range(1, 4): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line #CLI(net) @@ -224,6 +229,11 @@ def test_ripng_status(): assert failures == 0, "IPv6 RIPng status failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 4): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -276,6 +286,11 @@ def test_ripng_routes(): assert failures == 0, "SHOW IPv6 RIPng failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 4): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) @@ -325,6 +340,11 @@ def test_zebra_ipv6_routingTable(): assert failures == 0, "Zebra IPv6 Routing Table verification failed for router r%s:\n%s" % (i, diff) + # Make sure that all daemons are running + for i in range(1, 4): + fatal_error = net['r%s' % i].checkRouterRunning() + assert fatal_error == "", fatal_error + # For debugging after starting FRR/Quagga daemons, uncomment the next line # CLI(net) -- 2.39.5