2 # SPDX-License-Identifier: ISC
6 # Library of helper functions for NetDEF Topology Tests
8 # Copyright (c) 2016 by
9 # Network Device Education Foundation, Inc. ("NetDEF")
27 from copy
import deepcopy
29 import lib
.topolog
as topolog
30 from lib
.topolog
import logger
32 if sys
.version_info
[0] > 2:
34 from collections
.abc
import Mapping
36 import ConfigParser
as configparser
37 from collections
import Mapping
39 from lib
import micronet
40 from lib
.micronet_compat
import Node
45 def get_logs_path(rundir
):
46 logspath
= topolog
.get_test_logdir()
47 return os
.path
.join(rundir
, logspath
)
50 def gdb_core(obj
, daemon
, corefiles
):
66 gdbcmds
= [["-ex", i
.strip()] for i
in gdbcmds
.strip().split("\n")]
67 gdbcmds
= [item
for sl
in gdbcmds
for item
in sl
]
69 daemon_path
= os
.path
.join(obj
.daemondir
, daemon
)
70 backtrace
= subprocess
.check_output(
71 ["gdb", daemon_path
, corefiles
[0], "--batch"] + gdbcmds
74 "\n%s: %s crashed. Core file found - Backtrace follows:\n" % (obj
.name
, daemon
)
76 sys
.stderr
.write("%s" % backtrace
)
80 class json_cmp_result(object):
81 "json_cmp result class for better assertion messages"
86 def add_error(self
, error
):
87 "Append error message to the result"
88 for line
in error
.splitlines():
89 self
.errors
.append(line
)
92 "Returns True if there were errors, otherwise False."
93 return len(self
.errors
) > 0
96 headline
= ["Generated JSON diff error report:", ""]
97 return headline
+ self
.errors
101 "Generated JSON diff error report:\n\n\n" + "\n".join(self
.errors
) + "\n\n"
105 def gen_json_diff_report(d1
, d2
, exact
=False, path
="> $", acc
=(0, "")):
107 Internal workhorse which compares two JSON data structures and generates an error report suited to be read by a human eye.
111 if isinstance(v
, (dict, list)):
112 return "\t" + "\t".join(
113 json
.dumps(v
, indent
=4, separators
=(",", ": ")).splitlines(True)
116 return "'{}'".format(v
)
119 if isinstance(v
, (list, tuple)):
121 elif isinstance(v
, dict):
123 elif isinstance(v
, (int, float)):
125 elif isinstance(v
, bool):
127 elif isinstance(v
, str):
132 def get_errors(other_acc
):
135 def get_errors_n(other_acc
):
138 def add_error(acc
, msg
, points
=1):
139 return (acc
[0] + points
, acc
[1] + "{}: {}\n".format(path
, msg
))
141 def merge_errors(acc
, other_acc
):
142 return (acc
[0] + other_acc
[0], acc
[1] + other_acc
[1])
145 return "{}[{}]".format(path
, idx
)
148 return "{}->{}".format(path
, key
)
150 def has_errors(other_acc
):
151 return other_acc
[0] > 0
154 not isinstance(d1
, (list, dict))
155 and not isinstance(d2
, (list, dict))
160 not isinstance(d1
, (list, dict))
161 and not isinstance(d2
, (list, dict))
166 "d1 has element with value '{}' but in d2 it has value '{}'".format(d1
, d2
),
170 and isinstance(d2
, list)
171 and ((len(d2
) > 0 and d2
[0] == "__ordered__") or exact
)
175 if len(d1
) != len(d2
):
178 "d1 has Array of length {} but in d2 it is of length {}".format(
183 for idx
, v1
, v2
in zip(range(0, len(d1
)), d1
, d2
):
185 acc
, gen_json_diff_report(v1
, v2
, exact
=exact
, path
=add_idx(idx
))
187 elif isinstance(d1
, list) and isinstance(d2
, list):
188 if len(d1
) < len(d2
):
191 "d1 has Array of length {} but in d2 it is of length {}".format(
196 for idx2
, v2
in zip(range(0, len(d2
)), d2
):
200 for idx1
, v1
in zip(range(0, len(d1
)), d1
):
201 tmp_v1
= deepcopy(v1
)
202 tmp_v2
= deepcopy(v2
)
203 tmp_diff
= gen_json_diff_report(tmp_v1
, tmp_v2
, path
=add_idx(idx1
))
204 if not has_errors(tmp_diff
):
208 elif not closest_diff
or get_errors_n(tmp_diff
) < get_errors_n(
211 closest_diff
= tmp_diff
213 if not found_match
and isinstance(v2
, (list, dict)):
214 sub_error
= "\n\n\t{}".format(
215 "\t".join(get_errors(closest_diff
).splitlines(True))
220 "d2 has the following element at index {} which is not present in d1: "
221 + "\n\n{}\n\n\tClosest match in d1 is at index {} with the following errors: {}"
222 ).format(idx2
, dump_json(v2
), closest_idx
, sub_error
),
224 if not found_match
and not isinstance(v2
, (list, dict)):
227 "d2 has the following element at index {} which is not present in d1: {}".format(
231 elif isinstance(d1
, dict) and isinstance(d2
, dict) and exact
:
232 invalid_keys_d1
= [k
for k
in d1
.keys() if k
not in d2
.keys()]
233 invalid_keys_d2
= [k
for k
in d2
.keys() if k
not in d1
.keys()]
234 for k
in invalid_keys_d1
:
235 acc
= add_error(acc
, "d1 has key '{}' which is not present in d2".format(k
))
236 for k
in invalid_keys_d2
:
237 acc
= add_error(acc
, "d2 has key '{}' which is not present in d1".format(k
))
238 valid_keys_intersection
= [k
for k
in d1
.keys() if k
in d2
.keys()]
239 for k
in valid_keys_intersection
:
241 acc
, gen_json_diff_report(d1
[k
], d2
[k
], exact
=exact
, path
=add_key(k
))
243 elif isinstance(d1
, dict) and isinstance(d2
, dict):
244 none_keys
= [k
for k
, v
in d2
.items() if v
== None]
245 none_keys_present
= [k
for k
in d1
.keys() if k
in none_keys
]
246 for k
in none_keys_present
:
248 acc
, "d1 has key '{}' which is not supposed to be present".format(k
)
250 keys
= [k
for k
, v
in d2
.items() if v
!= None]
251 invalid_keys_intersection
= [k
for k
in keys
if k
not in d1
.keys()]
252 for k
in invalid_keys_intersection
:
253 acc
= add_error(acc
, "d2 has key '{}' which is not present in d1".format(k
))
254 valid_keys_intersection
= [k
for k
in keys
if k
in d1
.keys()]
255 for k
in valid_keys_intersection
:
257 acc
, gen_json_diff_report(d1
[k
], d2
[k
], exact
=exact
, path
=add_key(k
))
262 "d1 has element of type '{}' but the corresponding element in d2 is of type '{}'".format(
263 json_type(d1
), json_type(d2
)
271 def json_cmp(d1
, d2
, exact
=False):
273 JSON compare function. Receives two parameters:
274 * `d1`: parsed JSON data structure
275 * `d2`: parsed JSON data structure
277 Returns 'None' when all JSON Object keys and all Array elements of d2 have a match
278 in d1, i.e., when d2 is a "subset" of d1 without honoring any order. Otherwise an
279 error report is generated and wrapped in a 'json_cmp_result()'. There are special
280 parameters and notations explained below which can be used to cover rather unusual
283 * when 'exact is set to 'True' then d1 and d2 are tested for equality (including
284 order within JSON Arrays)
285 * using 'null' (or 'None' in Python) as JSON Object value is checking for key
287 * using '*' as JSON Object value or Array value is checking for presence in d1
288 without checking the values
289 * using '__ordered__' as first element in a JSON Array in d2 will also check the
290 order when it is compared to an Array in d1
293 (errors_n
, errors
) = gen_json_diff_report(deepcopy(d1
), deepcopy(d2
), exact
=exact
)
296 result
= json_cmp_result()
297 result
.add_error(errors
)
303 def router_output_cmp(router
, cmd
, expected
):
305 Runs `cmd` in router and compares the output with `expected`.
308 normalize_text(router
.vtysh_cmd(cmd
)),
309 normalize_text(expected
),
310 title1
="Current output",
311 title2
="Expected output",
315 def router_json_cmp(router
, cmd
, data
, exact
=False):
317 Runs `cmd` that returns JSON data (normally the command ends with 'json')
318 and compare with `data` contents.
320 return json_cmp(router
.vtysh_cmd(cmd
, isjson
=True), data
, exact
)
323 def run_and_expect(func
, what
, count
=20, wait
=3):
325 Run `func` and compare the result with `what`. Do it for `count` times
326 waiting `wait` seconds between tries. By default it tries 20 times with
327 3 seconds delay between tries.
329 Returns (True, func-return) on success or
330 (False, func-return) on failure.
334 Helper functions to use with this function:
338 start_time
= time
.time()
339 func_name
= "<unknown>"
340 if func
.__class
__ == functools
.partial
:
341 func_name
= func
.func
.__name
__
343 func_name
= func
.__name
__
345 # Just a safety-check to avoid running topotests with very
346 # small wait/count arguments.
347 wait_time
= wait
* count
351 ), "Waiting time is too small (count={}, wait={}), adjust timer values".format(
356 "'{}' polling started (interval {} secs, maximum {} tries)".format(
357 func_name
, wait
, count
368 end_time
= time
.time()
370 "'{}' succeeded after {:.2f} seconds".format(
371 func_name
, end_time
- start_time
374 return (True, result
)
376 end_time
= time
.time()
378 "'{}' failed after {:.2f} seconds".format(func_name
, end_time
- start_time
)
380 return (False, result
)
383 def run_and_expect_type(func
, etype
, count
=20, wait
=3, avalue
=None):
385 Run `func` and compare the result with `etype`. Do it for `count` times
386 waiting `wait` seconds between tries. By default it tries 20 times with
387 3 seconds delay between tries.
389 This function is used when you want to test the return type and,
390 optionally, the return value.
392 Returns (True, func-return) on success or
393 (False, func-return) on failure.
395 start_time
= time
.time()
396 func_name
= "<unknown>"
397 if func
.__class
__ == functools
.partial
:
398 func_name
= func
.func
.__name
__
400 func_name
= func
.__name
__
402 # Just a safety-check to avoid running topotests with very
403 # small wait/count arguments.
404 wait_time
= wait
* count
408 ), "Waiting time is too small (count={}, wait={}), adjust timer values".format(
413 "'{}' polling started (interval {} secs, maximum wait {} secs)".format(
414 func_name
, wait
, int(wait
* count
)
420 if not isinstance(result
, etype
):
422 "Expected result type '{}' got '{}' instead".format(etype
, type(result
))
428 if etype
!= type(None) and avalue
!= None and result
!= avalue
:
429 logger
.debug("Expected value '{}' got '{}' instead".format(avalue
, result
))
434 end_time
= time
.time()
436 "'{}' succeeded after {:.2f} seconds".format(
437 func_name
, end_time
- start_time
440 return (True, result
)
442 end_time
= time
.time()
444 "'{}' failed after {:.2f} seconds".format(func_name
, end_time
- start_time
)
446 return (False, result
)
449 def router_json_cmp_retry(router
, cmd
, data
, exact
=False, retry_timeout
=10.0):
451 Runs `cmd` that returns JSON data (normally the command ends with 'json')
452 and compare with `data` contents. Retry by default for 10 seconds
456 return router_json_cmp(router
, cmd
, data
, exact
)
458 ok
, _
= run_and_expect(test_func
, None, int(retry_timeout
), 1)
463 "Converting Integer to DPID"
467 dpid
= "0" * (16 - len(dpid
)) + dpid
471 "Unable to derive default datapath ID - "
472 "please either specify a dpid or use a "
473 "canonical switch name such as s23."
478 "Check whether pid exists in the current process table."
483 os
.waitpid(pid
, os
.WNOHANG
)
488 except OSError as err
:
489 if err
.errno
== errno
.ESRCH
:
490 # ESRCH == No such process
492 elif err
.errno
== errno
.EPERM
:
493 # EPERM clearly means there's a process to deny access to
496 # According to "man 2 kill" possible error values are
497 # (EINVAL, EPERM, ESRCH)
503 def get_textdiff(text1
, text2
, title1
="", title2
="", **opts
):
504 "Returns empty string if same or formatted diff"
507 difflib
.unified_diff(text1
, text2
, fromfile
=title1
, tofile
=title2
, **opts
)
509 # Clean up line endings
510 diff
= os
.linesep
.join([s
for s
in diff
.splitlines() if s
])
514 def difflines(text1
, text2
, title1
="", title2
="", **opts
):
515 "Wrapper for get_textdiff to avoid string transformations."
516 text1
= ("\n".join(text1
.rstrip().splitlines()) + "\n").splitlines(1)
517 text2
= ("\n".join(text2
.rstrip().splitlines()) + "\n").splitlines(1)
518 return get_textdiff(text1
, text2
, title1
, title2
, **opts
)
521 def get_file(content
):
523 Generates a temporary file in '/tmp' with `content` and returns the file name.
525 if isinstance(content
, list) or isinstance(content
, tuple):
526 content
= "\n".join(content
)
527 fde
= tempfile
.NamedTemporaryFile(mode
="w", delete
=False)
534 def normalize_text(text
):
536 Strips formating spaces/tabs, carriage returns and trailing whitespace.
538 text
= re
.sub(r
"[ \t]+", " ", text
)
539 text
= re
.sub(r
"\r", "", text
)
541 # Remove whitespace in the middle of text.
542 text
= re
.sub(r
"[ \t]+\n", "\n", text
)
543 # Remove whitespace at the end of the text.
551 Parses unix name output to check if running on GNU/Linux.
553 Returns True if running on Linux, returns False otherwise.
556 if os
.uname()[0] == "Linux":
561 def iproute2_is_vrf_capable():
563 Checks if the iproute2 version installed on the system is capable of
564 handling VRFs by interpreting the output of the 'ip' utility found in PATH.
566 Returns True if capability can be detected, returns False otherwise.
571 subp
= subprocess
.Popen(
572 ["ip", "route", "show", "vrf"],
573 stdout
=subprocess
.PIPE
,
574 stderr
=subprocess
.PIPE
,
575 stdin
=subprocess
.PIPE
,
577 iproute2_err
= subp
.communicate()[1].splitlines()[0].split()[0]
579 if iproute2_err
!= "Error:":
585 def iproute2_is_fdb_get_capable():
587 Checks if the iproute2 version installed on the system is capable of
588 handling `bridge fdb get` commands to query neigh table resolution.
590 Returns True if capability can be detected, returns False otherwise.
595 subp
= subprocess
.Popen(
596 ["bridge", "fdb", "get", "help"],
597 stdout
=subprocess
.PIPE
,
598 stderr
=subprocess
.PIPE
,
599 stdin
=subprocess
.PIPE
,
601 iproute2_out
= subp
.communicate()[1].splitlines()[0].split()[0]
603 if "Usage" in str(iproute2_out
):
609 def module_present_linux(module
, load
):
611 Returns whether `module` is present.
613 If `load` is true, it will try to load it via modprobe.
615 with
open("/proc/modules", "r") as modules_file
:
616 if module
.replace("-", "_") in modules_file
.read():
618 cmd
= "/sbin/modprobe {}{}".format("" if load
else "-n ", module
)
619 if os
.system(cmd
) != 0:
625 def module_present_freebsd(module
, load
):
629 def module_present(module
, load
=True):
630 if sys
.platform
.startswith("linux"):
631 return module_present_linux(module
, load
)
632 elif sys
.platform
.startswith("freebsd"):
633 return module_present_freebsd(module
, load
)
636 def version_cmp(v1
, v2
):
638 Compare two version strings and returns:
640 * `-1`: if `v1` is less than `v2`
641 * `0`: if `v1` is equal to `v2`
642 * `1`: if `v1` is greater than `v2`
644 Raises `ValueError` if versions are not well formated.
646 vregex
= r
"(?P<whole>\d+(\.(\d+))*)"
647 v1m
= re
.match(vregex
, v1
)
648 v2m
= re
.match(vregex
, v2
)
649 if v1m
is None or v2m
is None:
650 raise ValueError("got a invalid version string")
653 v1g
= v1m
.group("whole").split(".")
654 v2g
= v2m
.group("whole").split(".")
656 # Get the longest version string
661 # Reverse list because we are going to pop the tail
664 for _
in range(vnum
):
692 def interface_set_status(node
, ifacename
, ifaceaction
=False, vrf_name
=None):
694 str_ifaceaction
= "no shutdown"
696 str_ifaceaction
= "shutdown"
698 cmd
= 'vtysh -c "configure terminal" -c "interface {0}" -c "{1}"'.format(
699 ifacename
, str_ifaceaction
703 'vtysh -c "configure terminal" -c "interface {0} vrf {1}" -c "{2}"'.format(
704 ifacename
, vrf_name
, str_ifaceaction
710 def ip4_route_zebra(node
, vrf_name
=None):
712 Gets an output of 'show ip route' command. It can be used
713 with comparing the output to a reference
716 tmp
= node
.vtysh_cmd("show ip route")
718 tmp
= node
.vtysh_cmd("show ip route vrf {0}".format(vrf_name
))
719 output
= re
.sub(r
" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", tmp
)
721 lines
= output
.splitlines()
723 while lines
and (not lines
[0].strip() or not header_found
):
724 if "o - offload failure" in lines
[0]:
727 return "\n".join(lines
)
730 def ip6_route_zebra(node
, vrf_name
=None):
732 Retrieves the output of 'show ipv6 route [vrf vrf_name]', then
733 canonicalizes it by eliding link-locals.
737 tmp
= node
.vtysh_cmd("show ipv6 route")
739 tmp
= node
.vtysh_cmd("show ipv6 route vrf {0}".format(vrf_name
))
742 output
= re
.sub(r
" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", tmp
)
744 # Mask out the link-local addresses
745 output
= re
.sub(r
"fe80::[^ ]+,", "fe80::XXXX:XXXX:XXXX:XXXX,", output
)
747 lines
= output
.splitlines()
749 while lines
and (not lines
[0].strip() or not header_found
):
750 if "o - offload failure" in lines
[0]:
754 return "\n".join(lines
)
757 def proto_name_to_number(protocol
):
773 ) # default return same as input
778 Gets a structured return of the command 'ip route'. It can be used in
779 conjunction with json_cmp() to provide accurate assert explanations.
794 output
= normalize_text(node
.run("ip route")).splitlines()
797 columns
= line
.split(" ")
798 route
= result
[columns
[0]] = {}
800 for column
in columns
:
802 route
["dev"] = column
804 route
["via"] = column
806 # translate protocol names back to numbers
807 route
["proto"] = proto_name_to_number(column
)
809 route
["metric"] = column
811 route
["scope"] = column
817 def ip4_vrf_route(node
):
819 Gets a structured return of the command 'ip route show vrf {0}-cust1'.
820 It can be used in conjunction with json_cmp() to provide accurate assert explanations.
835 output
= normalize_text(
836 node
.run("ip route show vrf {0}-cust1".format(node
.name
))
841 columns
= line
.split(" ")
842 route
= result
[columns
[0]] = {}
844 for column
in columns
:
846 route
["dev"] = column
848 route
["via"] = column
850 # translate protocol names back to numbers
851 route
["proto"] = proto_name_to_number(column
)
853 route
["metric"] = column
855 route
["scope"] = column
863 Gets a structured return of the command 'ip -6 route'. It can be used in
864 conjunction with json_cmp() to provide accurate assert explanations.
878 output
= normalize_text(node
.run("ip -6 route")).splitlines()
881 columns
= line
.split(" ")
882 route
= result
[columns
[0]] = {}
884 for column
in columns
:
886 route
["dev"] = column
888 route
["via"] = column
890 # translate protocol names back to numbers
891 route
["proto"] = proto_name_to_number(column
)
893 route
["metric"] = column
895 route
["pref"] = column
901 def ip6_vrf_route(node
):
903 Gets a structured return of the command 'ip -6 route show vrf {0}-cust1'.
904 It can be used in conjunction with json_cmp() to provide accurate assert explanations.
918 output
= normalize_text(
919 node
.run("ip -6 route show vrf {0}-cust1".format(node
.name
))
923 columns
= line
.split(" ")
924 route
= result
[columns
[0]] = {}
926 for column
in columns
:
928 route
["dev"] = column
930 route
["via"] = column
932 # translate protocol names back to numbers
933 route
["proto"] = proto_name_to_number(column
)
935 route
["metric"] = column
937 route
["pref"] = column
945 Gets a structured return of the command 'ip rule'. It can be used in
946 conjunction with json_cmp() to provide accurate assert explanations.
962 "from": "1.2.0.0/16",
967 output
= normalize_text(node
.run("ip rule")).splitlines()
970 columns
= line
.split(" ")
973 # remove last character, since it is ':'
974 pref
= columns
[0][:-1]
977 for column
in columns
:
979 route
["from"] = column
983 route
["proto"] = column
985 route
["iif"] = column
987 route
["fwmark"] = column
994 def sleep(amount
, reason
=None):
996 Sleep wrapper that registers in the log the amount of sleep
999 logger
.info("Sleeping for {} seconds".format(amount
))
1001 logger
.info(reason
+ " ({} seconds)".format(amount
))
1006 def checkAddressSanitizerError(output
, router
, component
, logdir
=""):
1007 "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise"
1009 def processAddressSanitizerError(asanErrorRe
, output
, router
, component
):
1011 "%s: %s triggered an exception by AddressSanitizer\n" % (router
, component
)
1013 # Sanitizer Error found in log
1014 pidMark
= asanErrorRe
.group(1)
1015 addressSanitizerLog
= re
.search(
1016 "%s(.*)%s" % (pidMark
, pidMark
), output
, re
.DOTALL
1018 if addressSanitizerLog
:
1019 # Find Calling Test. Could be multiple steps back
1020 testframe
= sys
._current
_frames
().values()[0]
1023 test
= os
.path
.splitext(
1024 os
.path
.basename(testframe
.f_globals
["__file__"])
1026 if (test
!= "topotest") and (test
!= "topogen"):
1027 # Found the calling test
1028 callingTest
= os
.path
.basename(testframe
.f_globals
["__file__"])
1031 testframe
= testframe
.f_back
1033 # somehow couldn't find the test script.
1034 callingTest
= "unknownTest"
1036 # Now finding Calling Procedure
1039 callingProc
= sys
._getframe
(level
).f_code
.co_name
1041 (callingProc
!= "processAddressSanitizerError")
1042 and (callingProc
!= "checkAddressSanitizerError")
1043 and (callingProc
!= "checkRouterCores")
1044 and (callingProc
!= "stopRouter")
1045 and (callingProc
!= "stop")
1046 and (callingProc
!= "stop_topology")
1047 and (callingProc
!= "checkRouterRunning")
1048 and (callingProc
!= "check_router_running")
1049 and (callingProc
!= "routers_have_failure")
1051 # Found the calling test
1055 # something wrong - couldn't found the calling test function
1056 callingProc
= "unknownProc"
1057 with
open("/tmp/AddressSanitzer.txt", "a") as addrSanFile
:
1059 "AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n"
1060 % (callingTest
, callingProc
, router
)
1063 "\n".join(addressSanitizerLog
.group(1).splitlines()) + "\n"
1065 addrSanFile
.write("## Error: %s\n\n" % asanErrorRe
.group(2))
1067 "### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n"
1068 % (callingTest
, callingProc
, router
)
1072 + "\n ".join(addressSanitizerLog
.group(1).splitlines())
1075 addrSanFile
.write("\n---------------\n")
1078 addressSanitizerError
= re
.search(
1079 r
"(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", output
1081 if addressSanitizerError
:
1082 processAddressSanitizerError(addressSanitizerError
, output
, router
, component
)
1085 # No Address Sanitizer Error in Output. Now check for AddressSanitizer daemon file
1087 filepattern
= logdir
+ "/" + router
+ "/" + component
+ ".asan.*"
1089 "Log check for %s on %s, pattern %s\n" % (component
, router
, filepattern
)
1091 for file in glob
.glob(filepattern
):
1092 with
open(file, "r") as asanErrorFile
:
1093 asanError
= asanErrorFile
.read()
1094 addressSanitizerError
= re
.search(
1095 r
"(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", asanError
1097 if addressSanitizerError
:
1098 processAddressSanitizerError(
1099 addressSanitizerError
, asanError
, router
, component
1105 def _sysctl_atleast(commander
, variable
, min_value
):
1106 if isinstance(min_value
, tuple):
1107 min_value
= list(min_value
)
1108 is_list
= isinstance(min_value
, list)
1110 sval
= commander
.cmd_raises("sysctl -n " + variable
).strip()
1112 cur_val
= [int(x
) for x
in sval
.split()]
1118 for i
, v
in enumerate(cur_val
):
1119 if v
< min_value
[i
]:
1124 if cur_val
< min_value
:
1128 valstr
= " ".join([str(x
) for x
in min_value
])
1130 valstr
= str(min_value
)
1131 logger
.info("Increasing sysctl %s from %s to %s", variable
, cur_val
, valstr
)
1132 commander
.cmd_raises('sysctl -w {}="{}"\n'.format(variable
, valstr
))
1135 def _sysctl_assure(commander
, variable
, value
):
1136 if isinstance(value
, tuple):
1138 is_list
= isinstance(value
, list)
1140 sval
= commander
.cmd_raises("sysctl -n " + variable
).strip()
1142 cur_val
= [int(x
) for x
in sval
.split()]
1148 for i
, v
in enumerate(cur_val
):
1154 if cur_val
!= str(value
):
1159 valstr
= " ".join([str(x
) for x
in value
])
1162 logger
.info("Changing sysctl %s from %s to %s", variable
, cur_val
, valstr
)
1163 commander
.cmd_raises('sysctl -w {}="{}"\n'.format(variable
, valstr
))
1166 def sysctl_atleast(commander
, variable
, min_value
, raises
=False):
1168 if commander
is None:
1169 commander
= micronet
.Commander("topotest")
1170 return _sysctl_atleast(commander
, variable
, min_value
)
1171 except subprocess
.CalledProcessError
as error
:
1173 "%s: Failed to assure sysctl min value %s = %s",
1182 def sysctl_assure(commander
, variable
, value
, raises
=False):
1184 if commander
is None:
1185 commander
= micronet
.Commander("topotest")
1186 return _sysctl_assure(commander
, variable
, value
)
1187 except subprocess
.CalledProcessError
as error
:
1189 "%s: Failed to assure sysctl value %s = %s",
1199 def rlimit_atleast(rname
, min_value
, raises
=False):
1201 cval
= resource
.getrlimit(rname
)
1203 if soft
< min_value
:
1204 nval
= (min_value
, hard
if min_value
< hard
else min_value
)
1205 logger
.info("Increasing rlimit %s from %s to %s", rname
, cval
, nval
)
1206 resource
.setrlimit(rname
, nval
)
1207 except subprocess
.CalledProcessError
as error
:
1209 "Failed to assure rlimit [%s] = %s", rname
, min_value
, exc_info
=True
1215 def fix_netns_limits(ns
):
1217 # Maximum read and write socket buffer sizes
1218 sysctl_atleast(ns
, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2 ** 20])
1219 sysctl_atleast(ns
, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2 ** 20])
1221 sysctl_assure(ns
, "net.ipv4.conf.all.rp_filter", 0)
1222 sysctl_assure(ns
, "net.ipv4.conf.default.rp_filter", 0)
1223 sysctl_assure(ns
, "net.ipv4.conf.lo.rp_filter", 0)
1225 sysctl_assure(ns
, "net.ipv4.conf.all.forwarding", 1)
1226 sysctl_assure(ns
, "net.ipv4.conf.default.forwarding", 1)
1228 # XXX if things fail look here as this wasn't done previously
1229 sysctl_assure(ns
, "net.ipv6.conf.all.forwarding", 1)
1230 sysctl_assure(ns
, "net.ipv6.conf.default.forwarding", 1)
1233 sysctl_assure(ns
, "net.ipv4.conf.default.arp_announce", 2)
1234 sysctl_assure(ns
, "net.ipv4.conf.default.arp_notify", 1)
1235 # Setting this to 1 breaks topotests that rely on lo addresses being proxy arp'd for
1236 sysctl_assure(ns
, "net.ipv4.conf.default.arp_ignore", 0)
1237 sysctl_assure(ns
, "net.ipv4.conf.all.arp_announce", 2)
1238 sysctl_assure(ns
, "net.ipv4.conf.all.arp_notify", 1)
1239 # Setting this to 1 breaks topotests that rely on lo addresses being proxy arp'd for
1240 sysctl_assure(ns
, "net.ipv4.conf.all.arp_ignore", 0)
1242 sysctl_assure(ns
, "net.ipv4.icmp_errors_use_inbound_ifaddr", 1)
1244 # Keep ipv6 permanent addresses on an admin down
1245 sysctl_assure(ns
, "net.ipv6.conf.all.keep_addr_on_down", 1)
1246 if version_cmp(platform
.release(), "4.20") >= 0:
1247 sysctl_assure(ns
, "net.ipv6.route.skip_notify_on_dev_down", 1)
1249 sysctl_assure(ns
, "net.ipv4.conf.all.ignore_routes_with_linkdown", 1)
1250 sysctl_assure(ns
, "net.ipv6.conf.all.ignore_routes_with_linkdown", 1)
1253 sysctl_atleast(ns
, "net.ipv4.igmp_max_memberships", 1000)
1255 # Use neigh information on selection of nexthop for multipath hops
1256 sysctl_assure(ns
, "net.ipv4.fib_multipath_use_neigh", 1)
1259 def fix_host_limits():
1260 """Increase system limits."""
1262 rlimit_atleast(resource
.RLIMIT_NPROC
, 8 * 1024)
1263 rlimit_atleast(resource
.RLIMIT_NOFILE
, 16 * 1024)
1264 sysctl_atleast(None, "fs.file-max", 16 * 1024)
1265 sysctl_atleast(None, "kernel.pty.max", 16 * 1024)
1268 # Original on ubuntu 17.x, but apport won't save as in namespace
1269 # |/usr/share/apport/apport %p %s %c %d %P
1270 sysctl_assure(None, "kernel.core_pattern", "%e_core-sig_%s-pid_%p.dmp")
1271 sysctl_assure(None, "kernel.core_uses_pid", 1)
1272 sysctl_assure(None, "fs.suid_dumpable", 1)
1274 # Maximum connection backlog
1275 sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024)
1277 # Maximum read and write socket buffer sizes
1278 sysctl_atleast(None, "net.core.rmem_max", 16 * 2 ** 20)
1279 sysctl_atleast(None, "net.core.wmem_max", 16 * 2 ** 20)
1281 # Garbage Collection Settings for ARP and Neighbors
1282 sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024)
1283 sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh3", 8 * 1024)
1284 sysctl_atleast(None, "net.ipv6.neigh.default.gc_thresh2", 4 * 1024)
1285 sysctl_atleast(None, "net.ipv6.neigh.default.gc_thresh3", 8 * 1024)
1286 # Hold entries for 10 minutes
1287 sysctl_assure(None, "net.ipv4.neigh.default.base_reachable_time_ms", 10 * 60 * 1000)
1288 sysctl_assure(None, "net.ipv6.neigh.default.base_reachable_time_ms", 10 * 60 * 1000)
1291 sysctl_assure(None, "net.ipv4.neigh.default.mcast_solicit", 10)
1294 sysctl_atleast(None, "net.ipv6.mld_max_msf", 512)
1296 # Increase routing table size to 128K
1297 sysctl_atleast(None, "net.ipv4.route.max_size", 128 * 1024)
1298 sysctl_atleast(None, "net.ipv6.route.max_size", 128 * 1024)
1301 def setup_node_tmpdir(logdir
, name
):
1302 # Cleanup old log, valgrind, and core files.
1303 subprocess
.check_call(
1304 "rm -rf {0}/{1}.valgrind.* {1}.*.asan {0}/{1}/".format(logdir
, name
), shell
=True
1307 # Setup the per node directory.
1308 nodelogdir
= "{}/{}".format(logdir
, name
)
1309 subprocess
.check_call(
1310 "mkdir -p {0} && chmod 1777 {0}".format(nodelogdir
), shell
=True
1312 logfile
= "{0}/{1}.log".format(logdir
, name
)
1317 "A Node with IPv4/IPv6 forwarding enabled"
1319 def __init__(self
, name
, **params
):
1321 # Backward compatibility:
1322 # Load configuration defaults like topogen.
1323 self
.config_defaults
= configparser
.ConfigParser(
1325 "verbosity": "info",
1326 "frrdir": "/usr/lib/frr",
1327 "routertype": "frr",
1332 self
.config_defaults
.read(
1333 os
.path
.join(os
.path
.dirname(os
.path
.realpath(__file__
)), "../pytest.ini")
1336 # If this topology is using old API and doesn't have logdir
1337 # specified, then attempt to generate an unique logdir.
1338 self
.logdir
= params
.get("logdir")
1339 if self
.logdir
is None:
1340 self
.logdir
= get_logs_path(g_extra_config
["rundir"])
1342 if not params
.get("logger"):
1343 # If logger is present topogen has already set this up
1344 logfile
= setup_node_tmpdir(self
.logdir
, name
)
1345 l
= topolog
.get_logger(name
, log_level
="debug", target
=logfile
)
1346 params
["logger"] = l
1348 super(Router
, self
).__init
__(name
, **params
)
1350 self
.daemondir
= None
1351 self
.hasmpls
= False
1352 self
.routertype
= "frr"
1353 self
.unified_config
= None
1375 self
.daemons_options
= {"zebra": ""}
1376 self
.reportCores
= True
1379 self
.ns_cmd
= "sudo nsenter -a -t {} ".format(self
.pid
)
1381 # Allow escaping from running inside docker
1382 cgroup
= open("/proc/1/cgroup").read()
1383 m
= re
.search("[0-9]+:cpuset:/docker/([a-f0-9]+)", cgroup
)
1385 self
.ns_cmd
= "docker exec -it {} ".format(m
.group(1)) + self
.ns_cmd
1389 logger
.debug("CMD to enter {}: {}".format(self
.name
, self
.ns_cmd
))
1391 def _config_frr(self
, **params
):
1392 "Configure FRR binaries"
1393 self
.daemondir
= params
.get("frrdir")
1394 if self
.daemondir
is None:
1395 self
.daemondir
= self
.config_defaults
.get("topogen", "frrdir")
1397 zebra_path
= os
.path
.join(self
.daemondir
, "zebra")
1398 if not os
.path
.isfile(zebra_path
):
1399 raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path
))
1401 # pylint: disable=W0221
1402 # Some params are only meaningful for the parent class.
1403 def config(self
, **params
):
1404 super(Router
, self
).config(**params
)
1406 # User did not specify the daemons directory, try to autodetect it.
1407 self
.daemondir
= params
.get("daemondir")
1408 if self
.daemondir
is None:
1409 self
.routertype
= params
.get(
1410 "routertype", self
.config_defaults
.get("topogen", "routertype")
1412 self
._config
_frr
(**params
)
1414 # Test the provided path
1415 zpath
= os
.path
.join(self
.daemondir
, "zebra")
1416 if not os
.path
.isfile(zpath
):
1417 raise Exception("No zebra binary found in {}".format(zpath
))
1418 # Allow user to specify routertype when the path was specified.
1419 if params
.get("routertype") is not None:
1420 self
.routertype
= params
.get("routertype")
1422 # Set ownership of config files
1423 self
.cmd("chown {0}:{0}vty /etc/{0}".format(self
.routertype
))
1425 def terminate(self
):
1426 # Stop running FRR daemons
1428 super(Router
, self
).terminate()
1429 os
.system("chmod -R go+rw " + self
.logdir
)
1431 # Return count of running daemons
1432 def listDaemons(self
):
1434 rc
, stdout
, _
= self
.cmd_status(
1435 "ls -1 /var/run/%s/*.pid" % self
.routertype
, warn
=False
1439 for d
in stdout
.strip().split("\n"):
1442 pid
= int(self
.cmd_raises("cat %s" % pidfile
, warn
=False).strip())
1443 name
= os
.path
.basename(pidfile
[:-4])
1445 # probably not compatible with bsd.
1446 rc
, _
, _
= self
.cmd_status("test -d /proc/{}".format(pid
), warn
=False)
1449 "%s: %s exited leaving pidfile %s (%s)",
1455 self
.cmd("rm -- " + pidfile
)
1457 ret
.append((name
, pid
))
1458 except (subprocess
.CalledProcessError
, ValueError):
1462 def stopRouter(self
, assertOnError
=True, minErrorVersion
="5.1"):
1463 # Stop Running FRR Daemons
1464 running
= self
.listDaemons()
1468 logger
.info("%s: stopping %s", self
.name
, ", ".join([x
[0] for x
in running
]))
1469 for name
, pid
in running
:
1470 logger
.info("{}: sending SIGTERM to {}".format(self
.name
, name
))
1472 os
.kill(pid
, signal
.SIGTERM
)
1473 except OSError as err
:
1475 "%s: could not kill %s (%s): %s", self
.name
, name
, pid
, str(err
)
1478 running
= self
.listDaemons()
1480 for _
in range(0, 30):
1483 "{}: waiting for daemons stopping: {}".format(
1484 self
.name
, ", ".join([x
[0] for x
in running
])
1487 running
= self
.listDaemons()
1495 "%s: sending SIGBUS to: %s", self
.name
, ", ".join([x
[0] for x
in running
])
1497 for name
, pid
in running
:
1498 pidfile
= "/var/run/{}/{}.pid".format(self
.routertype
, name
)
1499 logger
.info("%s: killing %s", self
.name
, name
)
1500 self
.cmd("kill -SIGBUS %d" % pid
)
1501 self
.cmd("rm -- " + pidfile
)
1504 0.5, "%s: waiting for daemons to exit/core after initial SIGBUS" % self
.name
1507 errors
= self
.checkRouterCores(reportOnce
=True)
1508 if self
.checkRouterVersion("<", minErrorVersion
):
1509 # ignore errors in old versions
1511 if assertOnError
and (errors
is not None) and len(errors
) > 0:
1512 assert "Errors found - details follow:" == 0, errors
1515 def removeIPs(self
):
1516 for interface
in self
.intfNames():
1518 self
.intf_ip_cmd(interface
, "ip address flush " + interface
)
1519 except Exception as ex
:
1520 logger
.error("%s can't remove IPs %s", self
, str(ex
))
1522 # assert False, "can't remove IPs %s" % str(ex)
1524 def checkCapability(self
, daemon
, param
):
1525 if param
is not None:
1526 daemon_path
= os
.path
.join(self
.daemondir
, daemon
)
1527 daemon_search_option
= param
.replace("-", "")
1529 "{0} -h | grep {1}".format(daemon_path
, daemon_search_option
)
1531 if daemon_search_option
not in output
:
1535 def loadConf(self
, daemon
, source
=None, param
=None):
1536 """Enabled and set config for a daemon.
1538 Arranges for loading of daemon configuration from the specified source. Possible
1539 `source` values are `None` for an empty config file, a path name which is used
1540 directly, or a file name with no path components which is first looked for
1541 directly and then looked for under a sub-directory named after router.
1544 # Unfortunately this API allowsfor source to not exist for any and all routers.
1546 head
, tail
= os
.path
.split(source
)
1547 if not head
and not self
.path_exists(tail
):
1548 script_dir
= os
.environ
["PYTEST_TOPOTEST_SCRIPTDIR"]
1549 router_relative
= os
.path
.join(script_dir
, self
.name
, tail
)
1550 if self
.path_exists(router_relative
):
1551 source
= router_relative
1553 "using router relative configuration: {}".format(source
)
1556 # print "Daemons before:", self.daemons
1557 if daemon
in self
.daemons
.keys() or daemon
== "frr":
1559 self
.unified_config
= 1
1561 self
.daemons
[daemon
] = 1
1562 if param
is not None:
1563 self
.daemons_options
[daemon
] = param
1564 conf_file
= "/etc/{}/{}.conf".format(self
.routertype
, daemon
)
1565 if source
is None or not os
.path
.exists(source
):
1566 if daemon
== "frr" or not self
.unified_config
:
1567 self
.cmd_raises("rm -f " + conf_file
)
1568 self
.cmd_raises("touch " + conf_file
)
1570 self
.cmd_raises("cp {} {}".format(source
, conf_file
))
1572 if not self
.unified_config
or daemon
== "frr":
1573 self
.cmd_raises("chown {0}:{0} {1}".format(self
.routertype
, conf_file
))
1574 self
.cmd_raises("chmod 664 {}".format(conf_file
))
1576 if (daemon
== "snmpd") and (self
.routertype
== "frr"):
1577 # /etc/snmp is private mount now
1578 self
.cmd('echo "agentXSocket /etc/frr/agentx" >> /etc/snmp/frr.conf')
1579 self
.cmd('echo "mibs +ALL" > /etc/snmp/snmp.conf')
1581 if (daemon
== "zebra") and (self
.daemons
["staticd"] == 0):
1582 # Add staticd with zebra - if it exists
1584 staticd_path
= os
.path
.join(self
.daemondir
, "staticd")
1588 if os
.path
.isfile(staticd_path
):
1589 self
.daemons
["staticd"] = 1
1590 self
.daemons_options
["staticd"] = ""
1591 # Auto-Started staticd has no config, so it will read from zebra config
1593 logger
.info("No daemon {} known".format(daemon
))
1594 # print "Daemons after:", self.daemons
1596 def runInWindow(self
, cmd
, title
=None):
1597 return self
.run_in_window(cmd
, title
)
1599 def startRouter(self
, tgen
=None):
1600 if self
.unified_config
:
1602 'echo "service integrated-vtysh-config" >> /etc/%s/vtysh.conf'
1606 # Disable integrated-vtysh-config
1608 'echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf'
1613 "chown %s:%svty /etc/%s/vtysh.conf"
1614 % (self
.routertype
, self
.routertype
, self
.routertype
)
1616 # TODO remove the following lines after all tests are migrated to Topogen.
1617 # Try to find relevant old logfiles in /tmp and delete them
1618 map(os
.remove
, glob
.glob("{}/{}/*.log".format(self
.logdir
, self
.name
)))
1619 # Remove old core files
1620 map(os
.remove
, glob
.glob("{}/{}/*.dmp".format(self
.logdir
, self
.name
)))
1621 # Remove IP addresses from OS first - we have them in zebra.conf
1623 # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher
1624 # No error - but return message and skip all the tests
1625 if self
.daemons
["ldpd"] == 1:
1626 ldpd_path
= os
.path
.join(self
.daemondir
, "ldpd")
1627 if not os
.path
.isfile(ldpd_path
):
1628 logger
.info("LDP Test, but no ldpd compiled or installed")
1629 return "LDP Test, but no ldpd compiled or installed"
1631 if version_cmp(platform
.release(), "4.5") < 0:
1632 logger
.info("LDP Test need Linux Kernel 4.5 minimum")
1633 return "LDP Test need Linux Kernel 4.5 minimum"
1634 # Check if have mpls
1636 self
.hasmpls
= tgen
.hasmpls
1637 if self
.hasmpls
!= True:
1639 "LDP/MPLS Tests will be skipped, platform missing module(s)"
1642 # Test for MPLS Kernel modules available
1643 self
.hasmpls
= False
1644 if not module_present("mpls-router"):
1646 "MPLS tests will not run (missing mpls-router kernel module)"
1648 elif not module_present("mpls-iptunnel"):
1650 "MPLS tests will not run (missing mpls-iptunnel kernel module)"
1654 if self
.hasmpls
!= True:
1655 return "LDP/MPLS Tests need mpls kernel modules"
1657 # Really want to use sysctl_atleast here, but only when MPLS is actually being
1659 self
.cmd("echo 100000 > /proc/sys/net/mpls/platform_labels")
1661 shell_routers
= g_extra_config
["shell"]
1662 if "all" in shell_routers
or self
.name
in shell_routers
:
1663 self
.run_in_window(os
.getenv("SHELL", "bash"), title
="sh-%s" % self
.name
)
1665 if self
.daemons
["eigrpd"] == 1:
1666 eigrpd_path
= os
.path
.join(self
.daemondir
, "eigrpd")
1667 if not os
.path
.isfile(eigrpd_path
):
1668 logger
.info("EIGRP Test, but no eigrpd compiled or installed")
1669 return "EIGRP Test, but no eigrpd compiled or installed"
1671 if self
.daemons
["bfdd"] == 1:
1672 bfdd_path
= os
.path
.join(self
.daemondir
, "bfdd")
1673 if not os
.path
.isfile(bfdd_path
):
1674 logger
.info("BFD Test, but no bfdd compiled or installed")
1675 return "BFD Test, but no bfdd compiled or installed"
1677 status
= self
.startRouterDaemons(tgen
=tgen
)
1679 vtysh_routers
= g_extra_config
["vtysh"]
1680 if "all" in vtysh_routers
or self
.name
in vtysh_routers
:
1681 self
.run_in_window("vtysh", title
="vt-%s" % self
.name
)
1683 if self
.unified_config
:
1684 self
.cmd("vtysh -f /etc/frr/frr.conf")
1688 def getStdErr(self
, daemon
):
1689 return self
.getLog("err", daemon
)
1691 def getStdOut(self
, daemon
):
1692 return self
.getLog("out", daemon
)
1694 def getLog(self
, log
, daemon
):
1695 return self
.cmd("cat {}/{}/{}.{}".format(self
.logdir
, self
.name
, daemon
, log
))
1697 def startRouterDaemons(self
, daemons
=None, tgen
=None):
1698 "Starts FRR daemons for this router."
1700 asan_abort
= g_extra_config
["asan_abort"]
1701 gdb_breakpoints
= g_extra_config
["gdb_breakpoints"]
1702 gdb_daemons
= g_extra_config
["gdb_daemons"]
1703 gdb_routers
= g_extra_config
["gdb_routers"]
1704 valgrind_extra
= g_extra_config
["valgrind_extra"]
1705 valgrind_memleaks
= g_extra_config
["valgrind_memleaks"]
1706 strace_daemons
= g_extra_config
["strace_daemons"]
1708 # Get global bundle data
1709 if not self
.path_exists("/etc/frr/support_bundle_commands.conf"):
1710 # Copy global value if was covered by namespace mount
1712 if os
.path
.exists("/etc/frr/support_bundle_commands.conf"):
1713 with
open("/etc/frr/support_bundle_commands.conf", "r") as rf
:
1714 bundle_data
= rf
.read()
1716 "cat > /etc/frr/support_bundle_commands.conf",
1720 # Starts actual daemons without init (ie restart)
1721 # cd to per node directory
1722 self
.cmd("install -m 775 -o frr -g frr -d {}/{}".format(self
.logdir
, self
.name
))
1723 self
.set_cwd("{}/{}".format(self
.logdir
, self
.name
))
1724 self
.cmd("umask 000")
1726 # Re-enable to allow for report per run
1727 self
.reportCores
= True
1729 # XXX: glue code forward ported from removed function.
1730 if self
.version
== None:
1731 self
.version
= self
.cmd(
1732 os
.path
.join(self
.daemondir
, "bgpd") + " -v"
1734 logger
.info("{}: running version: {}".format(self
.name
, self
.version
))
1735 # If `daemons` was specified then some upper API called us with
1736 # specific daemons, otherwise just use our own configuration.
1738 if daemons
is not None:
1739 daemons_list
= daemons
1741 # Append all daemons configured.
1742 for daemon
in self
.daemons
:
1743 if self
.daemons
[daemon
] == 1:
1744 daemons_list
.append(daemon
)
1746 def start_daemon(daemon
, extra_opts
=None):
1747 daemon_opts
= self
.daemons_options
.get(daemon
, "")
1748 rediropt
= " > {0}.out 2> {0}.err".format(daemon
)
1749 if daemon
== "snmpd":
1750 binary
= "/usr/sbin/snmpd"
1752 cmdopt
= "{} -C -c /etc/frr/snmpd.conf -p ".format(
1754 ) + "/var/run/{}/snmpd.pid -x /etc/frr/agentx".format(self
.routertype
)
1756 binary
= os
.path
.join(self
.daemondir
, daemon
)
1758 cmdenv
= "ASAN_OPTIONS="
1760 cmdenv
= "abort_on_error=1:"
1761 cmdenv
+= "log_path={0}/{1}.{2}.asan ".format(
1762 self
.logdir
, self
.name
, daemon
1765 if valgrind_memleaks
:
1766 this_dir
= os
.path
.dirname(
1767 os
.path
.abspath(os
.path
.realpath(__file__
))
1769 supp_file
= os
.path
.abspath(
1770 os
.path
.join(this_dir
, "../../../tools/valgrind.supp")
1772 cmdenv
+= " /usr/bin/valgrind --num-callers=50 --log-file={1}/{2}.valgrind.{0}.%p --leak-check=full --suppressions={3}".format(
1773 daemon
, self
.logdir
, self
.name
, supp_file
1777 " --gen-suppressions=all --expensive-definedness-checks=yes"
1779 elif daemon
in strace_daemons
or "all" in strace_daemons
:
1780 cmdenv
= "strace -f -D -o {1}/{2}.strace.{0} ".format(
1781 daemon
, self
.logdir
, self
.name
1784 cmdopt
= "{} --command-log-always --log file:{}.log --log-level debug".format(
1788 cmdopt
+= " " + extra_opts
1791 (gdb_routers
or gdb_daemons
)
1793 not gdb_routers
or self
.name
in gdb_routers
or "all" in gdb_routers
1795 and (not gdb_daemons
or daemon
in gdb_daemons
or "all" in gdb_daemons
)
1797 if daemon
== "snmpd":
1801 gdbcmd
= "sudo -E gdb " + binary
1803 gdbcmd
+= " -ex 'set breakpoint pending on'"
1804 for bp
in gdb_breakpoints
:
1805 gdbcmd
+= " -ex 'b {}'".format(bp
)
1806 gdbcmd
+= " -ex 'run {}'".format(cmdopt
)
1808 self
.run_in_window(gdbcmd
, daemon
)
1811 "%s: %s %s launched in gdb window", self
, self
.routertype
, daemon
1814 if daemon
!= "snmpd":
1819 self
.cmd_raises(" ".join([cmdenv
, binary
, cmdopt
]), warn
=False)
1820 except subprocess
.CalledProcessError
as error
:
1822 '%s: Failed to launch "%s" daemon (%d) using: %s%s%s:',
1827 '\n:stdout: "{}"'.format(error
.stdout
.strip())
1830 '\n:stderr: "{}"'.format(error
.stderr
.strip())
1835 logger
.info("%s: %s %s started", self
, self
.routertype
, daemon
)
1838 if "zebra" in daemons_list
:
1839 start_daemon("zebra", "-s 90000000")
1840 while "zebra" in daemons_list
:
1841 daemons_list
.remove("zebra")
1843 # Start staticd next if required
1844 if "staticd" in daemons_list
:
1845 start_daemon("staticd")
1846 while "staticd" in daemons_list
:
1847 daemons_list
.remove("staticd")
1849 if "snmpd" in daemons_list
:
1850 # Give zerbra a chance to configure interface addresses that snmpd daemon
1854 start_daemon("snmpd")
1855 while "snmpd" in daemons_list
:
1856 daemons_list
.remove("snmpd")
1859 # Fix Link-Local Addresses on initial startup
1860 # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this
1861 _
, output
, _
= self
.cmd_status(
1862 "for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; echo $i: $mac; [ -z \"$mac\" ] && continue; IFS=':'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done",
1863 stderr
=subprocess
.STDOUT
,
1865 logger
.debug("Set MACs:\n%s", output
)
1867 # Now start all the other daemons
1868 for daemon
in daemons_list
:
1869 if self
.daemons
[daemon
] == 0:
1871 start_daemon(daemon
)
1873 # Check if daemons are running.
1874 rundaemons
= self
.cmd("ls -1 /var/run/%s/*.pid" % self
.routertype
)
1875 if re
.search(r
"No such file or directory", rundaemons
):
1876 return "Daemons are not running"
1878 # Update the permissions on the log files
1879 self
.cmd("chown frr:frr -R {}/{}".format(self
.logdir
, self
.name
))
1880 self
.cmd("chmod ug+rwX,o+r -R {}/{}".format(self
.logdir
, self
.name
))
1884 def killRouterDaemons(
1885 self
, daemons
, wait
=True, assertOnError
=True, minErrorVersion
="5.1"
1888 # Daemons(user specified daemon only) using SIGKILL
1889 rundaemons
= self
.cmd("ls -1 /var/run/%s/*.pid" % self
.routertype
)
1891 daemonsNotRunning
= []
1892 if re
.search(r
"No such file or directory", rundaemons
):
1894 for daemon
in daemons
:
1895 if rundaemons
is not None and daemon
in rundaemons
:
1897 dmns
= rundaemons
.split("\n")
1898 # Exclude empty string at end of list
1900 if re
.search(r
"%s" % daemon
, d
):
1901 daemonpidfile
= d
.rstrip()
1902 daemonpid
= self
.cmd("cat %s" % daemonpidfile
).rstrip()
1903 if daemonpid
.isdigit() and pid_exists(int(daemonpid
)):
1905 "{}: killing {}".format(
1907 os
.path
.basename(daemonpidfile
.rsplit(".", 1)[0]),
1910 os
.kill(int(daemonpid
), signal
.SIGKILL
)
1911 if pid_exists(int(daemonpid
)):
1913 while wait
and numRunning
> 0:
1916 "{}: waiting for {} daemon to be stopped".format(
1921 # 2nd round of kill if daemons didn't exit
1923 if re
.search(r
"%s" % daemon
, d
):
1924 daemonpid
= self
.cmd("cat %s" % d
.rstrip()).rstrip()
1925 if daemonpid
.isdigit() and pid_exists(
1929 "{}: killing {}".format(
1932 d
.rstrip().rsplit(".", 1)[0]
1936 os
.kill(int(daemonpid
), signal
.SIGKILL
)
1937 if daemonpid
.isdigit() and not pid_exists(
1941 self
.cmd("rm -- {}".format(daemonpidfile
))
1943 errors
= self
.checkRouterCores(reportOnce
=True)
1944 if self
.checkRouterVersion("<", minErrorVersion
):
1945 # ignore errors in old versions
1947 if assertOnError
and len(errors
) > 0:
1948 assert "Errors found - details follow:" == 0, errors
1950 daemonsNotRunning
.append(daemon
)
1951 if len(daemonsNotRunning
) > 0:
1952 errors
= errors
+ "Daemons are not running", daemonsNotRunning
1956 def checkRouterCores(self
, reportLeaks
=True, reportOnce
=False):
1957 if reportOnce
and not self
.reportCores
:
1961 for daemon
in self
.daemons
:
1962 if self
.daemons
[daemon
] == 1:
1963 # Look for core file
1964 corefiles
= glob
.glob(
1965 "{}/{}/{}_core*.dmp".format(self
.logdir
, self
.name
, daemon
)
1967 if len(corefiles
) > 0:
1968 backtrace
= gdb_core(self
, daemon
, corefiles
)
1971 + "\n%s: %s crashed. Core file found - Backtrace follows:\n%s"
1972 % (self
.name
, daemon
, backtrace
)
1976 log
= self
.getStdErr(daemon
)
1977 if "memstats" in log
:
1979 "%s: %s has memory leaks:\n" % (self
.name
, daemon
)
1981 traces
= traces
+ "\n%s: %s has memory leaks:\n" % (
1985 log
= re
.sub("core_handler: ", "", log
)
1987 r
"(showing active allocations in memory group [a-zA-Z0-9]+)",
1991 log
= re
.sub("memstats: ", " ", log
)
1992 sys
.stderr
.write(log
)
1994 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
1995 if checkAddressSanitizerError(
1996 self
.getStdErr(daemon
), self
.name
, daemon
, self
.logdir
1999 "%s: Daemon %s killed by AddressSanitizer" % (self
.name
, daemon
)
2001 traces
= traces
+ "\n%s: Daemon %s killed by AddressSanitizer" % (
2007 self
.reportCores
= False
2010 def checkRouterRunning(self
):
2011 "Check if router daemons are running and collect crashinfo they don't run"
2015 daemonsRunning
= self
.cmd(
2016 'vtysh -c "show logging" | grep "Logging configuration for"'
2018 # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found
2019 if checkAddressSanitizerError(daemonsRunning
, self
.name
, "vtysh"):
2020 return "%s: vtysh killed by AddressSanitizer" % (self
.name
)
2022 for daemon
in self
.daemons
:
2023 if daemon
== "snmpd":
2025 if (self
.daemons
[daemon
] == 1) and not (daemon
in daemonsRunning
):
2026 sys
.stderr
.write("%s: Daemon %s not running\n" % (self
.name
, daemon
))
2027 if daemon
== "staticd":
2029 "You may have a copy of staticd installed but are attempting to test against\n"
2032 "a version of FRR that does not have staticd, please cleanup the install dir\n"
2035 # Look for core file
2036 corefiles
= glob
.glob(
2037 "{}/{}/{}_core*.dmp".format(self
.logdir
, self
.name
, daemon
)
2039 if len(corefiles
) > 0:
2040 gdb_core(self
, daemon
, corefiles
)
2042 # No core found - If we find matching logfile in /tmp, then print last 20 lines from it.
2044 "{}/{}/{}.log".format(self
.logdir
, self
.name
, daemon
)
2046 log_tail
= subprocess
.check_output(
2048 "tail -n20 {}/{}/{}.log 2> /dev/null".format(
2049 self
.logdir
, self
.name
, daemon
2055 "\nFrom %s %s %s log file:\n"
2056 % (self
.routertype
, self
.name
, daemon
)
2058 sys
.stderr
.write("%s\n" % log_tail
)
2060 # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found
2061 if checkAddressSanitizerError(
2062 self
.getStdErr(daemon
), self
.name
, daemon
, self
.logdir
2064 return "%s: Daemon %s not running - killed by AddressSanitizer" % (
2069 return "%s: Daemon %s not running" % (self
.name
, daemon
)
2072 def checkRouterVersion(self
, cmpop
, version
):
2074 Compares router version using operation `cmpop` with `version`.
2075 Valid `cmpop` values:
2076 * `>=`: has the same version or greater
2077 * '>': has greater version
2078 * '=': has the same version
2079 * '<': has a lesser version
2080 * '<=': has the same version or lesser
2082 Usage example: router.checkRouterVersion('>', '1.0')
2085 # Make sure we have version information first
2086 if self
.version
== None:
2087 self
.version
= self
.cmd(
2088 os
.path
.join(self
.daemondir
, "bgpd") + " -v"
2090 logger
.info("{}: running version: {}".format(self
.name
, self
.version
))
2092 rversion
= self
.version
2093 if rversion
== None:
2096 result
= version_cmp(rversion
, version
)
2110 def get_ipv6_linklocal(self
):
2111 "Get LinkLocal Addresses from interfaces"
2115 ifaces
= self
.cmd("ip -6 address")
2116 # Fix newlines (make them all the same)
2117 ifaces
= ("\n".join(ifaces
.splitlines()) + "\n").splitlines()
2121 m
= re
.search("[0-9]+: ([^:@]+)[-@a-z0-9:]+ <", line
)
2123 interface
= m
.group(1)
2126 "inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link",
2131 ll_per_if_count
+= 1
2132 if ll_per_if_count
> 1:
2133 linklocal
+= [["%s-%s" % (interface
, ll_per_if_count
), local
]]
2135 linklocal
+= [[interface
, local
]]
2138 def daemon_available(self
, daemon
):
2139 "Check if specified daemon is installed (and for ldp if kernel supports MPLS)"
2141 daemon_path
= os
.path
.join(self
.daemondir
, daemon
)
2142 if not os
.path
.isfile(daemon_path
):
2144 if daemon
== "ldpd":
2145 if version_cmp(platform
.release(), "4.5") < 0:
2147 if not module_present("mpls-router", load
=False):
2149 if not module_present("mpls-iptunnel", load
=False):
2153 def get_routertype(self
):
2154 "Return the type of Router (frr)"
2156 return self
.routertype
2158 def report_memory_leaks(self
, filename_prefix
, testscript
):
2159 "Report Memory Leaks to file prefixed with given string"
2162 filename
= filename_prefix
+ re
.sub(r
"\.py", "", testscript
) + ".txt"
2163 for daemon
in self
.daemons
:
2164 if self
.daemons
[daemon
] == 1:
2165 log
= self
.getStdErr(daemon
)
2166 if "memstats" in log
:
2169 "\nRouter {} {} StdErr Log:\n{}".format(self
.name
, daemon
, log
)
2173 # Check if file already exists
2174 fileexists
= os
.path
.isfile(filename
)
2175 leakfile
= open(filename
, "a")
2177 # New file - add header
2179 "# Memory Leak Detection for topotest %s\n\n"
2182 leakfile
.write("## Router %s\n" % self
.name
)
2183 leakfile
.write("### Process %s\n" % daemon
)
2184 log
= re
.sub("core_handler: ", "", log
)
2186 r
"(showing active allocations in memory group [a-zA-Z0-9]+)",
2190 log
= re
.sub("memstats: ", " ", log
)
2192 leakfile
.write("\n")
2198 """Convert string to unicode, depending on python version"""
2199 if sys
.version_info
[0] > 2:
2202 return unicode(s
) # pylint: disable=E0602
2206 return isinstance(o
, Mapping
)