]>
Commit | Line | Data |
---|---|---|
594b1259 | 1 | #!/usr/bin/env python |
acddc0ed | 2 | # SPDX-License-Identifier: ISC |
594b1259 MW |
3 | |
4 | # | |
5 | # topotest.py | |
6 | # Library of helper functions for NetDEF Topology Tests | |
7 | # | |
8 | # Copyright (c) 2016 by | |
9 | # Network Device Education Foundation, Inc. ("NetDEF") | |
10 | # | |
594b1259 | 11 | |
249ac6f0 | 12 | import configparser |
49581587 | 13 | import difflib |
50c40bde | 14 | import errno |
fd858290 | 15 | import functools |
594b1259 | 16 | import glob |
49581587 CH |
17 | import json |
18 | import os | |
49581587 CH |
19 | import platform |
20 | import re | |
21 | import resource | |
22 | import signal | |
594b1259 | 23 | import subprocess |
49581587 | 24 | import sys |
1fca63c1 | 25 | import tempfile |
570f25d8 | 26 | import time |
249ac6f0 | 27 | from collections.abc import Mapping |
49581587 | 28 | from copy import deepcopy |
594b1259 | 29 | |
49581587 | 30 | import lib.topolog as topolog |
249ac6f0 | 31 | from lib.micronet_compat import Node |
6c131bd3 | 32 | from lib.topolog import logger |
260268c4 | 33 | from munet.base import Timeout |
6c131bd3 | 34 | |
49581587 | 35 | from lib import micronet |
594b1259 | 36 | |
249ac6f0 | 37 | g_pytest_config = None |
701a0192 | 38 | |
a53c08bc | 39 | |
49581587 CH |
40 | def get_logs_path(rundir): |
41 | logspath = topolog.get_test_logdir() | |
42 | return os.path.join(rundir, logspath) | |
43 | ||
0b25370e | 44 | |
79f6fdeb | 45 | def gdb_core(obj, daemon, corefiles): |
701a0192 | 46 | gdbcmds = """ |
79f6fdeb DL |
47 | info threads |
48 | bt full | |
49 | disassemble | |
50 | up | |
51 | disassemble | |
52 | up | |
53 | disassemble | |
54 | up | |
55 | disassemble | |
56 | up | |
57 | disassemble | |
58 | up | |
59 | disassemble | |
701a0192 | 60 | """ |
61 | gdbcmds = [["-ex", i.strip()] for i in gdbcmds.strip().split("\n")] | |
79f6fdeb DL |
62 | gdbcmds = [item for sl in gdbcmds for item in sl] |
63 | ||
64 | daemon_path = os.path.join(obj.daemondir, daemon) | |
65 | backtrace = subprocess.check_output( | |
701a0192 | 66 | ["gdb", daemon_path, corefiles[0], "--batch"] + gdbcmds |
79f6fdeb DL |
67 | ) |
68 | sys.stderr.write( | |
701a0192 | 69 | "\n%s: %s crashed. Core file found - Backtrace follows:\n" % (obj.name, daemon) |
79f6fdeb DL |
70 | ) |
71 | sys.stderr.write("%s" % backtrace) | |
72 | return backtrace | |
787e7624 | 73 | |
701a0192 | 74 | |
3668ed8d RZ |
75 | class json_cmp_result(object): |
76 | "json_cmp result class for better assertion messages" | |
77 | ||
78 | def __init__(self): | |
79 | self.errors = [] | |
80 | ||
81 | def add_error(self, error): | |
82 | "Append error message to the result" | |
2db5888d RZ |
83 | for line in error.splitlines(): |
84 | self.errors.append(line) | |
3668ed8d RZ |
85 | |
86 | def has_errors(self): | |
87 | "Returns True if there were errors, otherwise False." | |
88 | return len(self.errors) > 0 | |
89 | ||
849224d4 G |
90 | def gen_report(self): |
91 | headline = ["Generated JSON diff error report:", ""] | |
92 | return headline + self.errors | |
93 | ||
7fe06d55 | 94 | def __str__(self): |
849224d4 G |
95 | return ( |
96 | "Generated JSON diff error report:\n\n\n" + "\n".join(self.errors) + "\n\n" | |
97 | ) | |
7fe06d55 | 98 | |
da63d5b3 | 99 | |
849224d4 | 100 | def gen_json_diff_report(d1, d2, exact=False, path="> $", acc=(0, "")): |
7bd28cfc | 101 | """ |
849224d4 | 102 | Internal workhorse which compares two JSON data structures and generates an error report suited to be read by a human eye. |
7bd28cfc | 103 | """ |
849224d4 G |
104 | |
105 | def dump_json(v): | |
106 | if isinstance(v, (dict, list)): | |
107 | return "\t" + "\t".join( | |
108 | json.dumps(v, indent=4, separators=(",", ": ")).splitlines(True) | |
787e7624 | 109 | ) |
849224d4 G |
110 | else: |
111 | return "'{}'".format(v) | |
112 | ||
113 | def json_type(v): | |
114 | if isinstance(v, (list, tuple)): | |
115 | return "Array" | |
116 | elif isinstance(v, dict): | |
117 | return "Object" | |
118 | elif isinstance(v, (int, float)): | |
119 | return "Number" | |
120 | elif isinstance(v, bool): | |
121 | return "Boolean" | |
122 | elif isinstance(v, str): | |
123 | return "String" | |
124 | elif v == None: | |
125 | return "null" | |
126 | ||
127 | def get_errors(other_acc): | |
128 | return other_acc[1] | |
129 | ||
130 | def get_errors_n(other_acc): | |
131 | return other_acc[0] | |
132 | ||
133 | def add_error(acc, msg, points=1): | |
134 | return (acc[0] + points, acc[1] + "{}: {}\n".format(path, msg)) | |
135 | ||
136 | def merge_errors(acc, other_acc): | |
137 | return (acc[0] + other_acc[0], acc[1] + other_acc[1]) | |
138 | ||
139 | def add_idx(idx): | |
140 | return "{}[{}]".format(path, idx) | |
141 | ||
142 | def add_key(key): | |
143 | return "{}->{}".format(path, key) | |
144 | ||
145 | def has_errors(other_acc): | |
146 | return other_acc[0] > 0 | |
147 | ||
148 | if d2 == "*" or ( | |
149 | not isinstance(d1, (list, dict)) | |
150 | and not isinstance(d2, (list, dict)) | |
151 | and d1 == d2 | |
152 | ): | |
153 | return acc | |
154 | elif ( | |
155 | not isinstance(d1, (list, dict)) | |
156 | and not isinstance(d2, (list, dict)) | |
157 | and d1 != d2 | |
158 | ): | |
159 | acc = add_error( | |
160 | acc, | |
161 | "d1 has element with value '{}' but in d2 it has value '{}'".format(d1, d2), | |
787e7624 | 162 | ) |
849224d4 G |
163 | elif ( |
164 | isinstance(d1, list) | |
165 | and isinstance(d2, list) | |
166 | and ((len(d2) > 0 and d2[0] == "__ordered__") or exact) | |
167 | ): | |
168 | if not exact: | |
169 | del d2[0] | |
170 | if len(d1) != len(d2): | |
171 | acc = add_error( | |
172 | acc, | |
173 | "d1 has Array of length {} but in d2 it is of length {}".format( | |
174 | len(d1), len(d2) | |
175 | ), | |
787e7624 | 176 | ) |
849224d4 G |
177 | else: |
178 | for idx, v1, v2 in zip(range(0, len(d1)), d1, d2): | |
179 | acc = merge_errors( | |
180 | acc, gen_json_diff_report(v1, v2, exact=exact, path=add_idx(idx)) | |
181 | ) | |
182 | elif isinstance(d1, list) and isinstance(d2, list): | |
183 | if len(d1) < len(d2): | |
184 | acc = add_error( | |
185 | acc, | |
186 | "d1 has Array of length {} but in d2 it is of length {}".format( | |
187 | len(d1), len(d2) | |
188 | ), | |
189 | ) | |
190 | else: | |
191 | for idx2, v2 in zip(range(0, len(d2)), d2): | |
192 | found_match = False | |
193 | closest_diff = None | |
194 | closest_idx = None | |
195 | for idx1, v1 in zip(range(0, len(d1)), d1): | |
b3100f6c G |
196 | tmp_v1 = deepcopy(v1) |
197 | tmp_v2 = deepcopy(v2) | |
198 | tmp_diff = gen_json_diff_report(tmp_v1, tmp_v2, path=add_idx(idx1)) | |
849224d4 G |
199 | if not has_errors(tmp_diff): |
200 | found_match = True | |
201 | del d1[idx1] | |
202 | break | |
203 | elif not closest_diff or get_errors_n(tmp_diff) < get_errors_n( | |
204 | closest_diff | |
205 | ): | |
206 | closest_diff = tmp_diff | |
207 | closest_idx = idx1 | |
208 | if not found_match and isinstance(v2, (list, dict)): | |
209 | sub_error = "\n\n\t{}".format( | |
210 | "\t".join(get_errors(closest_diff).splitlines(True)) | |
211 | ) | |
212 | acc = add_error( | |
213 | acc, | |
214 | ( | |
215 | "d2 has the following element at index {} which is not present in d1: " | |
216 | + "\n\n{}\n\n\tClosest match in d1 is at index {} with the following errors: {}" | |
217 | ).format(idx2, dump_json(v2), closest_idx, sub_error), | |
218 | ) | |
219 | if not found_match and not isinstance(v2, (list, dict)): | |
220 | acc = add_error( | |
221 | acc, | |
222 | "d2 has the following element at index {} which is not present in d1: {}".format( | |
223 | idx2, dump_json(v2) | |
224 | ), | |
225 | ) | |
226 | elif isinstance(d1, dict) and isinstance(d2, dict) and exact: | |
227 | invalid_keys_d1 = [k for k in d1.keys() if k not in d2.keys()] | |
228 | invalid_keys_d2 = [k for k in d2.keys() if k not in d1.keys()] | |
229 | for k in invalid_keys_d1: | |
230 | acc = add_error(acc, "d1 has key '{}' which is not present in d2".format(k)) | |
231 | for k in invalid_keys_d2: | |
232 | acc = add_error(acc, "d2 has key '{}' which is not present in d1".format(k)) | |
233 | valid_keys_intersection = [k for k in d1.keys() if k in d2.keys()] | |
234 | for k in valid_keys_intersection: | |
235 | acc = merge_errors( | |
236 | acc, gen_json_diff_report(d1[k], d2[k], exact=exact, path=add_key(k)) | |
237 | ) | |
238 | elif isinstance(d1, dict) and isinstance(d2, dict): | |
239 | none_keys = [k for k, v in d2.items() if v == None] | |
240 | none_keys_present = [k for k in d1.keys() if k in none_keys] | |
241 | for k in none_keys_present: | |
242 | acc = add_error( | |
243 | acc, "d1 has key '{}' which is not supposed to be present".format(k) | |
244 | ) | |
245 | keys = [k for k, v in d2.items() if v != None] | |
246 | invalid_keys_intersection = [k for k in keys if k not in d1.keys()] | |
247 | for k in invalid_keys_intersection: | |
248 | acc = add_error(acc, "d2 has key '{}' which is not present in d1".format(k)) | |
249 | valid_keys_intersection = [k for k in keys if k in d1.keys()] | |
250 | for k in valid_keys_intersection: | |
251 | acc = merge_errors( | |
252 | acc, gen_json_diff_report(d1[k], d2[k], exact=exact, path=add_key(k)) | |
253 | ) | |
254 | else: | |
255 | acc = add_error( | |
256 | acc, | |
257 | "d1 has element of type '{}' but the corresponding element in d2 is of type '{}'".format( | |
258 | json_type(d1), json_type(d2) | |
259 | ), | |
260 | points=2, | |
787e7624 | 261 | ) |
a82e5f9a | 262 | |
849224d4 | 263 | return acc |
a82e5f9a | 264 | |
849224d4 G |
265 | |
266 | def json_cmp(d1, d2, exact=False): | |
09e21b44 RZ |
267 | """ |
268 | JSON compare function. Receives two parameters: | |
849224d4 G |
269 | * `d1`: parsed JSON data structure |
270 | * `d2`: parsed JSON data structure | |
271 | ||
272 | Returns 'None' when all JSON Object keys and all Array elements of d2 have a match | |
49581587 | 273 | in d1, i.e., when d2 is a "subset" of d1 without honoring any order. Otherwise an |
849224d4 G |
274 | error report is generated and wrapped in a 'json_cmp_result()'. There are special |
275 | parameters and notations explained below which can be used to cover rather unusual | |
276 | cases: | |
277 | ||
278 | * when 'exact is set to 'True' then d1 and d2 are tested for equality (including | |
279 | order within JSON Arrays) | |
280 | * using 'null' (or 'None' in Python) as JSON Object value is checking for key | |
281 | absence in d1 | |
282 | * using '*' as JSON Object value or Array value is checking for presence in d1 | |
283 | without checking the values | |
284 | * using '__ordered__' as first element in a JSON Array in d2 will also check the | |
285 | order when it is compared to an Array in d1 | |
09e21b44 | 286 | """ |
09e21b44 | 287 | |
849224d4 | 288 | (errors_n, errors) = gen_json_diff_report(deepcopy(d1), deepcopy(d2), exact=exact) |
3668ed8d | 289 | |
849224d4 G |
290 | if errors_n > 0: |
291 | result = json_cmp_result() | |
292 | result.add_error(errors) | |
3668ed8d | 293 | return result |
849224d4 G |
294 | else: |
295 | return None | |
09e21b44 | 296 | |
a82e5f9a | 297 | |
5cffda18 RZ |
298 | def router_output_cmp(router, cmd, expected): |
299 | """ | |
300 | Runs `cmd` in router and compares the output with `expected`. | |
301 | """ | |
787e7624 | 302 | return difflines( |
303 | normalize_text(router.vtysh_cmd(cmd)), | |
304 | normalize_text(expected), | |
305 | title1="Current output", | |
306 | title2="Expected output", | |
307 | ) | |
5cffda18 RZ |
308 | |
309 | ||
849224d4 | 310 | def router_json_cmp(router, cmd, data, exact=False): |
5cffda18 RZ |
311 | """ |
312 | Runs `cmd` that returns JSON data (normally the command ends with 'json') | |
313 | and compare with `data` contents. | |
314 | """ | |
849224d4 | 315 | return json_cmp(router.vtysh_cmd(cmd, isjson=True), data, exact) |
5cffda18 RZ |
316 | |
317 | ||
1fca63c1 RZ |
318 | def run_and_expect(func, what, count=20, wait=3): |
319 | """ | |
320 | Run `func` and compare the result with `what`. Do it for `count` times | |
321 | waiting `wait` seconds between tries. By default it tries 20 times with | |
322 | 3 seconds delay between tries. | |
323 | ||
324 | Returns (True, func-return) on success or | |
325 | (False, func-return) on failure. | |
5cffda18 RZ |
326 | |
327 | --- | |
328 | ||
329 | Helper functions to use with this function: | |
330 | - router_output_cmp | |
331 | - router_json_cmp | |
1fca63c1 | 332 | """ |
fd858290 RZ |
333 | start_time = time.time() |
334 | func_name = "<unknown>" | |
335 | if func.__class__ == functools.partial: | |
336 | func_name = func.func.__name__ | |
337 | else: | |
338 | func_name = func.__name__ | |
339 | ||
a5722d5a DA |
340 | # Just a safety-check to avoid running topotests with very |
341 | # small wait/count arguments. | |
342 | wait_time = wait * count | |
343 | if wait_time < 5: | |
344 | assert ( | |
345 | wait_time >= 5 | |
346 | ), "Waiting time is too small (count={}, wait={}), adjust timer values".format( | |
347 | count, wait | |
348 | ) | |
349 | ||
e8f7a22f | 350 | logger.debug( |
8d3dab20 IR |
351 | "'{}' polling started (interval {} secs, maximum {} tries)".format( |
352 | func_name, wait, count | |
787e7624 | 353 | ) |
354 | ) | |
fd858290 | 355 | |
1fca63c1 RZ |
356 | while count > 0: |
357 | result = func() | |
358 | if result != what: | |
570f25d8 | 359 | time.sleep(wait) |
1fca63c1 RZ |
360 | count -= 1 |
361 | continue | |
fd858290 RZ |
362 | |
363 | end_time = time.time() | |
e8f7a22f | 364 | logger.debug( |
787e7624 | 365 | "'{}' succeeded after {:.2f} seconds".format( |
366 | func_name, end_time - start_time | |
367 | ) | |
368 | ) | |
1fca63c1 | 369 | return (True, result) |
fd858290 RZ |
370 | |
371 | end_time = time.time() | |
787e7624 | 372 | logger.error( |
373 | "'{}' failed after {:.2f} seconds".format(func_name, end_time - start_time) | |
374 | ) | |
1fca63c1 RZ |
375 | return (False, result) |
376 | ||
377 | ||
a6fd124a RZ |
378 | def run_and_expect_type(func, etype, count=20, wait=3, avalue=None): |
379 | """ | |
380 | Run `func` and compare the result with `etype`. Do it for `count` times | |
381 | waiting `wait` seconds between tries. By default it tries 20 times with | |
382 | 3 seconds delay between tries. | |
383 | ||
384 | This function is used when you want to test the return type and, | |
385 | optionally, the return value. | |
386 | ||
387 | Returns (True, func-return) on success or | |
388 | (False, func-return) on failure. | |
389 | """ | |
390 | start_time = time.time() | |
391 | func_name = "<unknown>" | |
392 | if func.__class__ == functools.partial: | |
393 | func_name = func.func.__name__ | |
394 | else: | |
395 | func_name = func.__name__ | |
396 | ||
a5722d5a DA |
397 | # Just a safety-check to avoid running topotests with very |
398 | # small wait/count arguments. | |
399 | wait_time = wait * count | |
400 | if wait_time < 5: | |
401 | assert ( | |
402 | wait_time >= 5 | |
403 | ), "Waiting time is too small (count={}, wait={}), adjust timer values".format( | |
404 | count, wait | |
405 | ) | |
406 | ||
e8f7a22f | 407 | logger.debug( |
a6fd124a | 408 | "'{}' polling started (interval {} secs, maximum wait {} secs)".format( |
787e7624 | 409 | func_name, wait, int(wait * count) |
410 | ) | |
411 | ) | |
a6fd124a RZ |
412 | |
413 | while count > 0: | |
414 | result = func() | |
415 | if not isinstance(result, etype): | |
787e7624 | 416 | logger.debug( |
417 | "Expected result type '{}' got '{}' instead".format(etype, type(result)) | |
418 | ) | |
a6fd124a RZ |
419 | time.sleep(wait) |
420 | count -= 1 | |
421 | continue | |
422 | ||
423 | if etype != type(None) and avalue != None and result != avalue: | |
424 | logger.debug("Expected value '{}' got '{}' instead".format(avalue, result)) | |
425 | time.sleep(wait) | |
426 | count -= 1 | |
427 | continue | |
428 | ||
429 | end_time = time.time() | |
e8f7a22f | 430 | logger.debug( |
787e7624 | 431 | "'{}' succeeded after {:.2f} seconds".format( |
432 | func_name, end_time - start_time | |
433 | ) | |
434 | ) | |
a6fd124a RZ |
435 | return (True, result) |
436 | ||
437 | end_time = time.time() | |
787e7624 | 438 | logger.error( |
439 | "'{}' failed after {:.2f} seconds".format(func_name, end_time - start_time) | |
440 | ) | |
a6fd124a RZ |
441 | return (False, result) |
442 | ||
443 | ||
1375385a CH |
444 | def router_json_cmp_retry(router, cmd, data, exact=False, retry_timeout=10.0): |
445 | """ | |
446 | Runs `cmd` that returns JSON data (normally the command ends with 'json') | |
447 | and compare with `data` contents. Retry by default for 10 seconds | |
448 | """ | |
449 | ||
450 | def test_func(): | |
451 | return router_json_cmp(router, cmd, data, exact) | |
452 | ||
453 | ok, _ = run_and_expect(test_func, None, int(retry_timeout), 1) | |
454 | return ok | |
455 | ||
456 | ||
594b1259 MW |
457 | def int2dpid(dpid): |
458 | "Converting Integer to DPID" | |
459 | ||
460 | try: | |
461 | dpid = hex(dpid)[2:] | |
787e7624 | 462 | dpid = "0" * (16 - len(dpid)) + dpid |
594b1259 MW |
463 | return dpid |
464 | except IndexError: | |
787e7624 | 465 | raise Exception( |
466 | "Unable to derive default datapath ID - " | |
467 | "please either specify a dpid or use a " | |
468 | "canonical switch name such as s23." | |
469 | ) | |
470 | ||
594b1259 | 471 | |
bc2872fd | 472 | def get_textdiff(text1, text2, title1="", title2="", **opts): |
17070436 MW |
473 | "Returns empty string if same or formatted diff" |
474 | ||
787e7624 | 475 | diff = "\n".join( |
476 | difflib.unified_diff(text1, text2, fromfile=title1, tofile=title2, **opts) | |
477 | ) | |
17070436 MW |
478 | # Clean up line endings |
479 | diff = os.linesep.join([s for s in diff.splitlines() if s]) | |
480 | return diff | |
481 | ||
787e7624 | 482 | |
483 | def difflines(text1, text2, title1="", title2="", **opts): | |
1fca63c1 | 484 | "Wrapper for get_textdiff to avoid string transformations." |
787e7624 | 485 | text1 = ("\n".join(text1.rstrip().splitlines()) + "\n").splitlines(1) |
486 | text2 = ("\n".join(text2.rstrip().splitlines()) + "\n").splitlines(1) | |
bc2872fd | 487 | return get_textdiff(text1, text2, title1, title2, **opts) |
1fca63c1 | 488 | |
787e7624 | 489 | |
1fca63c1 RZ |
490 | def get_file(content): |
491 | """ | |
492 | Generates a temporary file in '/tmp' with `content` and returns the file name. | |
493 | """ | |
49581587 CH |
494 | if isinstance(content, list) or isinstance(content, tuple): |
495 | content = "\n".join(content) | |
787e7624 | 496 | fde = tempfile.NamedTemporaryFile(mode="w", delete=False) |
1fca63c1 RZ |
497 | fname = fde.name |
498 | fde.write(content) | |
499 | fde.close() | |
500 | return fname | |
501 | ||
787e7624 | 502 | |
f7840f6b RZ |
503 | def normalize_text(text): |
504 | """ | |
9683a1bb | 505 | Strips formating spaces/tabs, carriage returns and trailing whitespace. |
f7840f6b | 506 | """ |
787e7624 | 507 | text = re.sub(r"[ \t]+", " ", text) |
508 | text = re.sub(r"\r", "", text) | |
9683a1bb RZ |
509 | |
510 | # Remove whitespace in the middle of text. | |
787e7624 | 511 | text = re.sub(r"[ \t]+\n", "\n", text) |
9683a1bb RZ |
512 | # Remove whitespace at the end of the text. |
513 | text = text.rstrip() | |
514 | ||
f7840f6b RZ |
515 | return text |
516 | ||
787e7624 | 517 | |
0414a764 DS |
518 | def is_linux(): |
519 | """ | |
520 | Parses unix name output to check if running on GNU/Linux. | |
521 | ||
522 | Returns True if running on Linux, returns False otherwise. | |
523 | """ | |
524 | ||
525 | if os.uname()[0] == "Linux": | |
526 | return True | |
527 | return False | |
528 | ||
529 | ||
530 | def iproute2_is_vrf_capable(): | |
531 | """ | |
532 | Checks if the iproute2 version installed on the system is capable of | |
533 | handling VRFs by interpreting the output of the 'ip' utility found in PATH. | |
534 | ||
535 | Returns True if capability can be detected, returns False otherwise. | |
536 | """ | |
537 | ||
538 | if is_linux(): | |
539 | try: | |
540 | subp = subprocess.Popen( | |
541 | ["ip", "route", "show", "vrf"], | |
542 | stdout=subprocess.PIPE, | |
543 | stderr=subprocess.PIPE, | |
0b25370e | 544 | stdin=subprocess.PIPE, |
0414a764 DS |
545 | ) |
546 | iproute2_err = subp.communicate()[1].splitlines()[0].split()[0] | |
547 | ||
548 | if iproute2_err != "Error:": | |
549 | return True | |
550 | except Exception: | |
551 | pass | |
552 | return False | |
553 | ||
d6c755f2 | 554 | |
51c33a57 SW |
555 | def iproute2_is_fdb_get_capable(): |
556 | """ | |
557 | Checks if the iproute2 version installed on the system is capable of | |
558 | handling `bridge fdb get` commands to query neigh table resolution. | |
559 | ||
560 | Returns True if capability can be detected, returns False otherwise. | |
561 | """ | |
562 | ||
563 | if is_linux(): | |
564 | try: | |
565 | subp = subprocess.Popen( | |
566 | ["bridge", "fdb", "get", "help"], | |
567 | stdout=subprocess.PIPE, | |
568 | stderr=subprocess.PIPE, | |
569 | stdin=subprocess.PIPE, | |
570 | ) | |
571 | iproute2_out = subp.communicate()[1].splitlines()[0].split()[0] | |
572 | ||
573 | if "Usage" in str(iproute2_out): | |
574 | return True | |
575 | except Exception: | |
576 | pass | |
577 | return False | |
0414a764 | 578 | |
d6c755f2 | 579 | |
cc95fbd9 | 580 | def module_present_linux(module, load): |
f2d6ce41 CF |
581 | """ |
582 | Returns whether `module` is present. | |
583 | ||
584 | If `load` is true, it will try to load it via modprobe. | |
585 | """ | |
787e7624 | 586 | with open("/proc/modules", "r") as modules_file: |
587 | if module.replace("-", "_") in modules_file.read(): | |
f2d6ce41 | 588 | return True |
787e7624 | 589 | cmd = "/sbin/modprobe {}{}".format("" if load else "-n ", module) |
f2d6ce41 CF |
590 | if os.system(cmd) != 0: |
591 | return False | |
592 | else: | |
593 | return True | |
594 | ||
787e7624 | 595 | |
cc95fbd9 DS |
596 | def module_present_freebsd(module, load): |
597 | return True | |
598 | ||
787e7624 | 599 | |
cc95fbd9 DS |
600 | def module_present(module, load=True): |
601 | if sys.platform.startswith("linux"): | |
28440fd9 | 602 | return module_present_linux(module, load) |
cc95fbd9 | 603 | elif sys.platform.startswith("freebsd"): |
28440fd9 | 604 | return module_present_freebsd(module, load) |
cc95fbd9 | 605 | |
787e7624 | 606 | |
4190fe1e RZ |
607 | def version_cmp(v1, v2): |
608 | """ | |
609 | Compare two version strings and returns: | |
610 | ||
611 | * `-1`: if `v1` is less than `v2` | |
612 | * `0`: if `v1` is equal to `v2` | |
613 | * `1`: if `v1` is greater than `v2` | |
614 | ||
615 | Raises `ValueError` if versions are not well formated. | |
616 | """ | |
787e7624 | 617 | vregex = r"(?P<whole>\d+(\.(\d+))*)" |
4190fe1e RZ |
618 | v1m = re.match(vregex, v1) |
619 | v2m = re.match(vregex, v2) | |
620 | if v1m is None or v2m is None: | |
621 | raise ValueError("got a invalid version string") | |
622 | ||
623 | # Split values | |
787e7624 | 624 | v1g = v1m.group("whole").split(".") |
625 | v2g = v2m.group("whole").split(".") | |
4190fe1e RZ |
626 | |
627 | # Get the longest version string | |
628 | vnum = len(v1g) | |
629 | if len(v2g) > vnum: | |
630 | vnum = len(v2g) | |
631 | ||
632 | # Reverse list because we are going to pop the tail | |
633 | v1g.reverse() | |
634 | v2g.reverse() | |
635 | for _ in range(vnum): | |
636 | try: | |
637 | v1n = int(v1g.pop()) | |
638 | except IndexError: | |
639 | while v2g: | |
640 | v2n = int(v2g.pop()) | |
641 | if v2n > 0: | |
642 | return -1 | |
643 | break | |
644 | ||
645 | try: | |
646 | v2n = int(v2g.pop()) | |
647 | except IndexError: | |
648 | if v1n > 0: | |
649 | return 1 | |
650 | while v1g: | |
651 | v1n = int(v1g.pop()) | |
652 | if v1n > 0: | |
034237db | 653 | return 1 |
4190fe1e RZ |
654 | break |
655 | ||
656 | if v1n > v2n: | |
657 | return 1 | |
658 | if v1n < v2n: | |
659 | return -1 | |
660 | return 0 | |
661 | ||
787e7624 | 662 | |
f5612168 PG |
663 | def interface_set_status(node, ifacename, ifaceaction=False, vrf_name=None): |
664 | if ifaceaction: | |
787e7624 | 665 | str_ifaceaction = "no shutdown" |
f5612168 | 666 | else: |
787e7624 | 667 | str_ifaceaction = "shutdown" |
f5612168 | 668 | if vrf_name == None: |
787e7624 | 669 | cmd = 'vtysh -c "configure terminal" -c "interface {0}" -c "{1}"'.format( |
670 | ifacename, str_ifaceaction | |
671 | ) | |
f5612168 | 672 | else: |
9fa6ec14 | 673 | cmd = ( |
674 | 'vtysh -c "configure terminal" -c "interface {0} vrf {1}" -c "{2}"'.format( | |
675 | ifacename, vrf_name, str_ifaceaction | |
676 | ) | |
787e7624 | 677 | ) |
f5612168 PG |
678 | node.run(cmd) |
679 | ||
787e7624 | 680 | |
b220b3c8 PG |
681 | def ip4_route_zebra(node, vrf_name=None): |
682 | """ | |
683 | Gets an output of 'show ip route' command. It can be used | |
684 | with comparing the output to a reference | |
685 | """ | |
686 | if vrf_name == None: | |
787e7624 | 687 | tmp = node.vtysh_cmd("show ip route") |
b220b3c8 | 688 | else: |
787e7624 | 689 | tmp = node.vtysh_cmd("show ip route vrf {0}".format(vrf_name)) |
b220b3c8 | 690 | output = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", tmp) |
41077aa1 CF |
691 | |
692 | lines = output.splitlines() | |
693 | header_found = False | |
0eff5820 | 694 | while lines and (not lines[0].strip() or not header_found): |
5a3cf853 | 695 | if "o - offload failure" in lines[0]: |
41077aa1 CF |
696 | header_found = True |
697 | lines = lines[1:] | |
787e7624 | 698 | return "\n".join(lines) |
699 | ||
b220b3c8 | 700 | |
e394d9aa MS |
701 | def ip6_route_zebra(node, vrf_name=None): |
702 | """ | |
703 | Retrieves the output of 'show ipv6 route [vrf vrf_name]', then | |
704 | canonicalizes it by eliding link-locals. | |
705 | """ | |
706 | ||
707 | if vrf_name == None: | |
787e7624 | 708 | tmp = node.vtysh_cmd("show ipv6 route") |
e394d9aa | 709 | else: |
787e7624 | 710 | tmp = node.vtysh_cmd("show ipv6 route vrf {0}".format(vrf_name)) |
e394d9aa MS |
711 | |
712 | # Mask out timestamp | |
713 | output = re.sub(r" [0-2][0-9]:[0-5][0-9]:[0-5][0-9]", " XX:XX:XX", tmp) | |
714 | ||
715 | # Mask out the link-local addresses | |
787e7624 | 716 | output = re.sub(r"fe80::[^ ]+,", "fe80::XXXX:XXXX:XXXX:XXXX,", output) |
e394d9aa MS |
717 | |
718 | lines = output.splitlines() | |
719 | header_found = False | |
720 | while lines and (not lines[0].strip() or not header_found): | |
5a3cf853 | 721 | if "o - offload failure" in lines[0]: |
e394d9aa MS |
722 | header_found = True |
723 | lines = lines[1:] | |
724 | ||
787e7624 | 725 | return "\n".join(lines) |
e394d9aa MS |
726 | |
727 | ||
2f726781 MW |
728 | def proto_name_to_number(protocol): |
729 | return { | |
787e7624 | 730 | "bgp": "186", |
731 | "isis": "187", | |
732 | "ospf": "188", | |
733 | "rip": "189", | |
734 | "ripng": "190", | |
735 | "nhrp": "191", | |
736 | "eigrp": "192", | |
737 | "ldp": "193", | |
738 | "sharp": "194", | |
739 | "pbr": "195", | |
740 | "static": "196", | |
d1b5fa5b | 741 | "ospf6": "197", |
787e7624 | 742 | }.get( |
743 | protocol, protocol | |
744 | ) # default return same as input | |
2f726781 MW |
745 | |
746 | ||
99a7a912 RZ |
747 | def ip4_route(node): |
748 | """ | |
749 | Gets a structured return of the command 'ip route'. It can be used in | |
4563e204 | 750 | conjunction with json_cmp() to provide accurate assert explanations. |
99a7a912 RZ |
751 | |
752 | Return example: | |
753 | { | |
754 | '10.0.1.0/24': { | |
755 | 'dev': 'eth0', | |
756 | 'via': '172.16.0.1', | |
757 | 'proto': '188', | |
758 | }, | |
759 | '10.0.2.0/24': { | |
760 | 'dev': 'eth1', | |
761 | 'proto': 'kernel', | |
762 | } | |
763 | } | |
764 | """ | |
787e7624 | 765 | output = normalize_text(node.run("ip route")).splitlines() |
99a7a912 RZ |
766 | result = {} |
767 | for line in output: | |
787e7624 | 768 | columns = line.split(" ") |
99a7a912 RZ |
769 | route = result[columns[0]] = {} |
770 | prev = None | |
771 | for column in columns: | |
787e7624 | 772 | if prev == "dev": |
773 | route["dev"] = column | |
774 | if prev == "via": | |
775 | route["via"] = column | |
776 | if prev == "proto": | |
2f726781 | 777 | # translate protocol names back to numbers |
787e7624 | 778 | route["proto"] = proto_name_to_number(column) |
779 | if prev == "metric": | |
780 | route["metric"] = column | |
781 | if prev == "scope": | |
782 | route["scope"] = column | |
99a7a912 RZ |
783 | prev = column |
784 | ||
785 | return result | |
786 | ||
787e7624 | 787 | |
9375b5aa | 788 | def ip4_vrf_route(node): |
789 | """ | |
790 | Gets a structured return of the command 'ip route show vrf {0}-cust1'. | |
4563e204 | 791 | It can be used in conjunction with json_cmp() to provide accurate assert explanations. |
9375b5aa | 792 | |
793 | Return example: | |
794 | { | |
795 | '10.0.1.0/24': { | |
796 | 'dev': 'eth0', | |
797 | 'via': '172.16.0.1', | |
798 | 'proto': '188', | |
799 | }, | |
800 | '10.0.2.0/24': { | |
801 | 'dev': 'eth1', | |
802 | 'proto': 'kernel', | |
803 | } | |
804 | } | |
805 | """ | |
806 | output = normalize_text( | |
701a0192 | 807 | node.run("ip route show vrf {0}-cust1".format(node.name)) |
808 | ).splitlines() | |
9375b5aa | 809 | |
810 | result = {} | |
811 | for line in output: | |
812 | columns = line.split(" ") | |
813 | route = result[columns[0]] = {} | |
814 | prev = None | |
815 | for column in columns: | |
816 | if prev == "dev": | |
817 | route["dev"] = column | |
818 | if prev == "via": | |
819 | route["via"] = column | |
820 | if prev == "proto": | |
821 | # translate protocol names back to numbers | |
822 | route["proto"] = proto_name_to_number(column) | |
823 | if prev == "metric": | |
824 | route["metric"] = column | |
825 | if prev == "scope": | |
826 | route["scope"] = column | |
827 | prev = column | |
828 | ||
829 | return result | |
830 | ||
831 | ||
99a7a912 RZ |
832 | def ip6_route(node): |
833 | """ | |
834 | Gets a structured return of the command 'ip -6 route'. It can be used in | |
4563e204 | 835 | conjunction with json_cmp() to provide accurate assert explanations. |
99a7a912 RZ |
836 | |
837 | Return example: | |
838 | { | |
839 | '2001:db8:1::/64': { | |
840 | 'dev': 'eth0', | |
841 | 'proto': '188', | |
842 | }, | |
843 | '2001:db8:2::/64': { | |
844 | 'dev': 'eth1', | |
845 | 'proto': 'kernel', | |
846 | } | |
847 | } | |
848 | """ | |
787e7624 | 849 | output = normalize_text(node.run("ip -6 route")).splitlines() |
99a7a912 RZ |
850 | result = {} |
851 | for line in output: | |
787e7624 | 852 | columns = line.split(" ") |
99a7a912 RZ |
853 | route = result[columns[0]] = {} |
854 | prev = None | |
855 | for column in columns: | |
787e7624 | 856 | if prev == "dev": |
857 | route["dev"] = column | |
858 | if prev == "via": | |
859 | route["via"] = column | |
860 | if prev == "proto": | |
2f726781 | 861 | # translate protocol names back to numbers |
787e7624 | 862 | route["proto"] = proto_name_to_number(column) |
863 | if prev == "metric": | |
864 | route["metric"] = column | |
865 | if prev == "pref": | |
866 | route["pref"] = column | |
99a7a912 RZ |
867 | prev = column |
868 | ||
869 | return result | |
870 | ||
787e7624 | 871 | |
9375b5aa | 872 | def ip6_vrf_route(node): |
873 | """ | |
874 | Gets a structured return of the command 'ip -6 route show vrf {0}-cust1'. | |
4563e204 | 875 | It can be used in conjunction with json_cmp() to provide accurate assert explanations. |
9375b5aa | 876 | |
877 | Return example: | |
878 | { | |
879 | '2001:db8:1::/64': { | |
880 | 'dev': 'eth0', | |
881 | 'proto': '188', | |
882 | }, | |
883 | '2001:db8:2::/64': { | |
884 | 'dev': 'eth1', | |
885 | 'proto': 'kernel', | |
886 | } | |
887 | } | |
888 | """ | |
889 | output = normalize_text( | |
701a0192 | 890 | node.run("ip -6 route show vrf {0}-cust1".format(node.name)) |
891 | ).splitlines() | |
9375b5aa | 892 | result = {} |
893 | for line in output: | |
894 | columns = line.split(" ") | |
895 | route = result[columns[0]] = {} | |
896 | prev = None | |
897 | for column in columns: | |
898 | if prev == "dev": | |
899 | route["dev"] = column | |
900 | if prev == "via": | |
901 | route["via"] = column | |
902 | if prev == "proto": | |
903 | # translate protocol names back to numbers | |
904 | route["proto"] = proto_name_to_number(column) | |
905 | if prev == "metric": | |
906 | route["metric"] = column | |
907 | if prev == "pref": | |
908 | route["pref"] = column | |
909 | prev = column | |
910 | ||
911 | return result | |
912 | ||
913 | ||
9b7decf2 JU |
914 | def ip_rules(node): |
915 | """ | |
916 | Gets a structured return of the command 'ip rule'. It can be used in | |
4563e204 | 917 | conjunction with json_cmp() to provide accurate assert explanations. |
9b7decf2 JU |
918 | |
919 | Return example: | |
920 | [ | |
921 | { | |
922 | "pref": "0" | |
923 | "from": "all" | |
924 | }, | |
925 | { | |
926 | "pref": "32766" | |
927 | "from": "all" | |
928 | }, | |
929 | { | |
930 | "to": "3.4.5.0/24", | |
931 | "iif": "r1-eth2", | |
932 | "pref": "304", | |
933 | "from": "1.2.0.0/16", | |
934 | "proto": "zebra" | |
935 | } | |
936 | ] | |
937 | """ | |
938 | output = normalize_text(node.run("ip rule")).splitlines() | |
939 | result = [] | |
940 | for line in output: | |
941 | columns = line.split(" ") | |
942 | ||
943 | route = {} | |
944 | # remove last character, since it is ':' | |
945 | pref = columns[0][:-1] | |
946 | route["pref"] = pref | |
947 | prev = None | |
948 | for column in columns: | |
949 | if prev == "from": | |
950 | route["from"] = column | |
951 | if prev == "to": | |
952 | route["to"] = column | |
953 | if prev == "proto": | |
954 | route["proto"] = column | |
955 | if prev == "iif": | |
956 | route["iif"] = column | |
957 | if prev == "fwmark": | |
958 | route["fwmark"] = column | |
959 | prev = column | |
960 | ||
961 | result.append(route) | |
962 | return result | |
963 | ||
964 | ||
570f25d8 RZ |
965 | def sleep(amount, reason=None): |
966 | """ | |
967 | Sleep wrapper that registers in the log the amount of sleep | |
968 | """ | |
969 | if reason is None: | |
787e7624 | 970 | logger.info("Sleeping for {} seconds".format(amount)) |
570f25d8 | 971 | else: |
787e7624 | 972 | logger.info(reason + " ({} seconds)".format(amount)) |
570f25d8 RZ |
973 | |
974 | time.sleep(amount) | |
975 | ||
787e7624 | 976 | |
be2656ed | 977 | def checkAddressSanitizerError(output, router, component, logdir=""): |
4942f298 MW |
978 | "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise" |
979 | ||
be2656ed | 980 | def processAddressSanitizerError(asanErrorRe, output, router, component): |
787e7624 | 981 | sys.stderr.write( |
982 | "%s: %s triggered an exception by AddressSanitizer\n" % (router, component) | |
983 | ) | |
4942f298 | 984 | # Sanitizer Error found in log |
be2656ed | 985 | pidMark = asanErrorRe.group(1) |
6ee4440e | 986 | addressSanitizerLog = re.search( |
787e7624 | 987 | "%s(.*)%s" % (pidMark, pidMark), output, re.DOTALL |
988 | ) | |
6ee4440e | 989 | if addressSanitizerLog: |
be2656ed | 990 | # Find Calling Test. Could be multiple steps back |
9fa6ec14 | 991 | testframe = sys._current_frames().values()[0] |
992 | level = 0 | |
be2656ed | 993 | while level < 10: |
9fa6ec14 | 994 | test = os.path.splitext( |
995 | os.path.basename(testframe.f_globals["__file__"]) | |
996 | )[0] | |
be2656ed MW |
997 | if (test != "topotest") and (test != "topogen"): |
998 | # Found the calling test | |
9fa6ec14 | 999 | callingTest = os.path.basename(testframe.f_globals["__file__"]) |
be2656ed | 1000 | break |
9fa6ec14 | 1001 | level = level + 1 |
1002 | testframe = testframe.f_back | |
1003 | if level >= 10: | |
be2656ed | 1004 | # somehow couldn't find the test script. |
9fa6ec14 | 1005 | callingTest = "unknownTest" |
be2656ed MW |
1006 | # |
1007 | # Now finding Calling Procedure | |
9fa6ec14 | 1008 | level = 0 |
be2656ed | 1009 | while level < 20: |
9fa6ec14 | 1010 | callingProc = sys._getframe(level).f_code.co_name |
1011 | if ( | |
1012 | (callingProc != "processAddressSanitizerError") | |
1013 | and (callingProc != "checkAddressSanitizerError") | |
1014 | and (callingProc != "checkRouterCores") | |
1015 | and (callingProc != "stopRouter") | |
9fa6ec14 | 1016 | and (callingProc != "stop") |
1017 | and (callingProc != "stop_topology") | |
1018 | and (callingProc != "checkRouterRunning") | |
1019 | and (callingProc != "check_router_running") | |
1020 | and (callingProc != "routers_have_failure") | |
1021 | ): | |
be2656ed MW |
1022 | # Found the calling test |
1023 | break | |
9fa6ec14 | 1024 | level = level + 1 |
1025 | if level >= 20: | |
be2656ed | 1026 | # something wrong - couldn't found the calling test function |
9fa6ec14 | 1027 | callingProc = "unknownProc" |
4942f298 | 1028 | with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile: |
be2656ed MW |
1029 | sys.stderr.write( |
1030 | "AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" | |
1031 | % (callingTest, callingProc, router) | |
1032 | ) | |
787e7624 | 1033 | sys.stderr.write( |
6ee4440e | 1034 | "\n".join(addressSanitizerLog.group(1).splitlines()) + "\n" |
787e7624 | 1035 | ) |
be2656ed | 1036 | addrSanFile.write("## Error: %s\n\n" % asanErrorRe.group(2)) |
787e7624 | 1037 | addrSanFile.write( |
1038 | "### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" | |
1039 | % (callingTest, callingProc, router) | |
1040 | ) | |
1041 | addrSanFile.write( | |
1042 | " " | |
6ee4440e | 1043 | + "\n ".join(addressSanitizerLog.group(1).splitlines()) |
787e7624 | 1044 | + "\n" |
1045 | ) | |
4942f298 | 1046 | addrSanFile.write("\n---------------\n") |
be2656ed MW |
1047 | return |
1048 | ||
6ee4440e | 1049 | addressSanitizerError = re.search( |
49581587 | 1050 | r"(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", output |
be2656ed | 1051 | ) |
6ee4440e MS |
1052 | if addressSanitizerError: |
1053 | processAddressSanitizerError(addressSanitizerError, output, router, component) | |
4942f298 | 1054 | return True |
be2656ed MW |
1055 | |
1056 | # No Address Sanitizer Error in Output. Now check for AddressSanitizer daemon file | |
1057 | if logdir: | |
449e2555 | 1058 | filepattern = logdir + "/" + router + ".asan." + component + ".*" |
9fa6ec14 | 1059 | logger.debug( |
1060 | "Log check for %s on %s, pattern %s\n" % (component, router, filepattern) | |
1061 | ) | |
be2656ed MW |
1062 | for file in glob.glob(filepattern): |
1063 | with open(file, "r") as asanErrorFile: | |
9fa6ec14 | 1064 | asanError = asanErrorFile.read() |
6ee4440e | 1065 | addressSanitizerError = re.search( |
49581587 | 1066 | r"(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ", asanError |
9fa6ec14 | 1067 | ) |
6ee4440e | 1068 | if addressSanitizerError: |
9fa6ec14 | 1069 | processAddressSanitizerError( |
1070 | addressSanitizerError, asanError, router, component | |
1071 | ) | |
be2656ed | 1072 | return True |
6c131bd3 | 1073 | return False |
4942f298 | 1074 | |
787e7624 | 1075 | |
49581587 CH |
1076 | def _sysctl_atleast(commander, variable, min_value): |
1077 | if isinstance(min_value, tuple): | |
1078 | min_value = list(min_value) | |
1079 | is_list = isinstance(min_value, list) | |
594b1259 | 1080 | |
49581587 CH |
1081 | sval = commander.cmd_raises("sysctl -n " + variable).strip() |
1082 | if is_list: | |
1083 | cur_val = [int(x) for x in sval.split()] | |
1084 | else: | |
1085 | cur_val = int(sval) | |
1086 | ||
1087 | set_value = False | |
1088 | if is_list: | |
1089 | for i, v in enumerate(cur_val): | |
1090 | if v < min_value[i]: | |
1091 | set_value = True | |
1092 | else: | |
1093 | min_value[i] = v | |
1094 | else: | |
1095 | if cur_val < min_value: | |
1096 | set_value = True | |
1097 | if set_value: | |
1098 | if is_list: | |
1099 | valstr = " ".join([str(x) for x in min_value]) | |
1100 | else: | |
1101 | valstr = str(min_value) | |
e8f7a22f | 1102 | logger.debug("Increasing sysctl %s from %s to %s", variable, cur_val, valstr) |
8aba44e3 | 1103 | commander.cmd_raises('sysctl -w {}="{}"'.format(variable, valstr)) |
594b1259 | 1104 | |
787e7624 | 1105 | |
49581587 CH |
1106 | def _sysctl_assure(commander, variable, value): |
1107 | if isinstance(value, tuple): | |
1108 | value = list(value) | |
1109 | is_list = isinstance(value, list) | |
797e8dcf | 1110 | |
49581587 CH |
1111 | sval = commander.cmd_raises("sysctl -n " + variable).strip() |
1112 | if is_list: | |
1113 | cur_val = [int(x) for x in sval.split()] | |
1114 | else: | |
1115 | cur_val = sval | |
797e8dcf | 1116 | |
49581587 CH |
1117 | set_value = False |
1118 | if is_list: | |
1119 | for i, v in enumerate(cur_val): | |
1120 | if v != value[i]: | |
1121 | set_value = True | |
1122 | else: | |
1123 | value[i] = v | |
1124 | else: | |
1125 | if cur_val != str(value): | |
1126 | set_value = True | |
1127 | ||
1128 | if set_value: | |
1129 | if is_list: | |
1130 | valstr = " ".join([str(x) for x in value]) | |
1131 | else: | |
1132 | valstr = str(value) | |
e8f7a22f | 1133 | logger.debug("Changing sysctl %s from %s to %s", variable, cur_val, valstr) |
a53c08bc | 1134 | commander.cmd_raises('sysctl -w {}="{}"\n'.format(variable, valstr)) |
49581587 CH |
1135 | |
1136 | ||
1137 | def sysctl_atleast(commander, variable, min_value, raises=False): | |
1138 | try: | |
1139 | if commander is None: | |
1140 | commander = micronet.Commander("topotest") | |
1141 | return _sysctl_atleast(commander, variable, min_value) | |
1142 | except subprocess.CalledProcessError as error: | |
1143 | logger.warning( | |
1144 | "%s: Failed to assure sysctl min value %s = %s", | |
a53c08bc CH |
1145 | commander, |
1146 | variable, | |
1147 | min_value, | |
49581587 CH |
1148 | ) |
1149 | if raises: | |
1150 | raise | |
797e8dcf | 1151 | |
787e7624 | 1152 | |
49581587 CH |
1153 | def sysctl_assure(commander, variable, value, raises=False): |
1154 | try: | |
1155 | if commander is None: | |
1156 | commander = micronet.Commander("topotest") | |
1157 | return _sysctl_assure(commander, variable, value) | |
1158 | except subprocess.CalledProcessError as error: | |
1159 | logger.warning( | |
1160 | "%s: Failed to assure sysctl value %s = %s", | |
a53c08bc CH |
1161 | commander, |
1162 | variable, | |
1163 | value, | |
1164 | exc_info=True, | |
49581587 CH |
1165 | ) |
1166 | if raises: | |
1167 | raise | |
1168 | ||
1169 | ||
1170 | def rlimit_atleast(rname, min_value, raises=False): | |
1171 | try: | |
1172 | cval = resource.getrlimit(rname) | |
1173 | soft, hard = cval | |
1174 | if soft < min_value: | |
1175 | nval = (min_value, hard if min_value < hard else min_value) | |
e8f7a22f | 1176 | logger.debug("Increasing rlimit %s from %s to %s", rname, cval, nval) |
49581587 CH |
1177 | resource.setrlimit(rname, nval) |
1178 | except subprocess.CalledProcessError as error: | |
1179 | logger.warning( | |
a53c08bc | 1180 | "Failed to assure rlimit [%s] = %s", rname, min_value, exc_info=True |
49581587 CH |
1181 | ) |
1182 | if raises: | |
1183 | raise | |
1184 | ||
1185 | ||
1186 | def fix_netns_limits(ns): | |
49581587 | 1187 | # Maximum read and write socket buffer sizes |
e6079f4f CH |
1188 | sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2**20]) |
1189 | sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2**20]) | |
49581587 CH |
1190 | |
1191 | sysctl_assure(ns, "net.ipv4.conf.all.rp_filter", 0) | |
1192 | sysctl_assure(ns, "net.ipv4.conf.default.rp_filter", 0) | |
1193 | sysctl_assure(ns, "net.ipv4.conf.lo.rp_filter", 0) | |
1194 | ||
1195 | sysctl_assure(ns, "net.ipv4.conf.all.forwarding", 1) | |
1196 | sysctl_assure(ns, "net.ipv4.conf.default.forwarding", 1) | |
1197 | ||
1198 | # XXX if things fail look here as this wasn't done previously | |
1199 | sysctl_assure(ns, "net.ipv6.conf.all.forwarding", 1) | |
1200 | sysctl_assure(ns, "net.ipv6.conf.default.forwarding", 1) | |
1201 | ||
1202 | # ARP | |
1203 | sysctl_assure(ns, "net.ipv4.conf.default.arp_announce", 2) | |
1204 | sysctl_assure(ns, "net.ipv4.conf.default.arp_notify", 1) | |
1205 | # Setting this to 1 breaks topotests that rely on lo addresses being proxy arp'd for | |
1206 | sysctl_assure(ns, "net.ipv4.conf.default.arp_ignore", 0) | |
1207 | sysctl_assure(ns, "net.ipv4.conf.all.arp_announce", 2) | |
1208 | sysctl_assure(ns, "net.ipv4.conf.all.arp_notify", 1) | |
1209 | # Setting this to 1 breaks topotests that rely on lo addresses being proxy arp'd for | |
1210 | sysctl_assure(ns, "net.ipv4.conf.all.arp_ignore", 0) | |
1211 | ||
1212 | sysctl_assure(ns, "net.ipv4.icmp_errors_use_inbound_ifaddr", 1) | |
1213 | ||
1214 | # Keep ipv6 permanent addresses on an admin down | |
1215 | sysctl_assure(ns, "net.ipv6.conf.all.keep_addr_on_down", 1) | |
1216 | if version_cmp(platform.release(), "4.20") >= 0: | |
1217 | sysctl_assure(ns, "net.ipv6.route.skip_notify_on_dev_down", 1) | |
1218 | ||
1219 | sysctl_assure(ns, "net.ipv4.conf.all.ignore_routes_with_linkdown", 1) | |
1220 | sysctl_assure(ns, "net.ipv6.conf.all.ignore_routes_with_linkdown", 1) | |
1221 | ||
1222 | # igmp | |
1223 | sysctl_atleast(ns, "net.ipv4.igmp_max_memberships", 1000) | |
1224 | ||
1225 | # Use neigh information on selection of nexthop for multipath hops | |
1226 | sysctl_assure(ns, "net.ipv4.fib_multipath_use_neigh", 1) | |
1227 | ||
1228 | ||
1229 | def fix_host_limits(): | |
1230 | """Increase system limits.""" | |
1231 | ||
a53c08bc CH |
1232 | rlimit_atleast(resource.RLIMIT_NPROC, 8 * 1024) |
1233 | rlimit_atleast(resource.RLIMIT_NOFILE, 16 * 1024) | |
1234 | sysctl_atleast(None, "fs.file-max", 16 * 1024) | |
1235 | sysctl_atleast(None, "kernel.pty.max", 16 * 1024) | |
49581587 CH |
1236 | |
1237 | # Enable coredumps | |
1238 | # Original on ubuntu 17.x, but apport won't save as in namespace | |
1239 | # |/usr/share/apport/apport %p %s %c %d %P | |
1240 | sysctl_assure(None, "kernel.core_pattern", "%e_core-sig_%s-pid_%p.dmp") | |
1241 | sysctl_assure(None, "kernel.core_uses_pid", 1) | |
1242 | sysctl_assure(None, "fs.suid_dumpable", 1) | |
1243 | ||
1244 | # Maximum connection backlog | |
a53c08bc | 1245 | sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024) |
49581587 CH |
1246 | |
1247 | # Maximum read and write socket buffer sizes | |
e6079f4f CH |
1248 | sysctl_atleast(None, "net.core.rmem_max", 16 * 2**20) |
1249 | sysctl_atleast(None, "net.core.wmem_max", 16 * 2**20) | |
49581587 CH |
1250 | |
1251 | # Garbage Collection Settings for ARP and Neighbors | |
a53c08bc CH |
1252 | sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024) |
1253 | sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh3", 8 * 1024) | |
1254 | sysctl_atleast(None, "net.ipv6.neigh.default.gc_thresh2", 4 * 1024) | |
1255 | sysctl_atleast(None, "net.ipv6.neigh.default.gc_thresh3", 8 * 1024) | |
49581587 | 1256 | # Hold entries for 10 minutes |
a53c08bc CH |
1257 | sysctl_assure(None, "net.ipv4.neigh.default.base_reachable_time_ms", 10 * 60 * 1000) |
1258 | sysctl_assure(None, "net.ipv6.neigh.default.base_reachable_time_ms", 10 * 60 * 1000) | |
49581587 CH |
1259 | |
1260 | # igmp | |
1261 | sysctl_assure(None, "net.ipv4.neigh.default.mcast_solicit", 10) | |
1262 | ||
1263 | # MLD | |
1264 | sysctl_atleast(None, "net.ipv6.mld_max_msf", 512) | |
1265 | ||
1266 | # Increase routing table size to 128K | |
a53c08bc CH |
1267 | sysctl_atleast(None, "net.ipv4.route.max_size", 128 * 1024) |
1268 | sysctl_atleast(None, "net.ipv6.route.max_size", 128 * 1024) | |
49581587 CH |
1269 | |
1270 | ||
1271 | def setup_node_tmpdir(logdir, name): | |
1272 | # Cleanup old log, valgrind, and core files. | |
1273 | subprocess.check_call( | |
449e2555 CH |
1274 | "rm -rf {0}/{1}.valgrind.* {0}/{1}.asan.* {0}/{1}/".format(logdir, name), |
1275 | shell=True, | |
49581587 CH |
1276 | ) |
1277 | ||
1278 | # Setup the per node directory. | |
1279 | nodelogdir = "{}/{}".format(logdir, name) | |
a53c08bc CH |
1280 | subprocess.check_call( |
1281 | "mkdir -p {0} && chmod 1777 {0}".format(nodelogdir), shell=True | |
1282 | ) | |
49581587 CH |
1283 | logfile = "{0}/{1}.log".format(logdir, name) |
1284 | return logfile | |
797e8dcf | 1285 | |
594b1259 MW |
1286 | |
1287 | class Router(Node): | |
622c4996 | 1288 | "A Node with IPv4/IPv6 forwarding enabled" |
594b1259 | 1289 | |
60e03778 | 1290 | def __init__(self, name, *posargs, **params): |
04ce2b97 RZ |
1291 | # Backward compatibility: |
1292 | # Load configuration defaults like topogen. | |
787e7624 | 1293 | self.config_defaults = configparser.ConfigParser( |
701a0192 | 1294 | defaults={ |
787e7624 | 1295 | "verbosity": "info", |
1296 | "frrdir": "/usr/lib/frr", | |
787e7624 | 1297 | "routertype": "frr", |
11761ab0 | 1298 | "memleak_path": "", |
787e7624 | 1299 | } |
1300 | ) | |
49581587 | 1301 | |
04ce2b97 | 1302 | self.config_defaults.read( |
787e7624 | 1303 | os.path.join(os.path.dirname(os.path.realpath(__file__)), "../pytest.ini") |
04ce2b97 RZ |
1304 | ) |
1305 | ||
e6079f4f CH |
1306 | self.perf_daemons = {} |
1307 | ||
0d5e41c6 RZ |
1308 | # If this topology is using old API and doesn't have logdir |
1309 | # specified, then attempt to generate an unique logdir. | |
49581587 | 1310 | self.logdir = params.get("logdir") |
0d5e41c6 | 1311 | if self.logdir is None: |
249ac6f0 | 1312 | self.logdir = get_logs_path(g_pytest_config.getoption("--rundir")) |
49581587 CH |
1313 | |
1314 | if not params.get("logger"): | |
1315 | # If logger is present topogen has already set this up | |
1316 | logfile = setup_node_tmpdir(self.logdir, name) | |
1317 | l = topolog.get_logger(name, log_level="debug", target=logfile) | |
1318 | params["logger"] = l | |
1319 | ||
60e03778 | 1320 | super(Router, self).__init__(name, *posargs, **params) |
0d5e41c6 | 1321 | |
2ab85530 | 1322 | self.daemondir = None |
447f2d5a | 1323 | self.hasmpls = False |
787e7624 | 1324 | self.routertype = "frr" |
a4b4bb50 | 1325 | self.unified_config = None |
787e7624 | 1326 | self.daemons = { |
1327 | "zebra": 0, | |
1328 | "ripd": 0, | |
1329 | "ripngd": 0, | |
1330 | "ospfd": 0, | |
1331 | "ospf6d": 0, | |
1332 | "isisd": 0, | |
1333 | "bgpd": 0, | |
1334 | "pimd": 0, | |
e13f9c4f | 1335 | "pim6d": 0, |
787e7624 | 1336 | "ldpd": 0, |
1337 | "eigrpd": 0, | |
1338 | "nhrpd": 0, | |
1339 | "staticd": 0, | |
1340 | "bfdd": 0, | |
1341 | "sharpd": 0, | |
a0764a36 | 1342 | "babeld": 0, |
223f87f4 | 1343 | "pbrd": 0, |
92be50e6 BC |
1344 | "pathd": 0, |
1345 | "snmpd": 0, | |
f637ac01 | 1346 | "mgmtd": 0, |
787e7624 | 1347 | } |
1348 | self.daemons_options = {"zebra": ""} | |
2a59a86b | 1349 | self.reportCores = True |
fb80b81b | 1350 | self.version = None |
2ab85530 | 1351 | |
49581587 | 1352 | self.ns_cmd = "sudo nsenter -a -t {} ".format(self.pid) |
0ba1d257 CH |
1353 | try: |
1354 | # Allow escaping from running inside docker | |
1355 | cgroup = open("/proc/1/cgroup").read() | |
1356 | m = re.search("[0-9]+:cpuset:/docker/([a-f0-9]+)", cgroup) | |
1357 | if m: | |
1358 | self.ns_cmd = "docker exec -it {} ".format(m.group(1)) + self.ns_cmd | |
1359 | except IOError: | |
1360 | pass | |
1361 | else: | |
1362 | logger.debug("CMD to enter {}: {}".format(self.name, self.ns_cmd)) | |
1363 | ||
edd2bdf6 RZ |
1364 | def _config_frr(self, **params): |
1365 | "Configure FRR binaries" | |
787e7624 | 1366 | self.daemondir = params.get("frrdir") |
edd2bdf6 | 1367 | if self.daemondir is None: |
787e7624 | 1368 | self.daemondir = self.config_defaults.get("topogen", "frrdir") |
edd2bdf6 | 1369 | |
787e7624 | 1370 | zebra_path = os.path.join(self.daemondir, "zebra") |
edd2bdf6 RZ |
1371 | if not os.path.isfile(zebra_path): |
1372 | raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path)) | |
1373 | ||
f637ac01 | 1374 | mgmtd_path = os.path.join(self.daemondir, "mgmtd") |
1375 | if not os.path.isfile(mgmtd_path): | |
1376 | raise Exception("FRR MGMTD binary doesn't exist at {}".format(mgmtd_path)) | |
1377 | ||
2ab85530 RZ |
1378 | # pylint: disable=W0221 |
1379 | # Some params are only meaningful for the parent class. | |
60e03778 CH |
1380 | def config_host(self, **params): |
1381 | super(Router, self).config_host(**params) | |
594b1259 | 1382 | |
2ab85530 | 1383 | # User did not specify the daemons directory, try to autodetect it. |
787e7624 | 1384 | self.daemondir = params.get("daemondir") |
2ab85530 | 1385 | if self.daemondir is None: |
787e7624 | 1386 | self.routertype = params.get( |
1387 | "routertype", self.config_defaults.get("topogen", "routertype") | |
1388 | ) | |
622c4996 | 1389 | self._config_frr(**params) |
594b1259 | 1390 | else: |
2ab85530 | 1391 | # Test the provided path |
787e7624 | 1392 | zpath = os.path.join(self.daemondir, "zebra") |
2ab85530 | 1393 | if not os.path.isfile(zpath): |
787e7624 | 1394 | raise Exception("No zebra binary found in {}".format(zpath)) |
f637ac01 | 1395 | |
1396 | cpath = os.path.join(self.daemondir, "mgmtd") | |
1397 | if not os.path.isfile(zpath): | |
1398 | raise Exception("No MGMTD binary found in {}".format(cpath)) | |
2ab85530 | 1399 | # Allow user to specify routertype when the path was specified. |
787e7624 | 1400 | if params.get("routertype") is not None: |
1401 | self.routertype = params.get("routertype") | |
2ab85530 | 1402 | |
594b1259 | 1403 | # Set ownership of config files |
787e7624 | 1404 | self.cmd("chown {0}:{0}vty /etc/{0}".format(self.routertype)) |
2ab85530 | 1405 | |
594b1259 | 1406 | def terminate(self): |
cd79342c | 1407 | # Stop running FRR daemons |
99561211 | 1408 | self.stopRouter() |
594b1259 | 1409 | super(Router, self).terminate() |
49581587 | 1410 | os.system("chmod -R go+rw " + self.logdir) |
b0f0d980 | 1411 | |
cf865d1b | 1412 | # Return count of running daemons |
f033a78a DL |
1413 | def listDaemons(self): |
1414 | ret = [] | |
a53c08bc CH |
1415 | rc, stdout, _ = self.cmd_status( |
1416 | "ls -1 /var/run/%s/*.pid" % self.routertype, warn=False | |
1417 | ) | |
49581587 CH |
1418 | if rc: |
1419 | return ret | |
1420 | for d in stdout.strip().split("\n"): | |
1421 | pidfile = d.strip() | |
1422 | try: | |
1423 | pid = int(self.cmd_raises("cat %s" % pidfile, warn=False).strip()) | |
1424 | name = os.path.basename(pidfile[:-4]) | |
1425 | ||
1426 | # probably not compatible with bsd. | |
1427 | rc, _, _ = self.cmd_status("test -d /proc/{}".format(pid), warn=False) | |
1428 | if rc: | |
a53c08bc CH |
1429 | logger.warning( |
1430 | "%s: %s exited leaving pidfile %s (%s)", | |
1431 | self.name, | |
1432 | name, | |
1433 | pidfile, | |
1434 | pid, | |
1435 | ) | |
49581587 CH |
1436 | self.cmd("rm -- " + pidfile) |
1437 | else: | |
1438 | ret.append((name, pid)) | |
1439 | except (subprocess.CalledProcessError, ValueError): | |
1440 | pass | |
f033a78a | 1441 | return ret |
cf865d1b | 1442 | |
49581587 | 1443 | def stopRouter(self, assertOnError=True, minErrorVersion="5.1"): |
cf865d1b | 1444 | # Stop Running FRR Daemons |
49581587 CH |
1445 | running = self.listDaemons() |
1446 | if not running: | |
1447 | return "" | |
1448 | ||
1449 | logger.info("%s: stopping %s", self.name, ", ".join([x[0] for x in running])) | |
1450 | for name, pid in running: | |
e8f7a22f | 1451 | logger.debug("{}: sending SIGTERM to {}".format(self.name, name)) |
49581587 CH |
1452 | try: |
1453 | os.kill(pid, signal.SIGTERM) | |
1454 | except OSError as err: | |
e8f7a22f | 1455 | logger.debug( |
a53c08bc CH |
1456 | "%s: could not kill %s (%s): %s", self.name, name, pid, str(err) |
1457 | ) | |
49581587 CH |
1458 | |
1459 | running = self.listDaemons() | |
1460 | if running: | |
e9a59a2a | 1461 | for _ in range(0, 30): |
701a0192 | 1462 | sleep( |
49581587 | 1463 | 0.5, |
701a0192 | 1464 | "{}: waiting for daemons stopping: {}".format( |
49581587 | 1465 | self.name, ", ".join([x[0] for x in running]) |
701a0192 | 1466 | ), |
1467 | ) | |
f033a78a | 1468 | running = self.listDaemons() |
49581587 CH |
1469 | if not running: |
1470 | break | |
f033a78a | 1471 | |
c6686550 LB |
1472 | if running: |
1473 | logger.warning( | |
1474 | "%s: sending SIGBUS to: %s", | |
1475 | self.name, | |
1476 | ", ".join([x[0] for x in running]), | |
1477 | ) | |
1478 | for name, pid in running: | |
1479 | pidfile = "/var/run/{}/{}.pid".format(self.routertype, name) | |
1480 | logger.info("%s: killing %s", self.name, name) | |
1481 | self.cmd("kill -SIGBUS %d" % pid) | |
1482 | self.cmd("rm -- " + pidfile) | |
1483 | ||
1484 | sleep( | |
1485 | 0.5, | |
1486 | "%s: waiting for daemons to exit/core after initial SIGBUS" % self.name, | |
1487 | ) | |
f033a78a DL |
1488 | |
1489 | errors = self.checkRouterCores(reportOnce=True) | |
1490 | if self.checkRouterVersion("<", minErrorVersion): | |
1491 | # ignore errors in old versions | |
1492 | errors = "" | |
49581587 | 1493 | if assertOnError and (errors is not None) and len(errors) > 0: |
f033a78a | 1494 | assert "Errors found - details follow:" == 0, errors |
83c26937 | 1495 | return errors |
f76774ec | 1496 | |
594b1259 MW |
1497 | def removeIPs(self): |
1498 | for interface in self.intfNames(): | |
49581587 | 1499 | try: |
eb9e801f CH |
1500 | self.intf_ip_cmd(interface, "ip -4 address flush " + interface) |
1501 | self.intf_ip_cmd( | |
1502 | interface, "ip -6 address flush " + interface + " scope global" | |
1503 | ) | |
49581587 CH |
1504 | except Exception as ex: |
1505 | logger.error("%s can't remove IPs %s", self, str(ex)) | |
249ac6f0 | 1506 | # breakpoint() |
49581587 | 1507 | # assert False, "can't remove IPs %s" % str(ex) |
8dd5077d PG |
1508 | |
1509 | def checkCapability(self, daemon, param): | |
1510 | if param is not None: | |
1511 | daemon_path = os.path.join(self.daemondir, daemon) | |
787e7624 | 1512 | daemon_search_option = param.replace("-", "") |
1513 | output = self.cmd( | |
1514 | "{0} -h | grep {1}".format(daemon_path, daemon_search_option) | |
1515 | ) | |
8dd5077d PG |
1516 | if daemon_search_option not in output: |
1517 | return False | |
1518 | return True | |
1519 | ||
1520 | def loadConf(self, daemon, source=None, param=None): | |
02547745 CH |
1521 | """Enabled and set config for a daemon. |
1522 | ||
1523 | Arranges for loading of daemon configuration from the specified source. Possible | |
1524 | `source` values are `None` for an empty config file, a path name which is used | |
1525 | directly, or a file name with no path components which is first looked for | |
1526 | directly and then looked for under a sub-directory named after router. | |
1527 | """ | |
1528 | ||
49581587 | 1529 | # Unfortunately this API allowsfor source to not exist for any and all routers. |
27c6bfc2 CH |
1530 | source_was_none = source is None |
1531 | if source_was_none: | |
f3525b0b CH |
1532 | source = f"{daemon}.conf" |
1533 | ||
27c6bfc2 | 1534 | # "" to avoid loading a default config which is present in router dir |
02547745 CH |
1535 | if source: |
1536 | head, tail = os.path.split(source) | |
1537 | if not head and not self.path_exists(tail): | |
1538 | script_dir = os.environ["PYTEST_TOPOTEST_SCRIPTDIR"] | |
1539 | router_relative = os.path.join(script_dir, self.name, tail) | |
1540 | if self.path_exists(router_relative): | |
1541 | source = router_relative | |
e8f7a22f | 1542 | self.logger.debug( |
02547745 CH |
1543 | "using router relative configuration: {}".format(source) |
1544 | ) | |
49581587 | 1545 | |
594b1259 | 1546 | # print "Daemons before:", self.daemons |
a4b4bb50 JAG |
1547 | if daemon in self.daemons.keys() or daemon == "frr": |
1548 | if daemon == "frr": | |
1549 | self.unified_config = 1 | |
1550 | else: | |
1551 | self.daemons[daemon] = 1 | |
8dd5077d PG |
1552 | if param is not None: |
1553 | self.daemons_options[daemon] = param | |
49581587 | 1554 | conf_file = "/etc/{}/{}.conf".format(self.routertype, daemon) |
27c6bfc2 CH |
1555 | if source and not os.path.exists(source): |
1556 | logger.warn( | |
1557 | "missing config '%s' for '%s' creating empty file '%s'", | |
1558 | self.name, | |
1559 | source, | |
1560 | conf_file, | |
1561 | ) | |
a4b4bb50 JAG |
1562 | if daemon == "frr" or not self.unified_config: |
1563 | self.cmd_raises("rm -f " + conf_file) | |
1564 | self.cmd_raises("touch " + conf_file) | |
27c6bfc2 CH |
1565 | self.cmd_raises( |
1566 | "chown {0}:{0} {1}".format(self.routertype, conf_file) | |
1567 | ) | |
1568 | self.cmd_raises("chmod 664 {}".format(conf_file)) | |
1569 | elif source: | |
f637ac01 | 1570 | # copy zebra.conf to mgmtd folder, which can be used during startup |
27c6bfc2 | 1571 | if daemon == "zebra" and not self.unified_config: |
d6c755f2 | 1572 | conf_file_mgmt = "/etc/{}/{}.conf".format(self.routertype, "mgmtd") |
27c6bfc2 CH |
1573 | logger.debug( |
1574 | "copying '%s' as '%s' on '%s'", | |
1575 | source, | |
1576 | conf_file_mgmt, | |
1577 | self.name, | |
1578 | ) | |
f637ac01 | 1579 | self.cmd_raises("cp {} {}".format(source, conf_file_mgmt)) |
27c6bfc2 CH |
1580 | self.cmd_raises( |
1581 | "chown {0}:{0} {1}".format(self.routertype, conf_file_mgmt) | |
1582 | ) | |
1583 | self.cmd_raises("chmod 664 {}".format(conf_file_mgmt)) | |
a4b4bb50 | 1584 | |
27c6bfc2 CH |
1585 | logger.debug( |
1586 | "copying '%s' as '%s' on '%s'", source, conf_file, self.name | |
1587 | ) | |
1588 | self.cmd_raises("cp {} {}".format(source, conf_file)) | |
a4b4bb50 JAG |
1589 | self.cmd_raises("chown {0}:{0} {1}".format(self.routertype, conf_file)) |
1590 | self.cmd_raises("chmod 664 {}".format(conf_file)) | |
1591 | ||
92be50e6 | 1592 | if (daemon == "snmpd") and (self.routertype == "frr"): |
49581587 | 1593 | # /etc/snmp is private mount now |
a4b4bb50 | 1594 | self.cmd('echo "agentXSocket /etc/frr/agentx" >> /etc/snmp/frr.conf') |
49581587 CH |
1595 | self.cmd('echo "mibs +ALL" > /etc/snmp/snmp.conf') |
1596 | ||
f637ac01 | 1597 | if (daemon == "zebra") and (self.daemons["mgmtd"] == 0): |
1598 | # Add mgmtd with zebra - if it exists | |
249ac6f0 | 1599 | mgmtd_path = os.path.join(self.daemondir, "mgmtd") |
f637ac01 | 1600 | if os.path.isfile(mgmtd_path): |
1601 | self.daemons["mgmtd"] = 1 | |
1602 | self.daemons_options["mgmtd"] = "" | |
1603 | # Auto-Started mgmtd has no config, so it will read from zebra config | |
1604 | ||
787e7624 | 1605 | if (daemon == "zebra") and (self.daemons["staticd"] == 0): |
a2a1134c | 1606 | # Add staticd with zebra - if it exists |
249ac6f0 | 1607 | staticd_path = os.path.join(self.daemondir, "staticd") |
a2a1134c | 1608 | if os.path.isfile(staticd_path): |
787e7624 | 1609 | self.daemons["staticd"] = 1 |
1610 | self.daemons_options["staticd"] = "" | |
2c805e6c | 1611 | # Auto-Started staticd has no config, so it will read from zebra config |
f637ac01 | 1612 | |
594b1259 | 1613 | else: |
e8f7a22f | 1614 | logger.warning("No daemon {} known".format(daemon)) |
27c6bfc2 CH |
1615 | |
1616 | return source if os.path.exists(source) else "" | |
e1dfa45e | 1617 | |
3f950192 | 1618 | def runInWindow(self, cmd, title=None): |
49581587 | 1619 | return self.run_in_window(cmd, title) |
3f950192 | 1620 | |
9711fc7e | 1621 | def startRouter(self, tgen=None): |
a4b4bb50 JAG |
1622 | if self.unified_config: |
1623 | self.cmd( | |
1624 | 'echo "service integrated-vtysh-config" >> /etc/%s/vtysh.conf' | |
1625 | % self.routertype | |
1626 | ) | |
1627 | else: | |
1628 | # Disable integrated-vtysh-config | |
1629 | self.cmd( | |
1630 | 'echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' | |
1631 | % self.routertype | |
1632 | ) | |
1633 | ||
787e7624 | 1634 | self.cmd( |
1635 | "chown %s:%svty /etc/%s/vtysh.conf" | |
1636 | % (self.routertype, self.routertype, self.routertype) | |
1637 | ) | |
13e1fc49 | 1638 | # TODO remove the following lines after all tests are migrated to Topogen. |
594b1259 | 1639 | # Try to find relevant old logfiles in /tmp and delete them |
787e7624 | 1640 | map(os.remove, glob.glob("{}/{}/*.log".format(self.logdir, self.name))) |
594b1259 | 1641 | # Remove old core files |
787e7624 | 1642 | map(os.remove, glob.glob("{}/{}/*.dmp".format(self.logdir, self.name))) |
594b1259 MW |
1643 | # Remove IP addresses from OS first - we have them in zebra.conf |
1644 | self.removeIPs() | |
1645 | # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher | |
1646 | # No error - but return message and skip all the tests | |
787e7624 | 1647 | if self.daemons["ldpd"] == 1: |
1648 | ldpd_path = os.path.join(self.daemondir, "ldpd") | |
2ab85530 | 1649 | if not os.path.isfile(ldpd_path): |
222ea88b | 1650 | logger.info("LDP Test, but no ldpd compiled or installed") |
594b1259 | 1651 | return "LDP Test, but no ldpd compiled or installed" |
dd4eca4d | 1652 | |
787e7624 | 1653 | if version_cmp(platform.release(), "4.5") < 0: |
222ea88b | 1654 | logger.info("LDP Test need Linux Kernel 4.5 minimum") |
45619ee3 | 1655 | return "LDP Test need Linux Kernel 4.5 minimum" |
9711fc7e LB |
1656 | # Check if have mpls |
1657 | if tgen != None: | |
1658 | self.hasmpls = tgen.hasmpls | |
1659 | if self.hasmpls != True: | |
787e7624 | 1660 | logger.info( |
1661 | "LDP/MPLS Tests will be skipped, platform missing module(s)" | |
1662 | ) | |
9711fc7e LB |
1663 | else: |
1664 | # Test for MPLS Kernel modules available | |
1665 | self.hasmpls = False | |
787e7624 | 1666 | if not module_present("mpls-router"): |
1667 | logger.info( | |
1668 | "MPLS tests will not run (missing mpls-router kernel module)" | |
1669 | ) | |
1670 | elif not module_present("mpls-iptunnel"): | |
1671 | logger.info( | |
1672 | "MPLS tests will not run (missing mpls-iptunnel kernel module)" | |
1673 | ) | |
9711fc7e LB |
1674 | else: |
1675 | self.hasmpls = True | |
1676 | if self.hasmpls != True: | |
1677 | return "LDP/MPLS Tests need mpls kernel modules" | |
49581587 CH |
1678 | |
1679 | # Really want to use sysctl_atleast here, but only when MPLS is actually being | |
1680 | # used | |
787e7624 | 1681 | self.cmd("echo 100000 > /proc/sys/net/mpls/platform_labels") |
44a592b2 | 1682 | |
249ac6f0 | 1683 | if g_pytest_config.name_in_option_list(self.name, "--shell"): |
0bc76852 | 1684 | self.run_in_window(os.getenv("SHELL", "bash"), title="sh-%s" % self.name) |
3f950192 | 1685 | |
787e7624 | 1686 | if self.daemons["eigrpd"] == 1: |
1687 | eigrpd_path = os.path.join(self.daemondir, "eigrpd") | |
44a592b2 | 1688 | if not os.path.isfile(eigrpd_path): |
222ea88b | 1689 | logger.info("EIGRP Test, but no eigrpd compiled or installed") |
44a592b2 MW |
1690 | return "EIGRP Test, but no eigrpd compiled or installed" |
1691 | ||
787e7624 | 1692 | if self.daemons["bfdd"] == 1: |
1693 | bfdd_path = os.path.join(self.daemondir, "bfdd") | |
4d45d6d3 RZ |
1694 | if not os.path.isfile(bfdd_path): |
1695 | logger.info("BFD Test, but no bfdd compiled or installed") | |
1696 | return "BFD Test, but no bfdd compiled or installed" | |
1697 | ||
1726edc3 CH |
1698 | status = self.startRouterDaemons(tgen=tgen) |
1699 | ||
249ac6f0 | 1700 | if g_pytest_config.name_in_option_list(self.name, "--vtysh"): |
0bc76852 | 1701 | self.run_in_window("vtysh", title="vt-%s" % self.name) |
1726edc3 | 1702 | |
a4b4bb50 JAG |
1703 | if self.unified_config: |
1704 | self.cmd("vtysh -f /etc/frr/frr.conf") | |
1705 | ||
1726edc3 | 1706 | return status |
aa5261bf | 1707 | |
aa5261bf RZ |
1708 | def getStdErr(self, daemon): |
1709 | return self.getLog("err", daemon) | |
1710 | ||
1711 | def getStdOut(self, daemon): | |
1712 | return self.getLog("out", daemon) | |
1713 | ||
1714 | def getLog(self, log, daemon): | |
c6686550 LB |
1715 | filename = "{}/{}/{}.{}".format(self.logdir, self.name, daemon, log) |
1716 | log = "" | |
1717 | with open(filename) as file: | |
1718 | log = file.read() | |
1719 | return log | |
aa5261bf | 1720 | |
0c449b01 | 1721 | def startRouterDaemons(self, daemons=None, tgen=None): |
49581587 | 1722 | "Starts FRR daemons for this router." |
e1dfa45e | 1723 | |
249ac6f0 CH |
1724 | asan_abort = bool(g_pytest_config.option.asan_abort) |
1725 | gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints") | |
1726 | gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons") | |
1727 | gdb_routers = g_pytest_config.get_option_list("--gdb-routers") | |
1728 | valgrind_extra = bool(g_pytest_config.option.valgrind_extra) | |
1729 | valgrind_memleaks = bool(g_pytest_config.option.valgrind_memleaks) | |
1730 | strace_daemons = g_pytest_config.get_option_list("--strace-daemons") | |
3f950192 | 1731 | |
49581587 CH |
1732 | # Get global bundle data |
1733 | if not self.path_exists("/etc/frr/support_bundle_commands.conf"): | |
1734 | # Copy global value if was covered by namespace mount | |
1735 | bundle_data = "" | |
1736 | if os.path.exists("/etc/frr/support_bundle_commands.conf"): | |
1737 | with open("/etc/frr/support_bundle_commands.conf", "r") as rf: | |
1738 | bundle_data = rf.read() | |
1739 | self.cmd_raises( | |
1740 | "cat > /etc/frr/support_bundle_commands.conf", | |
1741 | stdin=bundle_data, | |
701a0192 | 1742 | ) |
c39fe454 | 1743 | |
e1dfa45e LB |
1744 | # Starts actual daemons without init (ie restart) |
1745 | # cd to per node directory | |
49581587 CH |
1746 | self.cmd("install -m 775 -o frr -g frr -d {}/{}".format(self.logdir, self.name)) |
1747 | self.set_cwd("{}/{}".format(self.logdir, self.name)) | |
787e7624 | 1748 | self.cmd("umask 000") |
aa5261bf | 1749 | |
787e7624 | 1750 | # Re-enable to allow for report per run |
2a59a86b | 1751 | self.reportCores = True |
aa5261bf RZ |
1752 | |
1753 | # XXX: glue code forward ported from removed function. | |
249ac6f0 | 1754 | if self.version is None: |
787e7624 | 1755 | self.version = self.cmd( |
c39fe454 | 1756 | os.path.join(self.daemondir, "bgpd") + " -v" |
787e7624 | 1757 | ).split()[2] |
1758 | logger.info("{}: running version: {}".format(self.name, self.version)) | |
ff28990e | 1759 | |
e6079f4f CH |
1760 | perfds = {} |
1761 | perf_options = g_pytest_config.get_option("--perf-options", "-g") | |
1762 | for perf in g_pytest_config.get_option("--perf", []): | |
1763 | if "," in perf: | |
1764 | daemon, routers = perf.split(",", 1) | |
1765 | perfds[daemon] = routers.split(",") | |
1766 | else: | |
1767 | daemon = perf | |
1768 | perfds[daemon] = ["all"] | |
1769 | ||
ff28990e CH |
1770 | logd_options = {} |
1771 | for logd in g_pytest_config.get_option("--logd", []): | |
1772 | if "," in logd: | |
1773 | daemon, routers = logd.split(",", 1) | |
1774 | logd_options[daemon] = routers.split(",") | |
1775 | else: | |
1776 | daemon = logd | |
1777 | logd_options[daemon] = ["all"] | |
1778 | ||
aa5261bf RZ |
1779 | # If `daemons` was specified then some upper API called us with |
1780 | # specific daemons, otherwise just use our own configuration. | |
1781 | daemons_list = [] | |
3f950192 | 1782 | if daemons is not None: |
bb91e9c0 MS |
1783 | daemons_list = daemons |
1784 | else: | |
aa5261bf RZ |
1785 | # Append all daemons configured. |
1786 | for daemon in self.daemons: | |
1787 | if self.daemons[daemon] == 1: | |
1788 | daemons_list.append(daemon) | |
1789 | ||
ff28990e | 1790 | tail_log_files = [] |
260268c4 | 1791 | check_daemon_files = [] |
ff28990e | 1792 | |
3f950192 CH |
1793 | def start_daemon(daemon, extra_opts=None): |
1794 | daemon_opts = self.daemons_options.get(daemon, "") | |
260268c4 CH |
1795 | |
1796 | # get pid and vty filenames and remove the files | |
1797 | m = re.match(r"(.* |^)-n (\d+)( ?.*|$)", daemon_opts) | |
1798 | dfname = daemon if not m else "{}-{}".format(daemon, m.group(2)) | |
1799 | runbase = "/var/run/{}/{}".format(self.routertype, dfname) | |
1800 | # If this is a new system bring-up remove the pid/vty files, otherwise | |
1801 | # do not since apparently presence of the pidfile impacts BGP GR | |
1802 | self.cmd_status("rm -f {0}.pid {0}.vty".format(runbase)) | |
1803 | ||
3f950192 CH |
1804 | rediropt = " > {0}.out 2> {0}.err".format(daemon) |
1805 | if daemon == "snmpd": | |
1806 | binary = "/usr/sbin/snmpd" | |
1807 | cmdenv = "" | |
1808 | cmdopt = "{} -C -c /etc/frr/snmpd.conf -p ".format( | |
1809 | daemon_opts | |
260268c4 CH |
1810 | ) + "{}.pid -x /etc/frr/agentx".format(runbase) |
1811 | # check_daemon_files.append(runbase + ".pid") | |
3f950192 CH |
1812 | else: |
1813 | binary = os.path.join(self.daemondir, daemon) | |
260268c4 | 1814 | check_daemon_files.extend([runbase + ".pid", runbase + ".vty"]) |
e58133a7 | 1815 | |
0ba1d257 CH |
1816 | cmdenv = "ASAN_OPTIONS=" |
1817 | if asan_abort: | |
449e2555 CH |
1818 | cmdenv += "abort_on_error=1:" |
1819 | cmdenv += "log_path={0}/{1}.asan.{2} ".format( | |
a53c08bc CH |
1820 | self.logdir, self.name, daemon |
1821 | ) | |
0ba1d257 | 1822 | |
e58133a7 | 1823 | if valgrind_memleaks: |
a53c08bc CH |
1824 | this_dir = os.path.dirname( |
1825 | os.path.abspath(os.path.realpath(__file__)) | |
1826 | ) | |
1827 | supp_file = os.path.abspath( | |
1828 | os.path.join(this_dir, "../../../tools/valgrind.supp") | |
1829 | ) | |
1830 | cmdenv += " /usr/bin/valgrind --num-callers=50 --log-file={1}/{2}.valgrind.{0}.%p --leak-check=full --suppressions={3}".format( | |
1831 | daemon, self.logdir, self.name, supp_file | |
1832 | ) | |
e58133a7 | 1833 | if valgrind_extra: |
a53c08bc | 1834 | cmdenv += ( |
f2415785 | 1835 | " --gen-suppressions=all --expensive-definedness-checks=yes" |
a53c08bc | 1836 | ) |
0ba1d257 | 1837 | elif daemon in strace_daemons or "all" in strace_daemons: |
a53c08bc CH |
1838 | cmdenv = "strace -f -D -o {1}/{2}.strace.{0} ".format( |
1839 | daemon, self.logdir, self.name | |
1840 | ) | |
0ba1d257 | 1841 | |
ff28990e CH |
1842 | cmdopt = "{} --command-log-always ".format(daemon_opts) |
1843 | cmdopt += "--log file:{}.log --log-level debug".format(daemon) | |
1844 | ||
1845 | if daemon in logd_options: | |
1846 | logdopt = logd_options[daemon] | |
1847 | if "all" in logdopt or self.name in logdopt: | |
1848 | tail_log_files.append( | |
1849 | "{}/{}/{}.log".format(self.logdir, self.name, daemon) | |
1850 | ) | |
3f950192 CH |
1851 | if extra_opts: |
1852 | cmdopt += " " + extra_opts | |
1853 | ||
1854 | if ( | |
1855 | (gdb_routers or gdb_daemons) | |
0b25370e DS |
1856 | and ( |
1857 | not gdb_routers or self.name in gdb_routers or "all" in gdb_routers | |
1858 | ) | |
1859 | and (not gdb_daemons or daemon in gdb_daemons or "all" in gdb_daemons) | |
3f950192 CH |
1860 | ): |
1861 | if daemon == "snmpd": | |
1862 | cmdopt += " -f " | |
1863 | ||
1864 | cmdopt += rediropt | |
1865 | gdbcmd = "sudo -E gdb " + binary | |
1866 | if gdb_breakpoints: | |
1867 | gdbcmd += " -ex 'set breakpoint pending on'" | |
1868 | for bp in gdb_breakpoints: | |
1869 | gdbcmd += " -ex 'b {}'".format(bp) | |
1870 | gdbcmd += " -ex 'run {}'".format(cmdopt) | |
1871 | ||
49581587 CH |
1872 | self.run_in_window(gdbcmd, daemon) |
1873 | ||
a53c08bc CH |
1874 | logger.info( |
1875 | "%s: %s %s launched in gdb window", self, self.routertype, daemon | |
1876 | ) | |
c6686550 LB |
1877 | elif daemon in perfds and ( |
1878 | self.name in perfds[daemon] or "all" in perfds[daemon] | |
1879 | ): | |
e6079f4f | 1880 | cmdopt += rediropt |
c6686550 LB |
1881 | cmd = " ".join( |
1882 | ["perf record {} --".format(perf_options), binary, cmdopt] | |
1883 | ) | |
e6079f4f CH |
1884 | p = self.popen(cmd) |
1885 | self.perf_daemons[daemon] = p | |
1886 | if p.poll() and p.returncode: | |
1887 | self.logger.error( | |
1888 | '%s: Failed to launch "%s" (%s) with perf using: %s', | |
1889 | self, | |
1890 | daemon, | |
1891 | p.returncode, | |
1892 | cmd, | |
1893 | ) | |
1894 | else: | |
1895 | logger.debug( | |
1896 | "%s: %s %s started with perf", self, self.routertype, daemon | |
1897 | ) | |
3f950192 CH |
1898 | else: |
1899 | if daemon != "snmpd": | |
1900 | cmdopt += " -d " | |
1901 | cmdopt += rediropt | |
49581587 CH |
1902 | |
1903 | try: | |
1904 | self.cmd_raises(" ".join([cmdenv, binary, cmdopt]), warn=False) | |
1905 | except subprocess.CalledProcessError as error: | |
1906 | self.logger.error( | |
1907 | '%s: Failed to launch "%s" daemon (%d) using: %s%s%s:', | |
a53c08bc CH |
1908 | self, |
1909 | daemon, | |
1910 | error.returncode, | |
1911 | error.cmd, | |
1912 | '\n:stdout: "{}"'.format(error.stdout.strip()) | |
1913 | if error.stdout | |
1914 | else "", | |
1915 | '\n:stderr: "{}"'.format(error.stderr.strip()) | |
1916 | if error.stderr | |
1917 | else "", | |
49581587 CH |
1918 | ) |
1919 | else: | |
e8f7a22f | 1920 | logger.debug("%s: %s %s started", self, self.routertype, daemon) |
3f950192 | 1921 | |
f637ac01 | 1922 | # Start mgmtd first |
1923 | if "mgmtd" in daemons_list: | |
1924 | start_daemon("mgmtd") | |
1925 | while "mgmtd" in daemons_list: | |
1926 | daemons_list.remove("mgmtd") | |
1927 | ||
1928 | # Start Zebra after mgmtd | |
3f950192 CH |
1929 | if "zebra" in daemons_list: |
1930 | start_daemon("zebra", "-s 90000000") | |
c39fe454 KK |
1931 | while "zebra" in daemons_list: |
1932 | daemons_list.remove("zebra") | |
aa5261bf | 1933 | |
a2a1134c | 1934 | # Start staticd next if required |
c39fe454 | 1935 | if "staticd" in daemons_list: |
3f950192 | 1936 | start_daemon("staticd") |
c39fe454 KK |
1937 | while "staticd" in daemons_list: |
1938 | daemons_list.remove("staticd") | |
aa5261bf | 1939 | |
92be50e6 | 1940 | if "snmpd" in daemons_list: |
49581587 CH |
1941 | # Give zerbra a chance to configure interface addresses that snmpd daemon |
1942 | # may then use. | |
1943 | time.sleep(2) | |
1944 | ||
3f950192 | 1945 | start_daemon("snmpd") |
92be50e6 BC |
1946 | while "snmpd" in daemons_list: |
1947 | daemons_list.remove("snmpd") | |
1948 | ||
594b1259 | 1949 | # Now start all the other daemons |
cb3e512d | 1950 | for daemon in daemons_list: |
aa5261bf | 1951 | if self.daemons[daemon] == 0: |
2ab85530 | 1952 | continue |
3f950192 | 1953 | start_daemon(daemon) |
787e7624 | 1954 | |
aa5261bf | 1955 | # Check if daemons are running. |
260268c4 CH |
1956 | wait_time = 30 if (gdb_routers or gdb_daemons) else 10 |
1957 | timeout = Timeout(wait_time) | |
1958 | for remaining in timeout: | |
1959 | if not check_daemon_files: | |
1960 | break | |
1961 | check = check_daemon_files[0] | |
1962 | if self.path_exists(check): | |
1963 | check_daemon_files.pop(0) | |
1964 | continue | |
1965 | self.logger.debug("Waiting {}s for {} to appear".format(remaining, check)) | |
1966 | time.sleep(0.5) | |
1967 | ||
1968 | if check_daemon_files: | |
1969 | assert False, "Timeout({}) waiting for {} to appear on {}".format( | |
1970 | wait_time, check_daemon_files[0], self.name | |
1971 | ) | |
c65a7e26 | 1972 | |
49581587 CH |
1973 | # Update the permissions on the log files |
1974 | self.cmd("chown frr:frr -R {}/{}".format(self.logdir, self.name)) | |
1975 | self.cmd("chmod ug+rwX,o+r -R {}/{}".format(self.logdir, self.name)) | |
1976 | ||
ff28990e CH |
1977 | if "frr" in logd_options: |
1978 | logdopt = logd_options["frr"] | |
1979 | if "all" in logdopt or self.name in logdopt: | |
1980 | tail_log_files.append("{}/{}/frr.log".format(self.logdir, self.name)) | |
1981 | ||
1982 | for tailf in tail_log_files: | |
f3525b0b | 1983 | self.run_in_window("tail -n10000 -F " + tailf, title=tailf, background=True) |
ff28990e | 1984 | |
c65a7e26 KK |
1985 | return "" |
1986 | ||
346374b0 CH |
1987 | def pid_exists(self, pid): |
1988 | if pid <= 0: | |
1989 | return False | |
1990 | try: | |
1991 | # If we are not using PID namespaces then we will be a parent of the pid, | |
1992 | # otherwise the init process of the PID namespace will have reaped the proc. | |
1993 | os.waitpid(pid, os.WNOHANG) | |
1994 | except Exception: | |
1995 | pass | |
1996 | ||
1997 | rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False) | |
1998 | return rc == 0 or "No such process" not in e | |
1999 | ||
c39fe454 KK |
2000 | def killRouterDaemons( |
2001 | self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1" | |
2002 | ): | |
622c4996 | 2003 | # Kill Running FRR |
c65a7e26 | 2004 | # Daemons(user specified daemon only) using SIGKILL |
c39fe454 | 2005 | rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) |
c65a7e26 KK |
2006 | errors = "" |
2007 | daemonsNotRunning = [] | |
2008 | if re.search(r"No such file or directory", rundaemons): | |
2009 | return errors | |
2010 | for daemon in daemons: | |
2011 | if rundaemons is not None and daemon in rundaemons: | |
2012 | numRunning = 0 | |
701a0192 | 2013 | dmns = rundaemons.split("\n") |
cd79342c MS |
2014 | # Exclude empty string at end of list |
2015 | for d in dmns[:-1]: | |
c65a7e26 | 2016 | if re.search(r"%s" % daemon, d): |
6c5045ce MW |
2017 | daemonpidfile = d.rstrip() |
2018 | daemonpid = self.cmd("cat %s" % daemonpidfile).rstrip() | |
346374b0 | 2019 | if daemonpid.isdigit() and self.pid_exists(int(daemonpid)): |
e8f7a22f | 2020 | logger.debug( |
c39fe454 KK |
2021 | "{}: killing {}".format( |
2022 | self.name, | |
6c5045ce | 2023 | os.path.basename(daemonpidfile.rsplit(".", 1)[0]), |
c39fe454 KK |
2024 | ) |
2025 | ) | |
346374b0 CH |
2026 | self.cmd_status("kill -KILL {}".format(daemonpid)) |
2027 | if self.pid_exists(int(daemonpid)): | |
c65a7e26 | 2028 | numRunning += 1 |
c9f92703 | 2029 | while wait and numRunning > 0: |
c39fe454 KK |
2030 | sleep( |
2031 | 2, | |
2032 | "{}: waiting for {} daemon to be stopped".format( | |
2033 | self.name, daemon | |
2034 | ), | |
2035 | ) | |
cd79342c | 2036 | |
c65a7e26 | 2037 | # 2nd round of kill if daemons didn't exit |
cd79342c | 2038 | for d in dmns[:-1]: |
c65a7e26 | 2039 | if re.search(r"%s" % daemon, d): |
c39fe454 | 2040 | daemonpid = self.cmd("cat %s" % d.rstrip()).rstrip() |
346374b0 | 2041 | if daemonpid.isdigit() and self.pid_exists( |
c39fe454 KK |
2042 | int(daemonpid) |
2043 | ): | |
2044 | logger.info( | |
2045 | "{}: killing {}".format( | |
2046 | self.name, | |
2047 | os.path.basename( | |
2048 | d.rstrip().rsplit(".", 1)[0] | |
2049 | ), | |
2050 | ) | |
2051 | ) | |
346374b0 CH |
2052 | self.cmd_status( |
2053 | "kill -KILL {}".format(daemonpid) | |
2054 | ) | |
2055 | if daemonpid.isdigit() and not self.pid_exists( | |
c9f92703 DS |
2056 | int(daemonpid) |
2057 | ): | |
2058 | numRunning -= 1 | |
6c5045ce | 2059 | self.cmd("rm -- {}".format(daemonpidfile)) |
c65a7e26 KK |
2060 | if wait: |
2061 | errors = self.checkRouterCores(reportOnce=True) | |
c39fe454 KK |
2062 | if self.checkRouterVersion("<", minErrorVersion): |
2063 | # ignore errors in old versions | |
c65a7e26 KK |
2064 | errors = "" |
2065 | if assertOnError and len(errors) > 0: | |
2066 | assert "Errors found - details follow:" == 0, errors | |
c65a7e26 KK |
2067 | else: |
2068 | daemonsNotRunning.append(daemon) | |
2069 | if len(daemonsNotRunning) > 0: | |
c39fe454 | 2070 | errors = errors + "Daemons are not running", daemonsNotRunning |
c65a7e26 KK |
2071 | |
2072 | return errors | |
2073 | ||
2a59a86b LB |
2074 | def checkRouterCores(self, reportLeaks=True, reportOnce=False): |
2075 | if reportOnce and not self.reportCores: | |
2076 | return | |
2077 | reportMade = False | |
83c26937 | 2078 | traces = "" |
f76774ec | 2079 | for daemon in self.daemons: |
787e7624 | 2080 | if self.daemons[daemon] == 1: |
f76774ec | 2081 | # Look for core file |
787e7624 | 2082 | corefiles = glob.glob( |
2083 | "{}/{}/{}_core*.dmp".format(self.logdir, self.name, daemon) | |
2084 | ) | |
2085 | if len(corefiles) > 0: | |
79f6fdeb | 2086 | backtrace = gdb_core(self, daemon, corefiles) |
787e7624 | 2087 | traces = ( |
2088 | traces | |
2089 | + "\n%s: %s crashed. Core file found - Backtrace follows:\n%s" | |
2090 | % (self.name, daemon, backtrace) | |
2091 | ) | |
2a59a86b | 2092 | reportMade = True |
f76774ec LB |
2093 | elif reportLeaks: |
2094 | log = self.getStdErr(daemon) | |
2095 | if "memstats" in log: | |
787e7624 | 2096 | sys.stderr.write( |
2097 | "%s: %s has memory leaks:\n" % (self.name, daemon) | |
2098 | ) | |
2099 | traces = traces + "\n%s: %s has memory leaks:\n" % ( | |
2100 | self.name, | |
2101 | daemon, | |
2102 | ) | |
f76774ec | 2103 | log = re.sub("core_handler: ", "", log) |
787e7624 | 2104 | log = re.sub( |
2105 | r"(showing active allocations in memory group [a-zA-Z0-9]+)", | |
2106 | r"\n ## \1", | |
2107 | log, | |
2108 | ) | |
f76774ec LB |
2109 | log = re.sub("memstats: ", " ", log) |
2110 | sys.stderr.write(log) | |
2a59a86b | 2111 | reportMade = True |
f76774ec | 2112 | # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found |
787e7624 | 2113 | if checkAddressSanitizerError( |
be2656ed | 2114 | self.getStdErr(daemon), self.name, daemon, self.logdir |
787e7624 | 2115 | ): |
2116 | sys.stderr.write( | |
2117 | "%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon) | |
2118 | ) | |
2119 | traces = traces + "\n%s: Daemon %s killed by AddressSanitizer" % ( | |
2120 | self.name, | |
2121 | daemon, | |
2122 | ) | |
2a59a86b LB |
2123 | reportMade = True |
2124 | if reportMade: | |
2125 | self.reportCores = False | |
83c26937 | 2126 | return traces |
f76774ec | 2127 | |
594b1259 | 2128 | def checkRouterRunning(self): |
597cabb7 MW |
2129 | "Check if router daemons are running and collect crashinfo they don't run" |
2130 | ||
594b1259 MW |
2131 | global fatal_error |
2132 | ||
787e7624 | 2133 | daemonsRunning = self.cmd( |
2134 | 'vtysh -c "show logging" | grep "Logging configuration for"' | |
2135 | ) | |
4942f298 MW |
2136 | # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found |
2137 | if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"): | |
2138 | return "%s: vtysh killed by AddressSanitizer" % (self.name) | |
2139 | ||
594b1259 | 2140 | for daemon in self.daemons: |
662c0576 KS |
2141 | if daemon == "snmpd": |
2142 | continue | |
594b1259 MW |
2143 | if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning): |
2144 | sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon)) | |
11761ab0 | 2145 | if daemon == "staticd": |
787e7624 | 2146 | sys.stderr.write( |
2147 | "You may have a copy of staticd installed but are attempting to test against\n" | |
2148 | ) | |
2149 | sys.stderr.write( | |
2150 | "a version of FRR that does not have staticd, please cleanup the install dir\n" | |
2151 | ) | |
d2132114 | 2152 | |
594b1259 | 2153 | # Look for core file |
787e7624 | 2154 | corefiles = glob.glob( |
2155 | "{}/{}/{}_core*.dmp".format(self.logdir, self.name, daemon) | |
2156 | ) | |
2157 | if len(corefiles) > 0: | |
79f6fdeb | 2158 | gdb_core(self, daemon, corefiles) |
594b1259 MW |
2159 | else: |
2160 | # No core found - If we find matching logfile in /tmp, then print last 20 lines from it. | |
787e7624 | 2161 | if os.path.isfile( |
2162 | "{}/{}/{}.log".format(self.logdir, self.name, daemon) | |
2163 | ): | |
2164 | log_tail = subprocess.check_output( | |
2165 | [ | |
2166 | "tail -n20 {}/{}/{}.log 2> /dev/null".format( | |
2167 | self.logdir, self.name, daemon | |
2168 | ) | |
2169 | ], | |
2170 | shell=True, | |
2171 | ) | |
2172 | sys.stderr.write( | |
2173 | "\nFrom %s %s %s log file:\n" | |
2174 | % (self.routertype, self.name, daemon) | |
2175 | ) | |
594b1259 | 2176 | sys.stderr.write("%s\n" % log_tail) |
4942f298 | 2177 | |
597cabb7 | 2178 | # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found |
787e7624 | 2179 | if checkAddressSanitizerError( |
be2656ed | 2180 | self.getStdErr(daemon), self.name, daemon, self.logdir |
787e7624 | 2181 | ): |
2182 | return "%s: Daemon %s not running - killed by AddressSanitizer" % ( | |
2183 | self.name, | |
2184 | daemon, | |
2185 | ) | |
84379e8e | 2186 | |
594b1259 MW |
2187 | return "%s: Daemon %s not running" % (self.name, daemon) |
2188 | return "" | |
fb80b81b LB |
2189 | |
2190 | def checkRouterVersion(self, cmpop, version): | |
2191 | """ | |
2192 | Compares router version using operation `cmpop` with `version`. | |
2193 | Valid `cmpop` values: | |
2194 | * `>=`: has the same version or greater | |
2195 | * '>': has greater version | |
2196 | * '=': has the same version | |
2197 | * '<': has a lesser version | |
2198 | * '<=': has the same version or lesser | |
2199 | ||
2200 | Usage example: router.checkRouterVersion('>', '1.0') | |
2201 | """ | |
6bfe4b8b MW |
2202 | |
2203 | # Make sure we have version information first | |
2204 | if self.version == None: | |
787e7624 | 2205 | self.version = self.cmd( |
2206 | os.path.join(self.daemondir, "bgpd") + " -v" | |
2207 | ).split()[2] | |
2208 | logger.info("{}: running version: {}".format(self.name, self.version)) | |
6bfe4b8b | 2209 | |
fb80b81b | 2210 | rversion = self.version |
11761ab0 | 2211 | if rversion == None: |
fb80b81b LB |
2212 | return False |
2213 | ||
2214 | result = version_cmp(rversion, version) | |
787e7624 | 2215 | if cmpop == ">=": |
fb80b81b | 2216 | return result >= 0 |
787e7624 | 2217 | if cmpop == ">": |
fb80b81b | 2218 | return result > 0 |
787e7624 | 2219 | if cmpop == "=": |
fb80b81b | 2220 | return result == 0 |
787e7624 | 2221 | if cmpop == "<": |
fb80b81b | 2222 | return result < 0 |
787e7624 | 2223 | if cmpop == "<": |
fb80b81b | 2224 | return result < 0 |
787e7624 | 2225 | if cmpop == "<=": |
fb80b81b LB |
2226 | return result <= 0 |
2227 | ||
594b1259 MW |
2228 | def get_ipv6_linklocal(self): |
2229 | "Get LinkLocal Addresses from interfaces" | |
2230 | ||
2231 | linklocal = [] | |
2232 | ||
787e7624 | 2233 | ifaces = self.cmd("ip -6 address") |
594b1259 | 2234 | # Fix newlines (make them all the same) |
787e7624 | 2235 | ifaces = ("\n".join(ifaces.splitlines()) + "\n").splitlines() |
2236 | interface = "" | |
2237 | ll_per_if_count = 0 | |
594b1259 | 2238 | for line in ifaces: |
fd03dacd | 2239 | m = re.search("[0-9]+: ([^:@]+)[-@a-z0-9:]+ <", line) |
594b1259 MW |
2240 | if m: |
2241 | interface = m.group(1) | |
2242 | ll_per_if_count = 0 | |
787e7624 | 2243 | m = re.search( |
2244 | "inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link", | |
2245 | line, | |
2246 | ) | |
594b1259 MW |
2247 | if m: |
2248 | local = m.group(1) | |
2249 | ll_per_if_count += 1 | |
787e7624 | 2250 | if ll_per_if_count > 1: |
594b1259 MW |
2251 | linklocal += [["%s-%s" % (interface, ll_per_if_count), local]] |
2252 | else: | |
2253 | linklocal += [[interface, local]] | |
2254 | return linklocal | |
787e7624 | 2255 | |
80eeefb7 MW |
2256 | def daemon_available(self, daemon): |
2257 | "Check if specified daemon is installed (and for ldp if kernel supports MPLS)" | |
2258 | ||
2ab85530 RZ |
2259 | daemon_path = os.path.join(self.daemondir, daemon) |
2260 | if not os.path.isfile(daemon_path): | |
80eeefb7 | 2261 | return False |
787e7624 | 2262 | if daemon == "ldpd": |
2263 | if version_cmp(platform.release(), "4.5") < 0: | |
b431b554 | 2264 | return False |
787e7624 | 2265 | if not module_present("mpls-router", load=False): |
80eeefb7 | 2266 | return False |
787e7624 | 2267 | if not module_present("mpls-iptunnel", load=False): |
b431b554 | 2268 | return False |
80eeefb7 | 2269 | return True |
f2d6ce41 | 2270 | |
80eeefb7 | 2271 | def get_routertype(self): |
622c4996 | 2272 | "Return the type of Router (frr)" |
80eeefb7 MW |
2273 | |
2274 | return self.routertype | |
787e7624 | 2275 | |
50c40bde MW |
2276 | def report_memory_leaks(self, filename_prefix, testscript): |
2277 | "Report Memory Leaks to file prefixed with given string" | |
2278 | ||
2279 | leakfound = False | |
2280 | filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt" | |
2281 | for daemon in self.daemons: | |
787e7624 | 2282 | if self.daemons[daemon] == 1: |
50c40bde MW |
2283 | log = self.getStdErr(daemon) |
2284 | if "memstats" in log: | |
2285 | # Found memory leak | |
e8f7a22f | 2286 | logger.warning( |
787e7624 | 2287 | "\nRouter {} {} StdErr Log:\n{}".format(self.name, daemon, log) |
2288 | ) | |
50c40bde MW |
2289 | if not leakfound: |
2290 | leakfound = True | |
2291 | # Check if file already exists | |
2292 | fileexists = os.path.isfile(filename) | |
2293 | leakfile = open(filename, "a") | |
2294 | if not fileexists: | |
2295 | # New file - add header | |
787e7624 | 2296 | leakfile.write( |
2297 | "# Memory Leak Detection for topotest %s\n\n" | |
2298 | % testscript | |
2299 | ) | |
50c40bde MW |
2300 | leakfile.write("## Router %s\n" % self.name) |
2301 | leakfile.write("### Process %s\n" % daemon) | |
2302 | log = re.sub("core_handler: ", "", log) | |
787e7624 | 2303 | log = re.sub( |
2304 | r"(showing active allocations in memory group [a-zA-Z0-9]+)", | |
2305 | r"\n#### \1\n", | |
2306 | log, | |
2307 | ) | |
50c40bde MW |
2308 | log = re.sub("memstats: ", " ", log) |
2309 | leakfile.write(log) | |
2310 | leakfile.write("\n") | |
2311 | if leakfound: | |
2312 | leakfile.close() | |
80eeefb7 | 2313 | |
787e7624 | 2314 | |
61196140 | 2315 | def frr_unicode(s): |
701a0192 | 2316 | """Convert string to unicode, depending on python version""" |
61196140 MS |
2317 | if sys.version_info[0] > 2: |
2318 | return s | |
2319 | else: | |
49581587 | 2320 | return unicode(s) # pylint: disable=E0602 |
c8e5983d CH |
2321 | |
2322 | ||
2323 | def is_mapping(o): | |
2324 | return isinstance(o, Mapping) |