]>
Commit | Line | Data |
---|---|---|
594b1259 MW |
1 | #!/usr/bin/env python |
2 | ||
3 | # | |
4 | # topotest.py | |
5 | # Library of helper functions for NetDEF Topology Tests | |
6 | # | |
7 | # Copyright (c) 2016 by | |
8 | # Network Device Education Foundation, Inc. ("NetDEF") | |
9 | # | |
10 | # Permission to use, copy, modify, and/or distribute this software | |
11 | # for any purpose with or without fee is hereby granted, provided | |
12 | # that the above copyright notice and this permission notice appear | |
13 | # in all copies. | |
14 | # | |
15 | # THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES | |
16 | # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF | |
17 | # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR | |
18 | # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY | |
19 | # DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, | |
20 | # WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS | |
21 | # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE | |
22 | # OF THIS SOFTWARE. | |
23 | # | |
24 | ||
7bd28cfc | 25 | import json |
594b1259 | 26 | import os |
50c40bde | 27 | import errno |
594b1259 MW |
28 | import re |
29 | import sys | |
30 | import glob | |
31 | import StringIO | |
32 | import subprocess | |
1fca63c1 | 33 | import tempfile |
594b1259 | 34 | import platform |
17070436 | 35 | import difflib |
570f25d8 | 36 | import time |
594b1259 | 37 | |
6c131bd3 RZ |
38 | from lib.topolog import logger |
39 | ||
594b1259 MW |
40 | from mininet.topo import Topo |
41 | from mininet.net import Mininet | |
42 | from mininet.node import Node, OVSSwitch, Host | |
43 | from mininet.log import setLogLevel, info | |
44 | from mininet.cli import CLI | |
45 | from mininet.link import Intf | |
46 | ||
3668ed8d RZ |
47 | class json_cmp_result(object): |
48 | "json_cmp result class for better assertion messages" | |
49 | ||
50 | def __init__(self): | |
51 | self.errors = [] | |
52 | ||
53 | def add_error(self, error): | |
54 | "Append error message to the result" | |
2db5888d RZ |
55 | for line in error.splitlines(): |
56 | self.errors.append(line) | |
3668ed8d RZ |
57 | |
58 | def has_errors(self): | |
59 | "Returns True if there were errors, otherwise False." | |
60 | return len(self.errors) > 0 | |
61 | ||
7bd28cfc RZ |
62 | def json_diff(d1, d2): |
63 | """ | |
64 | Returns a string with the difference between JSON data. | |
65 | """ | |
66 | json_format_opts = { | |
67 | 'indent': 4, | |
68 | 'sort_keys': True, | |
69 | } | |
70 | dstr1 = json.dumps(d1, **json_format_opts) | |
71 | dstr2 = json.dumps(d2, **json_format_opts) | |
72 | return difflines(dstr2, dstr1, title1='Expected value', title2='Current value', n=0) | |
09e21b44 | 73 | |
566567e9 | 74 | def json_cmp(d1, d2): |
09e21b44 RZ |
75 | """ |
76 | JSON compare function. Receives two parameters: | |
77 | * `d1`: json value | |
78 | * `d2`: json subset which we expect | |
79 | ||
80 | Returns `None` when all keys that `d1` has matches `d2`, | |
81 | otherwise a string containing what failed. | |
82 | ||
83 | Note: key absence can be tested by adding a key with value `None`. | |
84 | """ | |
3668ed8d RZ |
85 | squeue = [(d1, d2, 'json')] |
86 | result = json_cmp_result() | |
09e21b44 | 87 | for s in squeue: |
3668ed8d | 88 | nd1, nd2, parent = s |
09e21b44 RZ |
89 | s1, s2 = set(nd1), set(nd2) |
90 | ||
91 | # Expect all required fields to exist. | |
92 | s2_req = set([key for key in nd2 if nd2[key] is not None]) | |
93 | diff = s2_req - s1 | |
94 | if diff != set({}): | |
08533b7b RZ |
95 | result.add_error('expected key(s) {} in {} (have {}):\n{}'.format( |
96 | str(list(diff)), parent, str(list(s1)), json_diff(nd1, nd2))) | |
09e21b44 RZ |
97 | |
98 | for key in s2.intersection(s1): | |
99 | # Test for non existence of key in d2 | |
100 | if nd2[key] is None: | |
08533b7b RZ |
101 | result.add_error('"{}" should not exist in {} (have {}):\n{}'.format( |
102 | key, parent, str(s1), json_diff(nd1[key], nd2[key]))) | |
3668ed8d | 103 | continue |
09e21b44 RZ |
104 | # If nd1 key is a dict, we have to recurse in it later. |
105 | if isinstance(nd2[key], type({})): | |
dc0d3fc5 RZ |
106 | if not isinstance(nd1[key], type({})): |
107 | result.add_error( | |
108 | '{}["{}"] has different type than expected '.format(parent, key) + | |
08533b7b RZ |
109 | '(have {}, expected {}):\n{}'.format( |
110 | type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key]))) | |
dc0d3fc5 | 111 | continue |
3668ed8d RZ |
112 | nparent = '{}["{}"]'.format(parent, key) |
113 | squeue.append((nd1[key], nd2[key], nparent)) | |
09e21b44 | 114 | continue |
dc0d3fc5 RZ |
115 | # Check list items |
116 | if isinstance(nd2[key], type([])): | |
117 | if not isinstance(nd1[key], type([])): | |
118 | result.add_error( | |
119 | '{}["{}"] has different type than expected '.format(parent, key) + | |
08533b7b RZ |
120 | '(have {}, expected {}):\n{}'.format( |
121 | type(nd1[key]), type(nd2[key]), json_diff(nd1[key], nd2[key]))) | |
dc0d3fc5 RZ |
122 | continue |
123 | # Check list size | |
124 | if len(nd2[key]) > len(nd1[key]): | |
125 | result.add_error( | |
126 | '{}["{}"] too few items '.format(parent, key) + | |
7bd28cfc RZ |
127 | '(have {}, expected {}:\n {})'.format( |
128 | len(nd1[key]), len(nd2[key]), | |
129 | json_diff(nd1[key], nd2[key]))) | |
dc0d3fc5 RZ |
130 | continue |
131 | ||
132 | # List all unmatched items errors | |
133 | unmatched = [] | |
134 | for expected in nd2[key]: | |
135 | matched = False | |
136 | for value in nd1[key]: | |
137 | if json_cmp({'json': value}, {'json': expected}) is None: | |
138 | matched = True | |
139 | break | |
140 | ||
141 | if matched: | |
142 | break | |
143 | if not matched: | |
144 | unmatched.append(expected) | |
145 | ||
146 | # If there are unmatched items, error out. | |
147 | if unmatched: | |
148 | result.add_error( | |
7bd28cfc RZ |
149 | '{}["{}"] value is different (\n{})'.format( |
150 | parent, key, json_diff(nd1[key], nd2[key]))) | |
dc0d3fc5 RZ |
151 | continue |
152 | ||
09e21b44 RZ |
153 | # Compare JSON values |
154 | if nd1[key] != nd2[key]: | |
3668ed8d | 155 | result.add_error( |
7bd28cfc RZ |
156 | '{}["{}"] value is different (\n{})'.format( |
157 | parent, key, json_diff(nd1[key], nd2[key]))) | |
3668ed8d RZ |
158 | continue |
159 | ||
160 | if result.has_errors(): | |
161 | return result | |
09e21b44 RZ |
162 | |
163 | return None | |
164 | ||
1fca63c1 RZ |
165 | def run_and_expect(func, what, count=20, wait=3): |
166 | """ | |
167 | Run `func` and compare the result with `what`. Do it for `count` times | |
168 | waiting `wait` seconds between tries. By default it tries 20 times with | |
169 | 3 seconds delay between tries. | |
170 | ||
171 | Returns (True, func-return) on success or | |
172 | (False, func-return) on failure. | |
173 | """ | |
174 | while count > 0: | |
175 | result = func() | |
176 | if result != what: | |
570f25d8 | 177 | time.sleep(wait) |
1fca63c1 RZ |
178 | count -= 1 |
179 | continue | |
180 | return (True, result) | |
181 | return (False, result) | |
182 | ||
183 | ||
594b1259 MW |
184 | def int2dpid(dpid): |
185 | "Converting Integer to DPID" | |
186 | ||
187 | try: | |
188 | dpid = hex(dpid)[2:] | |
189 | dpid = '0'*(16-len(dpid))+dpid | |
190 | return dpid | |
191 | except IndexError: | |
192 | raise Exception('Unable to derive default datapath ID - ' | |
193 | 'please either specify a dpid or use a ' | |
194 | 'canonical switch name such as s23.') | |
195 | ||
50c40bde MW |
196 | def pid_exists(pid): |
197 | "Check whether pid exists in the current process table." | |
198 | ||
199 | if pid <= 0: | |
200 | return False | |
201 | try: | |
202 | os.kill(pid, 0) | |
203 | except OSError as err: | |
204 | if err.errno == errno.ESRCH: | |
205 | # ESRCH == No such process | |
206 | return False | |
207 | elif err.errno == errno.EPERM: | |
208 | # EPERM clearly means there's a process to deny access to | |
209 | return True | |
210 | else: | |
211 | # According to "man 2 kill" possible error values are | |
212 | # (EINVAL, EPERM, ESRCH) | |
213 | raise | |
214 | else: | |
215 | return True | |
216 | ||
bc2872fd | 217 | def get_textdiff(text1, text2, title1="", title2="", **opts): |
17070436 MW |
218 | "Returns empty string if same or formatted diff" |
219 | ||
91733ef8 | 220 | diff = '\n'.join(difflib.unified_diff(text1, text2, |
bc2872fd | 221 | fromfile=title1, tofile=title2, **opts)) |
17070436 MW |
222 | # Clean up line endings |
223 | diff = os.linesep.join([s for s in diff.splitlines() if s]) | |
224 | return diff | |
225 | ||
bc2872fd | 226 | def difflines(text1, text2, title1='', title2='', **opts): |
1fca63c1 RZ |
227 | "Wrapper for get_textdiff to avoid string transformations." |
228 | text1 = ('\n'.join(text1.rstrip().splitlines()) + '\n').splitlines(1) | |
229 | text2 = ('\n'.join(text2.rstrip().splitlines()) + '\n').splitlines(1) | |
bc2872fd | 230 | return get_textdiff(text1, text2, title1, title2, **opts) |
1fca63c1 RZ |
231 | |
232 | def get_file(content): | |
233 | """ | |
234 | Generates a temporary file in '/tmp' with `content` and returns the file name. | |
235 | """ | |
236 | fde = tempfile.NamedTemporaryFile(mode='w', delete=False) | |
237 | fname = fde.name | |
238 | fde.write(content) | |
239 | fde.close() | |
240 | return fname | |
241 | ||
f7840f6b RZ |
242 | def normalize_text(text): |
243 | """ | |
244 | Strips formating spaces/tabs and carriage returns. | |
245 | """ | |
246 | text = re.sub(r'[ \t]+', ' ', text) | |
247 | text = re.sub(r'\r', '', text) | |
248 | return text | |
249 | ||
4190fe1e RZ |
250 | def version_cmp(v1, v2): |
251 | """ | |
252 | Compare two version strings and returns: | |
253 | ||
254 | * `-1`: if `v1` is less than `v2` | |
255 | * `0`: if `v1` is equal to `v2` | |
256 | * `1`: if `v1` is greater than `v2` | |
257 | ||
258 | Raises `ValueError` if versions are not well formated. | |
259 | """ | |
260 | vregex = r'(?P<whole>\d+(\.(\d+))*)' | |
261 | v1m = re.match(vregex, v1) | |
262 | v2m = re.match(vregex, v2) | |
263 | if v1m is None or v2m is None: | |
264 | raise ValueError("got a invalid version string") | |
265 | ||
266 | # Split values | |
267 | v1g = v1m.group('whole').split('.') | |
268 | v2g = v2m.group('whole').split('.') | |
269 | ||
270 | # Get the longest version string | |
271 | vnum = len(v1g) | |
272 | if len(v2g) > vnum: | |
273 | vnum = len(v2g) | |
274 | ||
275 | # Reverse list because we are going to pop the tail | |
276 | v1g.reverse() | |
277 | v2g.reverse() | |
278 | for _ in range(vnum): | |
279 | try: | |
280 | v1n = int(v1g.pop()) | |
281 | except IndexError: | |
282 | while v2g: | |
283 | v2n = int(v2g.pop()) | |
284 | if v2n > 0: | |
285 | return -1 | |
286 | break | |
287 | ||
288 | try: | |
289 | v2n = int(v2g.pop()) | |
290 | except IndexError: | |
291 | if v1n > 0: | |
292 | return 1 | |
293 | while v1g: | |
294 | v1n = int(v1g.pop()) | |
295 | if v1n > 0: | |
034237db | 296 | return 1 |
4190fe1e RZ |
297 | break |
298 | ||
299 | if v1n > v2n: | |
300 | return 1 | |
301 | if v1n < v2n: | |
302 | return -1 | |
303 | return 0 | |
304 | ||
99a7a912 RZ |
305 | def ip4_route(node): |
306 | """ | |
307 | Gets a structured return of the command 'ip route'. It can be used in | |
308 | conjuction with json_cmp() to provide accurate assert explanations. | |
309 | ||
310 | Return example: | |
311 | { | |
312 | '10.0.1.0/24': { | |
313 | 'dev': 'eth0', | |
314 | 'via': '172.16.0.1', | |
315 | 'proto': '188', | |
316 | }, | |
317 | '10.0.2.0/24': { | |
318 | 'dev': 'eth1', | |
319 | 'proto': 'kernel', | |
320 | } | |
321 | } | |
322 | """ | |
323 | output = normalize_text(node.run('ip route')).splitlines() | |
324 | result = {} | |
325 | for line in output: | |
326 | columns = line.split(' ') | |
327 | route = result[columns[0]] = {} | |
328 | prev = None | |
329 | for column in columns: | |
330 | if prev == 'dev': | |
331 | route['dev'] = column | |
332 | if prev == 'via': | |
333 | route['via'] = column | |
334 | if prev == 'proto': | |
335 | route['proto'] = column | |
336 | if prev == 'metric': | |
337 | route['metric'] = column | |
338 | if prev == 'scope': | |
339 | route['scope'] = column | |
340 | prev = column | |
341 | ||
342 | return result | |
343 | ||
344 | def ip6_route(node): | |
345 | """ | |
346 | Gets a structured return of the command 'ip -6 route'. It can be used in | |
347 | conjuction with json_cmp() to provide accurate assert explanations. | |
348 | ||
349 | Return example: | |
350 | { | |
351 | '2001:db8:1::/64': { | |
352 | 'dev': 'eth0', | |
353 | 'proto': '188', | |
354 | }, | |
355 | '2001:db8:2::/64': { | |
356 | 'dev': 'eth1', | |
357 | 'proto': 'kernel', | |
358 | } | |
359 | } | |
360 | """ | |
361 | output = normalize_text(node.run('ip -6 route')).splitlines() | |
362 | result = {} | |
363 | for line in output: | |
364 | columns = line.split(' ') | |
365 | route = result[columns[0]] = {} | |
366 | prev = None | |
367 | for column in columns: | |
368 | if prev == 'dev': | |
369 | route['dev'] = column | |
370 | if prev == 'via': | |
371 | route['via'] = column | |
372 | if prev == 'proto': | |
373 | route['proto'] = column | |
374 | if prev == 'metric': | |
375 | route['metric'] = column | |
376 | if prev == 'pref': | |
377 | route['pref'] = column | |
378 | prev = column | |
379 | ||
380 | return result | |
381 | ||
570f25d8 RZ |
382 | def sleep(amount, reason=None): |
383 | """ | |
384 | Sleep wrapper that registers in the log the amount of sleep | |
385 | """ | |
386 | if reason is None: | |
387 | logger.info('Sleeping for {} seconds'.format(amount)) | |
388 | else: | |
389 | logger.info(reason + ' ({} seconds)'.format(amount)) | |
390 | ||
391 | time.sleep(amount) | |
392 | ||
4942f298 MW |
393 | def checkAddressSanitizerError(output, router, component): |
394 | "Checks for AddressSanitizer in output. If found, then logs it and returns true, false otherwise" | |
395 | ||
396 | addressSantizerError = re.search('(==[0-9]+==)ERROR: AddressSanitizer: ([^\s]*) ', output) | |
397 | if addressSantizerError: | |
398 | sys.stderr.write("%s: %s triggered an exception by AddressSanitizer\n" % (router, component)) | |
399 | # Sanitizer Error found in log | |
400 | pidMark = addressSantizerError.group(1) | |
401 | addressSantizerLog = re.search('%s(.*)%s' % (pidMark, pidMark), output, re.DOTALL) | |
402 | if addressSantizerLog: | |
403 | callingTest = os.path.basename(sys._current_frames().values()[0].f_back.f_back.f_globals['__file__']) | |
404 | callingProc = sys._getframe(2).f_code.co_name | |
405 | with open("/tmp/AddressSanitzer.txt", "a") as addrSanFile: | |
406 | sys.stderr.write('\n'.join(addressSantizerLog.group(1).splitlines()) + '\n') | |
407 | addrSanFile.write("## Error: %s\n\n" % addressSantizerError.group(2)) | |
408 | addrSanFile.write("### AddressSanitizer error in topotest `%s`, test `%s`, router `%s`\n\n" % (callingTest, callingProc, router)) | |
409 | addrSanFile.write(' '+ '\n '.join(addressSantizerLog.group(1).splitlines()) + '\n') | |
410 | addrSanFile.write("\n---------------\n") | |
411 | return True | |
6c131bd3 | 412 | return False |
4942f298 | 413 | |
594b1259 | 414 | def addRouter(topo, name): |
80eeefb7 | 415 | "Adding a FRRouter (or Quagga) to Topology" |
594b1259 MW |
416 | |
417 | MyPrivateDirs = ['/etc/frr', | |
418 | '/etc/quagga', | |
419 | '/var/run/frr', | |
420 | '/var/run/quagga', | |
421 | '/var/log'] | |
422 | return topo.addNode(name, cls=Router, privateDirs=MyPrivateDirs) | |
423 | ||
797e8dcf RZ |
424 | def set_sysctl(node, sysctl, value): |
425 | "Set a sysctl value and return None on success or an error string" | |
426 | valuestr = '{}'.format(value) | |
427 | command = "sysctl {0}={1}".format(sysctl, valuestr) | |
428 | cmdret = node.cmd(command) | |
429 | ||
430 | matches = re.search(r'([^ ]+) = ([^\s]+)', cmdret) | |
431 | if matches is None: | |
432 | return cmdret | |
433 | if matches.group(1) != sysctl: | |
434 | return cmdret | |
435 | if matches.group(2) != valuestr: | |
436 | return cmdret | |
437 | ||
438 | return None | |
439 | ||
440 | def assert_sysctl(node, sysctl, value): | |
441 | "Set and assert that the sysctl is set with the specified value." | |
442 | assert set_sysctl(node, sysctl, value) is None | |
443 | ||
594b1259 MW |
444 | class LinuxRouter(Node): |
445 | "A Node with IPv4/IPv6 forwarding enabled." | |
446 | ||
447 | def config(self, **params): | |
448 | super(LinuxRouter, self).config(**params) | |
449 | # Enable forwarding on the router | |
797e8dcf RZ |
450 | assert_sysctl(self, 'net.ipv4.ip_forward', 1) |
451 | assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1) | |
594b1259 MW |
452 | def terminate(self): |
453 | """ | |
454 | Terminate generic LinuxRouter Mininet instance | |
455 | """ | |
797e8dcf RZ |
456 | set_sysctl(self, 'net.ipv4.ip_forward', 0) |
457 | set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0) | |
594b1259 MW |
458 | super(LinuxRouter, self).terminate() |
459 | ||
460 | class Router(Node): | |
461 | "A Node with IPv4/IPv6 forwarding enabled and Quagga as Routing Engine" | |
462 | ||
2ab85530 RZ |
463 | def __init__(self, name, **params): |
464 | super(Router, self).__init__(name, **params) | |
13e1fc49 | 465 | self.logdir = params.get('logdir', '/tmp') |
2ab85530 | 466 | self.daemondir = None |
447f2d5a | 467 | self.hasmpls = False |
2ab85530 RZ |
468 | self.routertype = 'frr' |
469 | self.daemons = {'zebra': 0, 'ripd': 0, 'ripngd': 0, 'ospfd': 0, | |
470 | 'ospf6d': 0, 'isisd': 0, 'bgpd': 0, 'pimd': 0, | |
cda83bee | 471 | 'ldpd': 0, 'eigrpd': 0, 'nhrpd': 0} |
8dd5077d | 472 | self.daemons_options = {'zebra': ''} |
2ab85530 | 473 | |
edd2bdf6 RZ |
474 | def _config_frr(self, **params): |
475 | "Configure FRR binaries" | |
476 | self.daemondir = params.get('frrdir') | |
477 | if self.daemondir is None: | |
478 | self.daemondir = '/usr/lib/frr' | |
479 | ||
480 | zebra_path = os.path.join(self.daemondir, 'zebra') | |
481 | if not os.path.isfile(zebra_path): | |
482 | raise Exception("FRR zebra binary doesn't exist at {}".format(zebra_path)) | |
483 | ||
484 | def _config_quagga(self, **params): | |
485 | "Configure Quagga binaries" | |
486 | self.daemondir = params.get('quaggadir') | |
487 | if self.daemondir is None: | |
488 | self.daemondir = '/usr/lib/quagga' | |
489 | ||
490 | zebra_path = os.path.join(self.daemondir, 'zebra') | |
491 | if not os.path.isfile(zebra_path): | |
492 | raise Exception("Quagga zebra binary doesn't exist at {}".format(zebra_path)) | |
493 | ||
2ab85530 RZ |
494 | # pylint: disable=W0221 |
495 | # Some params are only meaningful for the parent class. | |
594b1259 MW |
496 | def config(self, **params): |
497 | super(Router, self).config(**params) | |
498 | ||
2ab85530 RZ |
499 | # User did not specify the daemons directory, try to autodetect it. |
500 | self.daemondir = params.get('daemondir') | |
501 | if self.daemondir is None: | |
edd2bdf6 RZ |
502 | self.routertype = params.get('routertype', 'frr') |
503 | if self.routertype == 'quagga': | |
504 | self._config_quagga(**params) | |
505 | else: | |
506 | self._config_frr(**params) | |
594b1259 | 507 | else: |
2ab85530 RZ |
508 | # Test the provided path |
509 | zpath = os.path.join(self.daemondir, 'zebra') | |
510 | if not os.path.isfile(zpath): | |
511 | raise Exception('No zebra binary found in {}'.format(zpath)) | |
512 | # Allow user to specify routertype when the path was specified. | |
513 | if params.get('routertype') is not None: | |
514 | self.routertype = self.params.get('routertype') | |
515 | ||
594b1259 | 516 | # Enable forwarding on the router |
797e8dcf RZ |
517 | assert_sysctl(self, 'net.ipv4.ip_forward', 1) |
518 | assert_sysctl(self, 'net.ipv6.conf.all.forwarding', 1) | |
594b1259 | 519 | # Enable coredumps |
797e8dcf | 520 | assert_sysctl(self, 'kernel.core_uses_pid', 1) |
e1dfa45e LB |
521 | assert_sysctl(self, 'fs.suid_dumpable', 1) |
522 | #this applies to the kernel not the namespace... | |
523 | #original on ubuntu 17.x, but apport won't save as in namespace | |
524 | # |/usr/share/apport/apport %p %s %c %d %P | |
525 | corefile = '%e_core-sig_%s-pid_%p.dmp' | |
797e8dcf | 526 | assert_sysctl(self, 'kernel.core_pattern', corefile) |
594b1259 MW |
527 | self.cmd('ulimit -c unlimited') |
528 | # Set ownership of config files | |
2ab85530 RZ |
529 | self.cmd('chown {0}:{0}vty /etc/{0}'.format(self.routertype)) |
530 | ||
594b1259 MW |
531 | def terminate(self): |
532 | # Delete Running Quagga or FRR Daemons | |
99561211 MW |
533 | self.stopRouter() |
534 | # rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) | |
535 | # for d in StringIO.StringIO(rundaemons): | |
536 | # self.cmd('kill -7 `cat %s`' % d.rstrip()) | |
537 | # self.waitOutput() | |
594b1259 | 538 | # Disable forwarding |
797e8dcf RZ |
539 | set_sysctl(self, 'net.ipv4.ip_forward', 0) |
540 | set_sysctl(self, 'net.ipv6.conf.all.forwarding', 0) | |
594b1259 | 541 | super(Router, self).terminate() |
3a568b9c | 542 | def stopRouter(self, wait=True): |
99561211 MW |
543 | # Stop Running Quagga or FRR Daemons |
544 | rundaemons = self.cmd('ls -1 /var/run/%s/*.pid' % self.routertype) | |
e600b2d9 RZ |
545 | if re.search(r"No such file or directory", rundaemons): |
546 | return | |
99561211 | 547 | if rundaemons is not None: |
3a568b9c | 548 | numRunning = 0 |
7551168c MW |
549 | for d in StringIO.StringIO(rundaemons): |
550 | daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip() | |
551 | if (daemonpid.isdigit() and pid_exists(int(daemonpid))): | |
065bd557 RZ |
552 | logger.info('{}: stopping {}'.format( |
553 | self.name, | |
554 | os.path.basename(d.rstrip().rsplit(".", 1)[0]) | |
555 | )) | |
7551168c MW |
556 | self.cmd('kill -TERM %s' % daemonpid) |
557 | self.waitOutput() | |
3a568b9c LB |
558 | if pid_exists(int(daemonpid)): |
559 | numRunning += 1 | |
560 | if wait and numRunning > 0: | |
065bd557 | 561 | sleep(2, '{}: waiting for daemons stopping'.format(self.name)) |
3a568b9c LB |
562 | # 2nd round of kill if daemons didn't exit |
563 | for d in StringIO.StringIO(rundaemons): | |
564 | daemonpid = self.cmd('cat %s' % d.rstrip()).rstrip() | |
565 | if (daemonpid.isdigit() and pid_exists(int(daemonpid))): | |
065bd557 RZ |
566 | logger.info('{}: killing {}'.format( |
567 | self.name, | |
568 | os.path.basename(d.rstrip().rsplit(".", 1)[0]) | |
569 | )) | |
3a568b9c LB |
570 | self.cmd('kill -7 %s' % daemonpid) |
571 | self.waitOutput() | |
e600b2d9 | 572 | self.cmd('rm -- {}'.format(d.rstrip())) |
f76774ec LB |
573 | if wait: |
574 | self.checkRouterCores() | |
575 | ||
594b1259 MW |
576 | def removeIPs(self): |
577 | for interface in self.intfNames(): | |
578 | self.cmd('ip address flush', interface) | |
8dd5077d PG |
579 | |
580 | def checkCapability(self, daemon, param): | |
581 | if param is not None: | |
582 | daemon_path = os.path.join(self.daemondir, daemon) | |
583 | daemon_search_option = param.replace('-','') | |
584 | output = self.cmd('{0} -h | grep {1}'.format( | |
585 | daemon_path, daemon_search_option)) | |
586 | if daemon_search_option not in output: | |
587 | return False | |
588 | return True | |
589 | ||
590 | def loadConf(self, daemon, source=None, param=None): | |
594b1259 MW |
591 | # print "Daemons before:", self.daemons |
592 | if daemon in self.daemons.keys(): | |
593 | self.daemons[daemon] = 1 | |
8dd5077d PG |
594 | if param is not None: |
595 | self.daemons_options[daemon] = param | |
594b1259 MW |
596 | if source is None: |
597 | self.cmd('touch /etc/%s/%s.conf' % (self.routertype, daemon)) | |
598 | self.waitOutput() | |
599 | else: | |
600 | self.cmd('cp %s /etc/%s/%s.conf' % (source, self.routertype, daemon)) | |
601 | self.waitOutput() | |
602 | self.cmd('chmod 640 /etc/%s/%s.conf' % (self.routertype, daemon)) | |
603 | self.waitOutput() | |
604 | self.cmd('chown %s:%s /etc/%s/%s.conf' % (self.routertype, self.routertype, self.routertype, daemon)) | |
605 | self.waitOutput() | |
606 | else: | |
222ea88b | 607 | logger.info('No daemon {} known'.format(daemon)) |
594b1259 | 608 | # print "Daemons after:", self.daemons |
e1dfa45e | 609 | |
9711fc7e | 610 | def startRouter(self, tgen=None): |
594b1259 | 611 | # Disable integrated-vtysh-config |
a93477ec | 612 | self.cmd('echo "no service integrated-vtysh-config" >> /etc/%s/vtysh.conf' % self.routertype) |
594b1259 | 613 | self.cmd('chown %s:%svty /etc/%s/vtysh.conf' % (self.routertype, self.routertype, self.routertype)) |
13e1fc49 | 614 | # TODO remove the following lines after all tests are migrated to Topogen. |
594b1259 | 615 | # Try to find relevant old logfiles in /tmp and delete them |
e1dfa45e | 616 | map(os.remove, glob.glob('{}/{}/*.log'.format(self.logdir, self.name))) |
594b1259 | 617 | # Remove old core files |
e1dfa45e | 618 | map(os.remove, glob.glob('{}/{}/*.dmp'.format(self.logdir, self.name))) |
594b1259 MW |
619 | # Remove IP addresses from OS first - we have them in zebra.conf |
620 | self.removeIPs() | |
621 | # If ldp is used, check for LDP to be compiled and Linux Kernel to be 4.5 or higher | |
622 | # No error - but return message and skip all the tests | |
623 | if self.daemons['ldpd'] == 1: | |
2ab85530 RZ |
624 | ldpd_path = os.path.join(self.daemondir, 'ldpd') |
625 | if not os.path.isfile(ldpd_path): | |
222ea88b | 626 | logger.info("LDP Test, but no ldpd compiled or installed") |
594b1259 | 627 | return "LDP Test, but no ldpd compiled or installed" |
dd4eca4d | 628 | |
45619ee3 | 629 | if version_cmp(platform.release(), '4.5') < 0: |
222ea88b | 630 | logger.info("LDP Test need Linux Kernel 4.5 minimum") |
45619ee3 | 631 | return "LDP Test need Linux Kernel 4.5 minimum" |
9711fc7e LB |
632 | # Check if have mpls |
633 | if tgen != None: | |
634 | self.hasmpls = tgen.hasmpls | |
635 | if self.hasmpls != True: | |
636 | logger.info("LDP/MPLS Tests will be skipped, platform missing module(s)") | |
637 | else: | |
638 | # Test for MPLS Kernel modules available | |
639 | self.hasmpls = False | |
640 | if os.system('/sbin/modprobe mpls-router') != 0: | |
641 | logger.info('MPLS tests will not run (missing mpls-router kernel module)') | |
642 | elif os.system('/sbin/modprobe mpls-iptunnel') != 0: | |
643 | logger.info('MPLS tests will not run (missing mpls-iptunnel kernel module)') | |
644 | else: | |
645 | self.hasmpls = True | |
646 | if self.hasmpls != True: | |
647 | return "LDP/MPLS Tests need mpls kernel modules" | |
594b1259 | 648 | self.cmd('echo 100000 > /proc/sys/net/mpls/platform_labels') |
44a592b2 MW |
649 | |
650 | if self.daemons['eigrpd'] == 1: | |
651 | eigrpd_path = os.path.join(self.daemondir, 'eigrpd') | |
652 | if not os.path.isfile(eigrpd_path): | |
222ea88b | 653 | logger.info("EIGRP Test, but no eigrpd compiled or installed") |
44a592b2 MW |
654 | return "EIGRP Test, but no eigrpd compiled or installed" |
655 | ||
99561211 MW |
656 | self.restartRouter() |
657 | return "" | |
e1dfa45e | 658 | |
99561211 | 659 | def restartRouter(self): |
e1dfa45e LB |
660 | # Starts actual daemons without init (ie restart) |
661 | # cd to per node directory | |
662 | self.cmd('cd {}/{}'.format(self.logdir, self.name)) | |
594b1259 MW |
663 | # Start Zebra first |
664 | if self.daemons['zebra'] == 1: | |
2ab85530 | 665 | zebra_path = os.path.join(self.daemondir, 'zebra') |
8dd5077d | 666 | zebra_option = self.daemons_options['zebra'] |
e1dfa45e | 667 | self.cmd('{0} {1} > zebra.out 2> zebra.err &'.format( |
8dd5077d | 668 | zebra_path, zebra_option, self.logdir, self.name |
2ab85530 | 669 | )) |
594b1259 | 670 | self.waitOutput() |
6c131bd3 | 671 | logger.debug('{}: {} zebra started'.format(self, self.routertype)) |
63038f4b | 672 | sleep(1, '{}: waiting for zebra to start'.format(self.name)) |
594b1259 MW |
673 | # Fix Link-Local Addresses |
674 | # Somehow (on Mininet only), Zebra removes the IPv6 Link-Local addresses on start. Fix this | |
675 | self.cmd('for i in `ls /sys/class/net/` ; do mac=`cat /sys/class/net/$i/address`; IFS=\':\'; set $mac; unset IFS; ip address add dev $i scope link fe80::$(printf %02x $((0x$1 ^ 2)))$2:${3}ff:fe$4:$5$6/64; done') | |
676 | # Now start all the other daemons | |
677 | for daemon in self.daemons: | |
2ab85530 RZ |
678 | # Skip disabled daemons and zebra |
679 | if self.daemons[daemon] == 0 or daemon == 'zebra': | |
680 | continue | |
681 | ||
682 | daemon_path = os.path.join(self.daemondir, daemon) | |
e1dfa45e | 683 | self.cmd('{0} > {3}.out 2> {3}.err &'.format( |
13e1fc49 | 684 | daemon_path, self.logdir, self.name, daemon |
2ab85530 RZ |
685 | )) |
686 | self.waitOutput() | |
6c131bd3 | 687 | logger.debug('{}: {} {} started'.format(self, self.routertype, daemon)) |
99561211 MW |
688 | def getStdErr(self, daemon): |
689 | return self.getLog('err', daemon) | |
690 | def getStdOut(self, daemon): | |
691 | return self.getLog('out', daemon) | |
692 | def getLog(self, log, daemon): | |
e1dfa45e | 693 | return self.cmd('cat {}/{}/{}.{}'.format(self.logdir, self.name, daemon, log)) |
f76774ec LB |
694 | |
695 | def checkRouterCores(self, reportLeaks=True): | |
696 | for daemon in self.daemons: | |
697 | if (self.daemons[daemon] == 1): | |
698 | # Look for core file | |
699 | corefiles = glob.glob('{}/{}/{}_core*.dmp'.format( | |
700 | self.logdir, self.name, daemon)) | |
701 | if (len(corefiles) > 0): | |
702 | daemon_path = os.path.join(self.daemondir, daemon) | |
703 | backtrace = subprocess.check_output([ | |
704 | "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0]) | |
705 | ], shell=True) | |
706 | sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon)) | |
707 | sys.stderr.write("%s" % backtrace) | |
708 | elif reportLeaks: | |
709 | log = self.getStdErr(daemon) | |
710 | if "memstats" in log: | |
711 | sys.stderr.write("%s: %s has memory leaks:\n" % (self.name, daemon)) | |
712 | log = re.sub("core_handler: ", "", log) | |
713 | log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n ## \1", log) | |
714 | log = re.sub("memstats: ", " ", log) | |
715 | sys.stderr.write(log) | |
716 | # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found | |
717 | if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon): | |
718 | sys.stderr.write("%s: Daemon %s killed by AddressSanitizer" % (self.name, daemon)) | |
719 | ||
594b1259 | 720 | def checkRouterRunning(self): |
597cabb7 MW |
721 | "Check if router daemons are running and collect crashinfo they don't run" |
722 | ||
594b1259 MW |
723 | global fatal_error |
724 | ||
725 | daemonsRunning = self.cmd('vtysh -c "show log" | grep "Logging configuration for"') | |
4942f298 MW |
726 | # Look for AddressSanitizer Errors in vtysh output and append to /tmp/AddressSanitzer.txt if found |
727 | if checkAddressSanitizerError(daemonsRunning, self.name, "vtysh"): | |
728 | return "%s: vtysh killed by AddressSanitizer" % (self.name) | |
729 | ||
594b1259 MW |
730 | for daemon in self.daemons: |
731 | if (self.daemons[daemon] == 1) and not (daemon in daemonsRunning): | |
732 | sys.stderr.write("%s: Daemon %s not running\n" % (self.name, daemon)) | |
733 | # Look for core file | |
e1dfa45e | 734 | corefiles = glob.glob('{}/{}/{}_core*.dmp'.format( |
13e1fc49 | 735 | self.logdir, self.name, daemon)) |
594b1259 | 736 | if (len(corefiles) > 0): |
2ab85530 RZ |
737 | daemon_path = os.path.join(self.daemondir, daemon) |
738 | backtrace = subprocess.check_output([ | |
739 | "gdb {} {} --batch -ex bt 2> /dev/null".format(daemon_path, corefiles[0]) | |
740 | ], shell=True) | |
594b1259 MW |
741 | sys.stderr.write("\n%s: %s crashed. Core file found - Backtrace follows:\n" % (self.name, daemon)) |
742 | sys.stderr.write("%s\n" % backtrace) | |
743 | else: | |
744 | # No core found - If we find matching logfile in /tmp, then print last 20 lines from it. | |
e1dfa45e | 745 | if os.path.isfile('{}/{}/{}.log'.format(self.logdir, self.name, daemon)): |
13e1fc49 | 746 | log_tail = subprocess.check_output([ |
e1dfa45e | 747 | "tail -n20 {}/{}/{}.log 2> /dev/null".format( |
13e1fc49 RZ |
748 | self.logdir, self.name, daemon) |
749 | ], shell=True) | |
594b1259 MW |
750 | sys.stderr.write("\nFrom %s %s %s log file:\n" % (self.routertype, self.name, daemon)) |
751 | sys.stderr.write("%s\n" % log_tail) | |
4942f298 | 752 | |
597cabb7 | 753 | # Look for AddressSanitizer Errors and append to /tmp/AddressSanitzer.txt if found |
4942f298 | 754 | if checkAddressSanitizerError(self.getStdErr(daemon), self.name, daemon): |
84379e8e MW |
755 | return "%s: Daemon %s not running - killed by AddressSanitizer" % (self.name, daemon) |
756 | ||
594b1259 MW |
757 | return "%s: Daemon %s not running" % (self.name, daemon) |
758 | return "" | |
759 | def get_ipv6_linklocal(self): | |
760 | "Get LinkLocal Addresses from interfaces" | |
761 | ||
762 | linklocal = [] | |
763 | ||
764 | ifaces = self.cmd('ip -6 address') | |
765 | # Fix newlines (make them all the same) | |
766 | ifaces = ('\n'.join(ifaces.splitlines()) + '\n').splitlines() | |
767 | interface="" | |
768 | ll_per_if_count=0 | |
769 | for line in ifaces: | |
770 | m = re.search('[0-9]+: ([^:@]+)[@if0-9:]+ <', line) | |
771 | if m: | |
772 | interface = m.group(1) | |
773 | ll_per_if_count = 0 | |
774 | m = re.search('inet6 (fe80::[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+)[/0-9]* scope link', line) | |
775 | if m: | |
776 | local = m.group(1) | |
777 | ll_per_if_count += 1 | |
778 | if (ll_per_if_count > 1): | |
779 | linklocal += [["%s-%s" % (interface, ll_per_if_count), local]] | |
780 | else: | |
781 | linklocal += [[interface, local]] | |
782 | return linklocal | |
80eeefb7 MW |
783 | def daemon_available(self, daemon): |
784 | "Check if specified daemon is installed (and for ldp if kernel supports MPLS)" | |
785 | ||
2ab85530 RZ |
786 | daemon_path = os.path.join(self.daemondir, daemon) |
787 | if not os.path.isfile(daemon_path): | |
80eeefb7 MW |
788 | return False |
789 | if (daemon == 'ldpd'): | |
b431b554 MW |
790 | if version_cmp(platform.release(), '4.5') < 0: |
791 | return False | |
792 | if self.cmd('/sbin/modprobe -n mpls-router' ) != "": | |
80eeefb7 | 793 | return False |
b431b554 MW |
794 | if self.cmd('/sbin/modprobe -n mpls-iptunnel') != "": |
795 | return False | |
796 | ||
80eeefb7 MW |
797 | return True |
798 | def get_routertype(self): | |
799 | "Return the type of Router (frr or quagga)" | |
800 | ||
801 | return self.routertype | |
50c40bde MW |
802 | def report_memory_leaks(self, filename_prefix, testscript): |
803 | "Report Memory Leaks to file prefixed with given string" | |
804 | ||
805 | leakfound = False | |
806 | filename = filename_prefix + re.sub(r"\.py", "", testscript) + ".txt" | |
807 | for daemon in self.daemons: | |
808 | if (self.daemons[daemon] == 1): | |
809 | log = self.getStdErr(daemon) | |
810 | if "memstats" in log: | |
811 | # Found memory leak | |
6c131bd3 RZ |
812 | logger.info('\nRouter {} {} StdErr Log:\n{}'.format( |
813 | self.name, daemon, log)) | |
50c40bde MW |
814 | if not leakfound: |
815 | leakfound = True | |
816 | # Check if file already exists | |
817 | fileexists = os.path.isfile(filename) | |
818 | leakfile = open(filename, "a") | |
819 | if not fileexists: | |
820 | # New file - add header | |
821 | leakfile.write("# Memory Leak Detection for topotest %s\n\n" % testscript) | |
822 | leakfile.write("## Router %s\n" % self.name) | |
823 | leakfile.write("### Process %s\n" % daemon) | |
824 | log = re.sub("core_handler: ", "", log) | |
825 | log = re.sub(r"(showing active allocations in memory group [a-zA-Z0-9]+)", r"\n#### \1\n", log) | |
826 | log = re.sub("memstats: ", " ", log) | |
827 | leakfile.write(log) | |
828 | leakfile.write("\n") | |
829 | if leakfound: | |
830 | leakfile.close() | |
80eeefb7 | 831 | |
594b1259 MW |
832 | |
833 | class LegacySwitch(OVSSwitch): | |
834 | "A Legacy Switch without OpenFlow" | |
835 | ||
836 | def __init__(self, name, **params): | |
837 | OVSSwitch.__init__(self, name, failMode='standalone', **params) | |
838 | self.switchIP = None |