1 # -*- coding: utf-8 eval: (blacken-mode 1) -*-
2 # SPDX-License-Identifier: GPL-2.0-or-later
4 # July 9 2021, Christian Hopps <chopps@labn.net>
6 # Copyright 2021, LabN Consulting, L.L.C.
8 """A module that implements core functionality for library or standalone use."""
23 import time
as time_mod
25 from collections
import defaultdict
26 from pathlib
import Path
27 from typing
import Union
29 from . import config
as munet_config
36 from pexpect
.fdpexpect
import fdspawn
37 from pexpect
.popen_spawn
import PopenSpawn
43 PEXPECT_PROMPT
= "PEXPECT_PROMPT>"
44 PEXPECT_CONTINUATION_PROMPT
= "PEXPECT_PROMPT+"
46 root_hostname
= subprocess
.check_output("hostname")
50 class MunetError(Exception):
51 """A generic munet error."""
54 class CalledProcessError(subprocess
.CalledProcessError
):
55 """Improved logging subclass of subprocess.CalledProcessError."""
58 o
= self
.output
.strip() if self
.output
else ""
59 e
= self
.stderr
.strip() if self
.stderr
else ""
60 s
= f
"returncode: {self.returncode} command: {self.cmd}"
61 o
= "\n\tstdout: " + o
if o
else ""
62 e
= "\n\tstderr: " + e
if e
else ""
66 o
= self
.output
.strip() if self
.output
else ""
67 e
= self
.stderr
.strip() if self
.stderr
else ""
68 return f
"munet.base.CalledProcessError({self.returncode}, {self.cmd}, {o}, {e})"
72 """An object to passively monitor for timeouts."""
74 def __init__(self
, delta
):
75 self
.delta
= datetime
.timedelta(seconds
=delta
)
76 self
.started_on
= datetime
.datetime
.now()
77 self
.expires_on
= self
.started_on
+ self
.delta
80 elapsed
= datetime
.datetime
.now() - self
.started_on
81 return elapsed
.total_seconds()
84 return datetime
.datetime
.now() > self
.expires_on
87 remaining
= self
.expires_on
- datetime
.datetime
.now()
88 return remaining
.total_seconds()
94 remaining
= self
.remaining()
100 def fsafe_name(name
):
101 return "".join(x
if x
.isalnum() else "_" for x
in name
)
105 return "\t" + s
.replace("\n", "\n\t")
108 def shell_quote(command
):
109 """Return command wrapped in single quotes."""
110 if sys
.version_info
[0] >= 3:
111 return shlex
.quote(command
)
112 return "'" + command
.replace("'", "'\"'\"'") + "'"
115 def cmd_error(rc
, o
, e
):
117 o
= "\n\tstdout: " + o
.strip() if o
and o
.strip() else ""
118 e
= "\n\tstderr: " + e
.strip() if e
and e
.strip() else ""
123 if hasattr(p
, "args"):
124 args
= p
.args
if isinstance(p
.args
, str) else " ".join(p
.args
)
127 return f
"proc pid: {p.pid} args: {args}"
130 def proc_error(p
, o
, e
):
131 if hasattr(p
, "args"):
132 args
= p
.args
if isinstance(p
.args
, str) else " ".join(p
.args
)
136 s
= f
"rc {p.returncode} pid {p.pid}"
137 a
= "\n\targs: " + args
if args
else ""
138 o
= "\n\tstdout: " + (o
.strip() if o
and o
.strip() else "*empty*")
139 e
= "\n\tstderr: " + (e
.strip() if e
and e
.strip() else "*empty*")
145 assert rc
is not None
146 if not hasattr(p
, "saved_output"):
147 p
.saved_output
= p
.communicate()
148 return proc_error(p
, *p
.saved_output
)
151 async def acomm_error(p
):
153 assert rc
is not None
154 if not hasattr(p
, "saved_output"):
155 p
.saved_output
= await p
.communicate()
156 return proc_error(p
, *p
.saved_output
)
159 def get_kernel_version():
161 subprocess
.check_output("uname -r", shell
=True, text
=True).strip().split("-", 1)
163 kv
= kvs
[0].split(".")
164 kv
= [int(x
) for x
in kv
]
168 def convert_number(value
) -> int:
169 """Convert a number value with a possible suffix to an integer.
171 >>> convert_number("100k") == 100 * 1024
173 >>> convert_number("100M") == 100 * 1000 * 1000
175 >>> convert_number("100Gi") == 100 * 1024 * 1024 * 1024
177 >>> convert_number("55") == 55
181 raise ValueError("Invalid value None for convert_number")
188 index
= suffix
.find(rate
[-1])
191 index
= suffix
.lower().find(rate
[-1])
194 return int(rate
) * base
** (index
+ 1)
197 def is_file_like(fo
):
198 return isinstance(fo
, int) or hasattr(fo
, "fileno")
201 def get_tc_bits_value(user_value
):
202 value
= convert_number(user_value
) / 1000
203 return f
"{value:03f}kbit"
206 def get_tc_bytes_value(user_value
):
207 # Raw numbers are bytes in tc
208 return convert_number(user_value
)
211 def get_tmp_dir(uniq
):
212 return os
.path
.join(tempfile
.mkdtemp(), uniq
)
215 async def _async_get_exec_path(binary
, cmdf
, cache
):
216 if isinstance(binary
, str):
224 rc
, output
, _
= await cmdf("which " + b
, warn
=False)
226 cache
[b
] = os
.path
.abspath(output
.strip())
231 def _get_exec_path(binary
, cmdf
, cache
):
232 if isinstance(binary
, str):
240 rc
, output
, _
= cmdf("which " + b
, warn
=False)
242 cache
[b
] = os
.path
.abspath(output
.strip())
247 def get_event_loop():
248 """Configure and return our non-thread using event loop.
250 This function configures a new child watcher to not use threads.
251 Threads cannot be used when we inline unshare a PID namespace.
253 policy
= asyncio
.get_event_loop_policy()
254 loop
= policy
.get_event_loop()
255 owatcher
= policy
.get_child_watcher()
257 "event_loop_fixture: global policy %s, current loop %s, current watcher %s",
263 policy
.set_child_watcher(None)
267 watcher
= asyncio
.PidfdChildWatcher() # pylint: disable=no-member
269 watcher
= asyncio
.SafeChildWatcher()
270 loop
= policy
.get_event_loop()
273 "event_loop_fixture: attaching new watcher %s to loop and setting in policy",
276 watcher
.attach_loop(loop
)
277 policy
.set_child_watcher(watcher
)
278 policy
.set_event_loop(loop
)
279 assert asyncio
.get_event_loop_policy().get_child_watcher() is watcher
284 class Commander
: # pylint: disable=R0904
285 """An object that can execute commands."""
289 def __init__(self
, name
, logger
=None, unet
=None, **kwargs
):
290 """Create a Commander.
293 name: name of the commander object
294 logger: logger to use for logging commands a defualt is supplied if this
296 unet: unet that owns this object, only used by Commander in run_in_window,
297 otherwise can be None.
299 # del kwargs # deal with lint warning
300 # logging.warning("Commander: name %s kwargs %s", name, kwargs)
304 self
.deleting
= False
309 logname
= f
"munet.{self.__class__.__name__.lower()}.{name}"
310 self
.logger
= logging
.getLogger(logname
)
311 self
.logger
.setLevel(logging
.DEBUG
)
315 super().__init
__(**kwargs
)
322 def is_container(self
):
325 def set_logger(self
, logfile
):
326 self
.logger
= logging
.getLogger(__name__
+ ".commander." + self
.name
)
327 self
.logger
.setLevel(logging
.DEBUG
)
328 if isinstance(logfile
, str):
329 handler
= logging
.FileHandler(logfile
, mode
="w")
331 handler
= logging
.StreamHandler(logfile
)
333 fmtstr
= "%(asctime)s.%(msecs)03d %(levelname)s: {}({}): %(message)s".format(
334 self
.__class
__.__name
__, self
.name
336 handler
.setFormatter(logging
.Formatter(fmt
=fmtstr
))
337 self
.logger
.addHandler(handler
)
339 def _get_pre_cmd(self
, use_str
, use_pty
, **kwargs
):
340 """Get the pre-user-command values.
342 The values returned here should be what is required to cause the user's command
343 to execute in the correct context (e.g., namespace, container, sshremote).
347 return "" if use_str
else []
350 return f
"{self.__class__.__name__}({self.name})"
352 async def async_get_exec_path(self
, binary
):
353 """Return the full path to the binary executable.
355 `binary` :: binary name or list of binary names
357 return await _async_get_exec_path(
358 binary
, self
.async_cmd_status_nsonly
, self
.exec_paths
361 def get_exec_path(self
, binary
):
362 """Return the full path to the binary executable.
364 `binary` :: binary name or list of binary names
366 return _get_exec_path(binary
, self
.cmd_status_nsonly
, self
.exec_paths
)
368 def get_exec_path_host(self
, binary
):
369 """Return the full path to the binary executable.
371 If the object is actually a derived class (e.g., a container) this method will
372 return the exec path for the native namespace rather than the container. The
373 path is the one which the other xxx_host methods will use.
375 `binary` :: binary name or list of binary names
377 return get_exec_path_host(binary
)
379 def test(self
, flags
, arg
):
380 """Run test binary, with flags and arg."""
381 test_path
= self
.get_exec_path(["test"])
382 rc
, _
, _
= self
.cmd_status([test_path
, flags
, arg
], warn
=False)
385 def test_nsonly(self
, flags
, arg
):
386 """Run test binary, with flags and arg."""
387 test_path
= self
.get_exec_path(["test"])
388 rc
, _
, _
= self
.cmd_status_nsonly([test_path
, flags
, arg
], warn
=False)
391 def path_exists(self
, path
):
392 """Check if path exists."""
393 return self
.test("-e", path
)
395 async def cleanup_pid(self
, pid
, kill_pid
=None):
396 """Signal a pid to exit with escalating forcefulness."""
400 for sn
in (signal
.SIGHUP
, signal
.SIGKILL
):
402 "%s: %s %s (wait %s)", self
, signal
.Signals(sn
).name
, kill_pid
, pid
405 os
.kill(kill_pid
, sn
)
407 # No need to wait after this.
408 if sn
== signal
.SIGKILL
:
411 # try each signal, waiting 15 seconds for exit before advancing
413 self
.logger
.debug("%s: waiting %ss for pid to exit", self
, wait_sec
)
414 for _
in Timeout(wait_sec
):
416 status
= os
.waitpid(pid
, os
.WNOHANG
)
418 await asyncio
.sleep(0.1)
420 self
.logger
.debug("pid %s exited status %s", pid
, status
)
422 except OSError as error
:
423 if error
.errno
== errno
.ECHILD
:
424 self
.logger
.debug("%s: pid %s was reaped", self
, pid
)
427 "%s: error waiting on pid %s: %s", self
, pid
, error
430 self
.logger
.debug("%s: timeout waiting on pid %s to exit", self
, pid
)
432 def _get_sub_args(self
, cmd_list
, defaults
, use_pty
=False, ns_only
=False, **kwargs
):
433 """Returns pre-command, cmd, and default keyword args."""
434 assert not isinstance(cmd_list
, str)
436 defaults
["shell"] = False
437 pre_cmd_list
= self
._get
_pre
_cmd
(False, use_pty
, ns_only
=ns_only
, **kwargs
)
438 cmd_list
= [str(x
) for x
in cmd_list
]
440 # os_env = {k: v for k, v in os.environ.items() if k.startswith("MUNET")}
441 # env = {**os_env, **(kwargs["env"] if "env" in kwargs else {})}
442 env
= {**(kwargs
["env"] if "env" in kwargs
else os
.environ
)}
443 if "MUNET_NODENAME" not in env
:
444 env
["MUNET_NODENAME"] = self
.name
447 defaults
.update(kwargs
)
449 return pre_cmd_list
, cmd_list
, defaults
451 def _common_prologue(self
, async_exec
, method
, cmd
, skip_pre_cmd
=False, **kwargs
):
452 cmd_list
= self
._get
_cmd
_as
_list
(cmd
)
453 if method
== "_spawn":
456 "codec_errors": "ignore",
460 "stdout": subprocess
.PIPE
,
461 "stderr": subprocess
.PIPE
,
464 defaults
["encoding"] = "utf-8"
466 pre_cmd_list
, cmd_list
, defaults
= self
._get
_sub
_args
(
467 cmd_list
, defaults
, **kwargs
470 use_pty
= kwargs
.get("use_pty", False)
471 if method
== "_spawn":
472 # spawn doesn't take "shell" keyword arg
473 if "shell" in defaults
:
474 del defaults
["shell"]
475 # this is required to avoid receiving a STOPPED signal on expect!
477 defaults
["preexec_fn"] = os
.setsid
478 defaults
["env"]["PS1"] = "$ "
481 '%s: %s %s("%s", pre_cmd: "%s" use_pty: %s kwargs: %.120s)',
483 "XXX" if method
== "_spawn" else "",
486 pre_cmd_list
if not skip_pre_cmd
else "",
491 actual_cmd_list
= cmd_list
if skip_pre_cmd
else pre_cmd_list
+ cmd_list
492 return actual_cmd_list
, defaults
494 async def _async_popen(self
, method
, cmd
, **kwargs
):
495 """Create a new asynchronous subprocess."""
496 acmd
, kwargs
= self
._common
_prologue
(True, method
, cmd
, **kwargs
)
497 p
= await asyncio
.create_subprocess_exec(*acmd
, **kwargs
)
500 def _popen(self
, method
, cmd
, **kwargs
):
501 """Create a subprocess."""
502 acmd
, kwargs
= self
._common
_prologue
(False, method
, cmd
, **kwargs
)
503 p
= subprocess
.Popen(acmd
, **kwargs
)
506 def _fdspawn(self
, fo
, **kwargs
):
508 defaults
.update(kwargs
)
510 if "echo" in defaults
:
513 if "encoding" not in defaults
:
514 defaults
["encoding"] = "utf-8"
515 if "codec_errors" not in defaults
:
516 defaults
["codec_errors"] = "ignore"
517 encoding
= defaults
["encoding"]
519 self
.logger
.debug("%s: _fdspawn(%s, kwargs: %s)", self
, fo
, defaults
)
521 p
= fdspawn(fo
, **defaults
)
523 # We don't have TTY like conversions of LF to CRLF
524 p
.crlf
= os
.linesep
.encode(encoding
)
526 # we own the socket now detach the file descriptor to keep it from closing
527 if hasattr(fo
, "detach"):
532 def _spawn(self
, cmd
, skip_pre_cmd
=False, use_pty
=False, echo
=False, **kwargs
):
534 '%s: XXX _spawn: cmd "%s" skip_pre_cmd %s use_pty %s echo %s kwargs %s',
542 actual_cmd
, defaults
= self
._common
_prologue
(
543 False, "_spawn", cmd
, skip_pre_cmd
=skip_pre_cmd
, use_pty
=use_pty
, **kwargs
547 '%s: XXX %s("%s", use_pty %s echo %s defaults: %s)',
549 "PopenSpawn" if not use_pty
else "pexpect.spawn",
556 # We don't specify a timeout it defaults to 30s is that OK?
558 p
= PopenSpawn(actual_cmd
, **defaults
)
560 p
= pexpect
.spawn(actual_cmd
[0], actual_cmd
[1:], echo
=echo
, **defaults
)
576 """Create a spawned send/expect process.
579 cmd: list of args to exec/popen with, or an already open socket
580 spawned_re: what to look for to know when done, `spawn` returns when seen
581 expects: a list of regex other than `spawned_re` to look for. Commonly,
582 "ogin:" or "[Pp]assword:"r.
583 sends: what to send when an element of `expects` matches. So e.g., the
584 username or password if thats what corresponding expect matched. Can
585 be the empty string to send nothing.
586 use_pty: true for pty based expect, otherwise uses popen (pipes/files)
587 trace: if true then log send/expects
588 **kwargs - kwargs passed on the _spawn.
594 pexpect.TIMEOUT, pexpect.EOF as documented in `pexpect`
595 CalledProcessError if EOF is seen and `cmd` exited then
596 raises a CalledProcessError to indicate the failure.
598 if is_file_like(cmd
):
601 p
= self
._fdspawn
(cmd
, **kwargs
)
603 p
, ac
= self
._spawn
(cmd
, use_pty
=use_pty
, **kwargs
)
608 p
.logfile_read
= logfile_read
610 p
.logfile_send
= logfile_send
612 # for spawned shells (i.e., a direct command an not a console)
613 # this is wrong and will cause 2 prompts
615 # This isn't very nice looking
617 if not is_file_like(cmd
):
618 p
.isalive
= lambda: p
.proc
.poll() is None
619 if not hasattr(p
, "close"):
622 # Do a quick check to see if we got the prompt right away, otherwise we may be
623 # at a console so we send a \n to re-issue the prompt
624 index
= p
.expect([spawned_re
, pexpect
.TIMEOUT
, pexpect
.EOF
], timeout
=0.1)
626 assert p
.match
is not None
628 "%s: got spawned_re quick: '%s' matching '%s'",
635 # Now send a CRLF to cause the prompt (or whatever else) to re-issue
638 patterns
= [spawned_re
, *expects
]
640 self
.logger
.debug("%s: expecting: %s", self
, patterns
)
642 while index
:= p
.expect(patterns
):
644 assert p
.match
is not None
646 "%s: got expect: '%s' matching %d '%s', sending '%s'",
654 p
.send(sends
[index
- 1])
656 self
.logger
.debug("%s: expecting again: %s", self
, patterns
)
658 "%s: got spawned_re: '%s' matching '%s'",
664 except pexpect
.TIMEOUT
:
666 "%s: TIMEOUT looking for spawned_re '%s' expect buffer so far:\n%s",
672 except pexpect
.EOF
as eoferr
:
676 before
= indent(p
.before
)
677 error
= CalledProcessError(rc
, ac
, output
=before
)
679 "%s: EOF looking for spawned_re '%s' before EOF:\n%s",
685 raise error
from eoferr
687 async def shell_spawn(
699 """Create a shell REPL (read-eval-print-loop).
702 cmd: shell and list of args to popen with, or an already open socket
703 prompt: the REPL prompt to look for, the function returns when seen
704 expects: a list of regex other than `spawned_re` to look for. Commonly,
705 "ogin:" or "[Pp]assword:"r.
706 sends: what to send when an element of `expects` matches. So e.g., the
707 username or password if thats what corresponding expect matched. Can
708 be the empty string to send nothing.
709 is_bourne: if False then do not modify shell prompt for internal
710 parser friently format, and do not expect continuation prompts.
711 init_newline: send an initial newline for non-bourne shell spawns, otherwise
712 expect the prompt simply from running the command
713 use_pty: true for pty based expect, otherwise uses popen (pipes/files)
714 will_echo: bash is buggy in that it echo's to non-tty unlike any other
715 sh/ksh, set this value to true if running back
716 **kwargs - kwargs passed on the _spawn.
718 combined_prompt
= r
"({}|{})".format(re
.escape(PEXPECT_PROMPT
), prompt
)
720 assert not is_file_like(cmd
) or not use_pty
735 return ShellWrapper(p
, prompt
, will_echo
=will_echo
)
738 ps2
= PEXPECT_CONTINUATION_PROMPT
740 # Avoid problems when =/usr/bin/env= prints the values
741 ps1p
= ps1
[:5] + "${UNSET_V}" + ps1
[5:]
742 ps2p
= ps2
[:5] + "${UNSET_V}" + ps2
[5:]
747 extra
= "PAGER=cat; export PAGER; TERM=dumb; unset HISTFILE; set +o emacs +o vi"
748 pchg
= "PS1='{0}' PS2='{1}' PROMPT_COMMAND=''\n".format(ps1p
, ps2p
)
750 return ShellWrapper(p
, ps1
, ps2
, extra_init_cmd
=extra
, will_echo
=will_echo
)
752 def popen(self
, cmd
, **kwargs
):
753 """Creates a pipe with the given `command`.
756 cmd: `str` or `list` of command to open a pipe with.
757 **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
758 then will be invoked with `bash -c`, otherwise `command` is a list and
759 will be invoked without a shell.
762 a subprocess.Popen object.
764 return self
._popen
("popen", cmd
, **kwargs
)[0]
766 def popen_nsonly(self
, cmd
, **kwargs
):
767 """Creates a pipe with the given `command`.
770 cmd: `str` or `list` of command to open a pipe with.
771 **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
772 then will be invoked with `bash -c`, otherwise `command` is a list and
773 will be invoked without a shell.
776 a subprocess.Popen object.
778 return self
._popen
("popen_nsonly", cmd
, ns_only
=True, **kwargs
)[0]
780 async def async_popen(self
, cmd
, **kwargs
):
781 """Creates a pipe with the given `command`.
784 cmd: `str` or `list` of command to open a pipe with.
785 **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
786 `command` is a string then will be invoked with `bash -c`, otherwise
787 `command` is a list and will be invoked without a shell.
790 a asyncio.subprocess.Process object.
792 p
, _
= await self
._async
_popen
("async_popen", cmd
, **kwargs
)
795 async def async_popen_nsonly(self
, cmd
, **kwargs
):
796 """Creates a pipe with the given `command`.
799 cmd: `str` or `list` of command to open a pipe with.
800 **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
801 `command` is a string then will be invoked with `bash -c`, otherwise
802 `command` is a list and will be invoked without a shell.
805 a asyncio.subprocess.Process object.
807 p
, _
= await self
._async
_popen
(
808 "async_popen_nsonly", cmd
, ns_only
=True, **kwargs
812 async def async_cleanup_proc(self
, p
, pid
=None):
813 """Terminate a process started with a popen call.
816 p: return value from :py:`async_popen`, :py:`popen`, et al.
817 pid: pid to signal instead of p.pid, typically a child of
821 None on success, the ``p`` if multiple timeouts occur even
822 after a SIGKILL sent.
827 if p
.returncode
is not None:
828 if isinstance(p
, subprocess
.Popen
):
829 o
, e
= p
.communicate()
831 o
, e
= await p
.communicate()
833 "%s: cmd_p already exited status: %s", self
, proc_error(p
, o
, e
)
840 self
.logger
.debug("%s: terminate process: %s (pid %s)", self
, proc_str(p
), pid
)
842 # This will SIGHUP and wait a while then SIGKILL and return immediately
843 await self
.cleanup_pid(p
.pid
, pid
)
845 # Wait another 2 seconds after the possible SIGKILL above for the
846 # parent nsenter to cleanup and exit
848 if isinstance(p
, subprocess
.Popen
):
849 o
, e
= p
.communicate(timeout
=wait_secs
)
851 o
, e
= await asyncio
.wait_for(p
.communicate(), timeout
=wait_secs
)
853 "%s: cmd_p exited after kill, status: %s", self
, proc_error(p
, o
, e
)
855 except (asyncio
.TimeoutError
, subprocess
.TimeoutExpired
):
856 self
.logger
.warning("%s: SIGKILL timeout", self
)
858 except Exception as error
:
860 "%s: kill unexpected exception: %s", self
, error
, exc_info
=True
866 def _cmd_status_input(stdin
):
868 if isinstance(stdin
, (bytes
, str)):
870 stdin
= subprocess
.PIPE
873 def _cmd_status_finish(self
, p
, c
, ac
, o
, e
, raises
, warn
):
875 self
.last
= (rc
, ac
, c
, o
, e
)
878 self
.logger
.warning("%s: proc failed: %s", self
, proc_error(p
, o
, e
))
880 # error = Exception("stderr: {}".format(stderr))
881 # This annoyingly doesnt' show stderr when printed normally
882 raise CalledProcessError(rc
, ac
, o
, e
)
885 def _cmd_status(self
, cmds
, raises
=False, warn
=True, stdin
=None, **kwargs
):
886 """Execute a command."""
887 pinput
, stdin
= Commander
._cmd
_status
_input
(stdin
)
888 p
, actual_cmd
= self
._popen
("cmd_status", cmds
, stdin
=stdin
, **kwargs
)
889 o
, e
= p
.communicate(pinput
)
890 return self
._cmd
_status
_finish
(p
, cmds
, actual_cmd
, o
, e
, raises
, warn
)
892 async def _async_cmd_status(
893 self
, cmds
, raises
=False, warn
=True, stdin
=None, text
=None, **kwargs
895 """Execute a command."""
896 pinput
, stdin
= Commander
._cmd
_status
_input
(stdin
)
897 p
, actual_cmd
= await self
._async
_popen
(
898 "async_cmd_status", cmds
, stdin
=stdin
, **kwargs
904 encoding
= kwargs
.get("encoding", "utf-8")
906 if encoding
is not None and isinstance(pinput
, str):
907 pinput
= pinput
.encode(encoding
)
908 o
, e
= await p
.communicate(pinput
)
909 if encoding
is not None:
910 o
= o
.decode(encoding
) if o
is not None else o
911 e
= e
.decode(encoding
) if e
is not None else e
912 return self
._cmd
_status
_finish
(p
, cmds
, actual_cmd
, o
, e
, raises
, warn
)
914 def _get_cmd_as_list(self
, cmd
):
915 """Given a list or string return a list form for execution.
917 If `cmd` is a string then the returned list uses bash and looks
918 like this: ["/bin/bash", "-c", cmd]. Some node types override
919 this function if they utilize a different shell as to return
920 a different list of values.
923 cmd: list or string representing the command to execute.
926 list of commands to execute.
928 if not isinstance(cmd
, str):
931 # Make sure the code doesn't think `cd` will work.
932 assert not re
.match(r
"cd(\s*|\s+(\S+))$", cmd
)
933 cmds
= ["/bin/bash", "-c", cmd
]
936 def cmd_nostatus(self
, cmd
, **kwargs
):
937 """Run given command returning output[s].
940 cmd: `str` or `list` of the command to execute. If a string is given
941 it is run using a shell, otherwise the list is executed directly
942 as the binary and arguments.
943 **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
944 then will be invoked with `bash -c`, otherwise `command` is a list and
945 will be invoked without a shell.
948 if "stderr" is in kwargs and not equal to subprocess.STDOUT, then
949 both stdout and stderr are returned, otherwise stderr is combined
950 with stdout and only stdout is returned.
953 # This method serves as the basis for all derived sync cmd variations, so to
954 # override sync cmd behavior simply override this function and *not* the other
955 # variations, unless you are changing only that variation's behavior
958 # XXX change this back to _cmd_status instead of cmd_status when we
959 # consolidate and cleanup the container overrides of *cmd_* functions
962 if "stderr" in kwargs
and kwargs
["stderr"] != subprocess
.STDOUT
:
963 _
, o
, e
= self
.cmd_status(cmds
, **kwargs
)
965 if "stderr" in kwargs
:
967 _
, o
, _
= self
.cmd_status(cmds
, stderr
=subprocess
.STDOUT
, **kwargs
)
970 def cmd_status(self
, cmd
, **kwargs
):
971 """Run given command returning status and outputs.
974 cmd: `str` or `list` of the command to execute. If a string is given
975 it is run using a shell, otherwise the list is executed directly
976 as the binary and arguments.
977 **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
978 then will be invoked with `bash -c`, otherwise `command` is a list and
979 will be invoked without a shell.
982 (status, output, error) are returned
983 status: the returncode of the command.
984 output: stdout as a string from the command.
985 error: stderr as a string from the command.
988 # This method serves as the basis for all derived sync cmd variations, so to
989 # override sync cmd behavior simply override this function and *not* the other
990 # variations, unless you are changing only that variation's behavior
992 return self
._cmd
_status
(cmd
, **kwargs
)
994 def cmd_raises(self
, cmd
, **kwargs
):
995 """Execute a command. Raise an exception on errors.
998 cmd: `str` or `list` of the command to execute. If a string is given
999 it is run using a shell, otherwise the list is executed directly
1000 as the binary and arguments.
1001 **kwargs: kwargs is eventually passed on to Popen. If `command` is a string
1002 then will be invoked with `bash -c`, otherwise `command` is a list and
1003 will be invoked without a shell.
1006 output: stdout as a string from the command.
1009 CalledProcessError: on non-zero exit status
1011 _
, stdout
, _
= self
._cmd
_status
(cmd
, raises
=True, **kwargs
)
1014 def cmd_nostatus_nsonly(self
, cmd
, **kwargs
):
1015 # Make sure the command runs on the host and not in any container.
1016 return self
.cmd_nostatus(cmd
, ns_only
=True, **kwargs
)
1018 def cmd_status_nsonly(self
, cmd
, **kwargs
):
1019 # Make sure the command runs on the host and not in any container.
1020 return self
._cmd
_status
(cmd
, ns_only
=True, **kwargs
)
1022 def cmd_raises_nsonly(self
, cmd
, **kwargs
):
1023 # Make sure the command runs on the host and not in any container.
1024 _
, stdout
, _
= self
._cmd
_status
(cmd
, raises
=True, ns_only
=True, **kwargs
)
1027 async def async_cmd_status(self
, cmd
, **kwargs
):
1028 """Run given command returning status and outputs.
1031 cmd: `str` or `list` of the command to execute. If a string is given
1032 it is run using a shell, otherwise the list is executed directly
1033 as the binary and arguments.
1034 **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
1035 `cmd` is a string then will be invoked with `bash -c`, otherwise
1036 `cmd` is a list and will be invoked without a shell.
1039 (status, output, error) are returned
1040 status: the returncode of the command.
1041 output: stdout as a string from the command.
1042 error: stderr as a string from the command.
1045 # This method serves as the basis for all derived async cmd variations, so to
1046 # override async cmd behavior simply override this function and *not* the other
1047 # variations, unless you are changing only that variation's behavior
1049 return await self
._async
_cmd
_status
(cmd
, **kwargs
)
1051 async def async_cmd_nostatus(self
, cmd
, **kwargs
):
1052 """Run given command returning output[s].
1055 cmd: `str` or `list` of the command to execute. If a string is given
1056 it is run using a shell, otherwise the list is executed directly
1057 as the binary and arguments.
1058 **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
1059 `cmd` is a string then will be invoked with `bash -c`, otherwise
1060 `cmd` is a list and will be invoked without a shell.
1063 if "stderr" is in kwargs and not equal to subprocess.STDOUT, then
1064 both stdout and stderr are returned, otherwise stderr is combined
1065 with stdout and only stdout is returned.
1068 # XXX change this back to _async_cmd_status instead of cmd_status when we
1069 # consolidate and cleanup the container overrides of *cmd_* functions
1072 if "stderr" in kwargs
and kwargs
["stderr"] != subprocess
.STDOUT
:
1073 _
, o
, e
= await self
._async
_cmd
_status
(cmds
, **kwargs
)
1075 if "stderr" in kwargs
:
1076 del kwargs
["stderr"]
1077 _
, o
, _
= await self
._async
_cmd
_status
(cmds
, stderr
=subprocess
.STDOUT
, **kwargs
)
1080 async def async_cmd_raises(self
, cmd
, **kwargs
):
1081 """Execute a command. Raise an exception on errors.
1084 cmd: `str` or `list` of the command to execute. If a string is given
1085 it is run using a shell, otherwise the list is executed directly
1086 as the binary and arguments.
1087 **kwargs: kwargs is eventually passed on to create_subprocess_exec. If
1088 `cmd` is a string then will be invoked with `bash -c`, otherwise
1089 `cmd` is a list and will be invoked without a shell.
1092 output: stdout as a string from the command.
1095 CalledProcessError: on non-zero exit status
1097 _
, stdout
, _
= await self
._async
_cmd
_status
(cmd
, raises
=True, **kwargs
)
1100 async def async_cmd_status_nsonly(self
, cmd
, **kwargs
):
1101 # Make sure the command runs on the host and not in any container.
1102 return await self
._async
_cmd
_status
(cmd
, ns_only
=True, **kwargs
)
1104 async def async_cmd_raises_nsonly(self
, cmd
, **kwargs
):
1105 # Make sure the command runs on the host and not in any container.
1106 _
, stdout
, _
= await self
._async
_cmd
_status
(
1107 cmd
, raises
=True, ns_only
=True, **kwargs
1111 def cmd_legacy(self
, cmd
, **kwargs
):
1112 """Execute a command with stdout and stderr joined, *IGNORES ERROR*."""
1113 defaults
= {"stderr": subprocess
.STDOUT
}
1114 defaults
.update(kwargs
)
1115 _
, stdout
, _
= self
._cmd
_status
(cmd
, raises
=False, **defaults
)
1118 # Run a command in a new window (gnome-terminal, screen, tmux, xterm)
1131 """Run a command in a new window (TMUX, Screen or XTerm).
1134 cmd: string to execute.
1135 wait_for: True to wait for exit from command or `str` as channel neme to
1136 signal on exit, otherwise False
1137 background: Do not change focus to new window.
1138 title: Title for new pane (tmux) or window (xterm).
1139 name: Name of the new window (tmux)
1140 forcex: Force use of X11.
1141 new_window: Open new window (instead of pane) in TMUX
1142 tmux_target: Target for tmux pane.
1145 the pane/window identifier from TMUX (depends on `new_window`)
1148 if isinstance(wait_for
, str):
1150 elif wait_for
is True:
1151 channel
= "{}-wait-{}".format(our_pid
, Commander
.tmux_wait_gen
)
1152 Commander
.tmux_wait_gen
+= 1
1154 if forcex
or ("TMUX" not in os
.environ
and "STY" not in os
.environ
):
1159 # SUDO: The important thing to note is that with all these methods we are
1160 # executing on the users windowing system, so even though we are normally
1161 # running as root, we will not be when the command is dispatched. Also
1162 # in the case of SCREEN and X11 we need to sudo *back* to the user as well
1163 # This is also done by SSHRemote by defualt so we should *not* sudo back
1164 # if we are SSHRemote.
1166 # XXX need to test ssh in screen
1167 # XXX need to test ssh in Xterm
1168 sudo_path
= get_exec_path_host(["sudo"])
1169 # This first test case seems same as last but using list instead of string?
1170 if self
.is_vm
and self
.use_ssh
: # pylint: disable=E1101
1171 if isinstance(cmd
, str):
1172 cmd
= shlex
.split(cmd
)
1173 cmd
= ["/usr/bin/env", f
"MUNET_NODENAME={self.name}"] + cmd
1176 cmd
= self
._get
_pre
_cmd
(False, True, ns_only
=ns_only
) + [shlex
.join(cmd
)]
1177 unet
= self
.unet
# pylint: disable=E1101
1178 uns_cmd
= unet
._get
_pre
_cmd
( # pylint: disable=W0212
1179 False, True, ns_only
=True, root_level
=root_level
1181 # get the nsenter for munet
1188 # This is the command to execute to be inside the namespace.
1189 # We are getting into trouble with quoting.
1190 # Why aren't we passing in MUNET_RUNDIR?
1191 cmd
= f
"/usr/bin/env MUNET_NODENAME={self.name} {cmd}"
1192 # We need sudo b/c we are executing as the user inside the window system.
1193 sudo_path
= get_exec_path_host(["sudo"])
1197 + self
._get
_pre
_cmd
(True, True, ns_only
=ns_only
, root_level
=root_level
)
1202 if "TMUX" in os
.environ
and not forcex
:
1203 cmd
= [get_exec_path_host("tmux")]
1205 cmd
.append("new-window")
1212 cmd
.append(tmux_target
)
1214 cmd
.append("split-window")
1218 tmux_target
= os
.getenv("TMUX_PANE", "")
1223 cmd
.append(tmux_target
)
1225 # nscmd is always added as single string argument
1226 if not isinstance(nscmd
, str):
1227 nscmd
= shlex
.join(nscmd
)
1229 nscmd
= f
"printf '\033]2;{title}\033\\'; {nscmd}"
1231 nscmd
= f
'trap "tmux wait -S {channel}; exit 0" EXIT; {nscmd}'
1234 elif "STY" in os
.environ
and not forcex
:
1235 # wait for not supported in screen for now
1237 cmd
= [get_exec_path_host("screen")]
1238 if not os
.path
.exists(
1239 "/run/screen/S-{}/{}".format(os
.environ
["USER"], os
.environ
["STY"])
1241 # XXX not appropriate for ssh
1242 cmd
= ["sudo", "-Eu", os
.environ
["SUDO_USER"]] + cmd
1248 if isinstance(nscmd
, str):
1249 nscmd
= shlex
.split(nscmd
)
1251 elif "DISPLAY" in os
.environ
:
1252 cmd
= [get_exec_path_host("xterm")]
1253 if "SUDO_USER" in os
.environ
:
1254 # Do this b/c making things work as root with xauth seems hard
1256 get_exec_path_host("sudo"),
1258 os
.environ
["SUDO_USER"],
1265 if isinstance(nscmd
, str):
1266 cmd
.extend(shlex
.split(nscmd
))
1271 # return self.cmd_raises(cmd, skip_pre_cmd=True)
1273 p
= commander
.popen(
1275 # skip_pre_cmd=True,
1279 # We should reap the child and report the error then.
1281 if p
.poll() is not None:
1282 self
.logger
.error("%s: Failed to launch xterm: %s", self
, comm_error(p
))
1286 "DISPLAY, STY, and TMUX not in environment, can't open window"
1288 raise Exception("Window requestd but TMUX, Screen and X11 not available")
1290 # pane_info = self.cmd_raises(cmd, skip_pre_cmd=True, ns_only=True).strip()
1291 # We are prepending the nsenter command, so use unet.rootcmd
1292 pane_info
= commander
.cmd_raises(cmd
).strip()
1294 # Re-adjust the layout
1295 if "TMUX" in os
.environ
:
1297 get_exec_path_host("tmux"),
1300 pane_info
if not tmux_target
else tmux_target
,
1303 commander
.cmd_status(cmd
)
1305 # Wait here if we weren't handed the channel to wait for
1306 if channel
and wait_for
is True:
1307 cmd
= [get_exec_path_host("tmux"), "wait", channel
]
1308 # commander.cmd_status(cmd, skip_pre_cmd=True)
1309 commander
.cmd_status(cmd
)
1314 """Calls self.async_delete within an exec loop."""
1315 asyncio
.run(self
.async_delete())
1317 async def _async_delete(self
):
1318 """Delete this objects resources.
1320 This is the actual implementation of the resource cleanup, each class
1321 should cleanup it's own resources, generally catching and reporting,
1322 but not reraising any exceptions for it's own cleanup, then it should
1323 invoke `super()._async_delete() without catching any exceptions raised
1324 therein. See other examples in `base.py` or `native.py`
1326 self
.logger
.info("%s: deleted", self
)
1328 async def async_delete(self
):
1329 """Delete the Commander (or derived object).
1331 The actual implementation for any class should be in `_async_delete`
1332 new derived classes should look at the documentation for that function.
1335 self
.deleting
= True
1336 await self
._async
_delete
()
1337 except Exception as error
:
1338 self
.logger
.error("%s: error while deleting: %s", self
, error
)
1341 class InterfaceMixin
:
1342 """A mixin class to support interface functionality."""
1344 def __init__(self
, *args
, **kwargs
):
1345 # del kwargs # get rid of lint
1346 # logging.warning("InterfaceMixin: args: %s kwargs: %s", args, kwargs)
1348 self
._intf
_addrs
= defaultdict(lambda: [None, None])
1350 self
.next_intf_index
= 0
1351 self
.basename
= "eth"
1352 # self.basename = name + "-eth"
1353 super().__init
__(*args
, **kwargs
)
1357 return sorted(self
._intf
_addrs
.keys())
1361 return sorted(self
.net_intfs
.keys())
1363 def get_intf_addr(self
, ifname
, ipv6
=False):
1364 if ifname
not in self
._intf
_addrs
:
1366 return self
._intf
_addrs
[ifname
][bool(ipv6
)]
1368 def set_intf_addr(self
, ifname
, ifaddr
):
1369 ifaddr
= ipaddress
.ip_interface(ifaddr
)
1370 self
._intf
_addrs
[ifname
][ifaddr
.version
== 6] = ifaddr
1372 def net_addr(self
, netname
, ipv6
=False):
1373 if netname
not in self
.net_intfs
:
1375 return self
.get_intf_addr(self
.net_intfs
[netname
], ipv6
=ipv6
)
1377 def set_intf_basename(self
, basename
):
1378 self
.basename
= basename
1380 def get_next_intf_name(self
):
1382 ifname
= self
.basename
+ str(self
.next_intf_index
)
1383 self
.next_intf_index
+= 1
1384 if ifname
not in self
._intf
_addrs
:
1388 def get_ns_ifname(self
, ifname
):
1389 """Return a namespace unique interface name.
1391 This function is primarily overriden by L3QemuVM, IOW by any class
1392 that doesn't create it's own network namespace and will share that
1393 with the root (unet) namespace.
1396 ifname: the interface name.
1399 A name unique to the namespace of this object. By defualt the assumption
1400 is the ifname is namespace unique.
1404 def register_interface(self
, ifname
):
1405 if ifname
not in self
._intf
_addrs
:
1406 self
._intf
_addrs
[ifname
] = [None, None]
1408 def register_network(self
, netname
, ifname
):
1409 if netname
in self
.net_intfs
:
1410 assert self
.net_intfs
[netname
] == ifname
1412 self
.net_intfs
[netname
] = ifname
1414 def get_linux_tc_args(self
, ifname
, config
):
1415 """Get interface constraints (jitter, delay, rate) for linux TC.
1417 The keys and their values are as follows:
1419 delay (int): number of microseconds
1420 jitter (int): number of microseconds
1421 jitter-correlation (float): % correlation to previous (default 10%)
1422 loss (float): % of loss
1423 loss-correlation (float): % correlation to previous (default 0%)
1424 rate (int or str): bits per second, string allows for use of
1425 {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000
1431 def get_number(c
, v
, d
=None):
1432 if v
not in c
or c
[v
] is None:
1434 return convert_number(c
[v
])
1436 delay
= get_number(config
, "delay")
1437 if delay
is not None:
1438 netem_args
+= f
" delay {delay}usec"
1440 jitter
= get_number(config
, "jitter")
1441 if jitter
is not None:
1443 raise ValueError("jitter but no delay specified")
1444 jitter_correlation
= get_number(config
, "jitter-correlation", 10)
1445 netem_args
+= f
" {jitter}usec {jitter_correlation}%"
1447 loss
= get_number(config
, "loss")
1448 if loss
is not None:
1449 loss_correlation
= get_number(config
, "loss-correlation", 0)
1450 if loss_correlation
:
1451 netem_args
+= f
" loss {loss}% {loss_correlation}%"
1453 netem_args
+= f
" loss {loss}%"
1455 if (o_rate
:= config
.get("rate")) is None:
1456 return netem_args
, ""
1459 # This comment is not correct, but is trying to talk through/learn the
1462 # tokens arrive at `rate` into token buffer.
1463 # limit - number of bytes that can be queued waiting for tokens
1465 # latency - maximum amount of time a packet may sit in TBF queue
1467 # So this just allows receiving faster than rate for latency amount of
1468 # time, before dropping.
1470 # latency = sizeofbucket(limit) / rate (peakrate?)
1473 # -------- = latency = 320ms
1477 # burst ([token] buffer) the largest number of instantaneous
1478 # tokens available (i.e, bucket size).
1484 tc_rate
= o_rate
["rate"]
1485 tc_rate
= convert_number(tc_rate
)
1486 limit
= convert_number(o_rate
.get("limit", DEFLIMIT
))
1487 burst
= convert_number(o_rate
.get("burst", DEFBURST
))
1488 except (KeyError, TypeError):
1489 tc_rate
= convert_number(o_rate
)
1490 limit
= convert_number(DEFLIMIT
)
1491 burst
= convert_number(DEFBURST
)
1492 tbf_args
+= f
" rate {tc_rate/1000}kbit"
1494 # give an extra 1/10 of buffer space to handle delay
1495 tbf_args
+= f
" limit {limit} burst {burst}"
1497 tbf_args
+= f
" limit {limit} burst {burst}"
1499 return netem_args
, tbf_args
1501 def set_intf_constraints(self
, ifname
, **constraints
):
1502 """Set interface outbound constraints.
1504 Set outbound constraints (jitter, delay, rate) for an interface. All arguments
1505 may also be passed as a string and will be converted to numerical format. All
1506 arguments are also optional. If not specified then that existing constraint will
1510 ifname: the name of the interface
1511 delay (int): number of microseconds.
1512 jitter (int): number of microseconds.
1513 jitter-correlation (float): Percent correlation to previous (default 10%).
1514 loss (float): Percent of loss.
1515 loss-correlation (float): Percent correlation to previous (default 25%).
1516 rate (int): bits per second, string allows for use of
1517 {KMGTKiMiGiTi} prefixes "i" means K == 1024 otherwise K == 1000.
1519 nsifname
= self
.get_ns_ifname(ifname
)
1520 netem_args
, tbf_args
= self
.get_linux_tc_args(nsifname
, constraints
)
1522 selector
= f
"root handle {count}:"
1525 f
"tc qdisc add dev {nsifname} {selector} netem {netem_args}"
1528 selector
= f
"parent {count-1}: handle {count}"
1529 # Place rate limit after delay otherwise limit/burst too complex
1531 self
.cmd_raises(f
"tc qdisc add dev {nsifname} {selector} tbf {tbf_args}")
1533 self
.cmd_raises(f
"tc qdisc show dev {nsifname}")
1536 class LinuxNamespace(Commander
, InterfaceMixin
):
1537 """A linux Namespace.
1539 An object that creates and executes commands in a linux namespace
1553 unshare_inline
=False,
1555 private_mounts
=None,
1558 """Create a new linux namespace.
1561 name: Internal name for the namespace.
1562 net: Create network namespace.
1563 mount: Create network namespace.
1564 uts: Create UTS (hostname) namespace.
1565 cgroup: Create cgroup namespace.
1566 ipc: Create IPC namespace.
1567 pid: Create PID namespace, also mounts new /proc.
1568 time: Create time namespace.
1569 user: Create user namespace, also keeps capabilities.
1570 set_hostname: Set the hostname to `name`, uts must also be True.
1571 private_mounts: List of strings of the form
1572 "[/external/path:]/internal/path. If no external path is specified a
1573 tmpfs is mounted on the internal path. Any paths specified are first
1574 passed to `mkdir -p`.
1575 unshare_inline: Unshare the process itself rather than using a proxy.
1576 logger: Passed to superclass.
1578 # logging.warning("LinuxNamespace: name %s kwargs %s", name, kwargs)
1580 super().__init
__(name
, **kwargs
)
1584 self
.logger
.debug("%s: creating", self
)
1586 self
.cwd
= os
.path
.abspath(os
.getcwd())
1591 self
.p_ns_fds
= None
1592 self
.p_ns_fnames
= None
1594 self
.init_pid
= None
1595 self
.unshare_inline
= unshare_inline
1596 self
.nsenter_fork
= True
1599 # Collect the namespaces to unshare
1601 if hasattr(self
, "proc_path") and self
.proc_path
: # pylint: disable=no-member
1602 pp
= Path(self
.proc_path
) # pylint: disable=no-member
1604 pp
= unet
.proc_path
if unet
else Path("/proc")
1605 pp
= pp
.joinpath("%P%", "ns")
1613 nslist
.append(nselm
)
1614 nsflags
.append(f
"--{nselm}={pp / nselm}")
1616 uflags |
= linux
.CLONE_NEWCGROUP
1619 nslist
.append(nselm
)
1620 nsflags
.append(f
"--{nselm}={pp / nselm}")
1622 uflags |
= linux
.CLONE_NEWIPC
1624 # We need a new mount namespace for pid
1626 nslist
.append(nselm
)
1627 nsflags
.append(f
"--mount={pp / nselm}")
1630 uflags |
= linux
.CLONE_NEWNS
1633 nslist
.append(nselm
)
1634 nsflags
.append(f
"--{nselm}={pp / nselm}")
1636 # os.system(f"touch /tmp/netns-{name}")
1637 # cmd.append(f"--net=/tmp/netns-{name}")
1640 uflags |
= linux
.CLONE_NEWNET
1643 # We look for this b/c the unshare pid will share with /sibn/init
1644 nselm
= "pid_for_children"
1645 nslist
.append(nselm
)
1646 nsflags
.append(f
"--pid={pp / nselm}")
1648 uflags |
= linux
.CLONE_NEWPID
1651 # XXX time_for_children?
1652 nslist
.append(nselm
)
1653 nsflags
.append(f
"--{nselm}={pp / nselm}")
1655 uflags |
= linux
.CLONE_NEWTIME
1658 nslist
.append(nselm
)
1659 nsflags
.append(f
"--{nselm}={pp / nselm}")
1661 uflags |
= linux
.CLONE_NEWUSER
1664 nslist
.append(nselm
)
1665 nsflags
.append(f
"--{nselm}={pp / nselm}")
1667 uflags |
= linux
.CLONE_NEWUTS
1669 assert flags
, "LinuxNamespace with no namespaces requested"
1671 # Should look path up using resources maybe...
1672 mutini_path
= get_our_script_path("mutini")
1674 mutini_path
= get_our_script_path("mutini.py")
1676 cmd
= [mutini_path
, f
"--unshare-flags={flags}", "-v"]
1677 fname
= fsafe_name(self
.name
) + "-mutini.log"
1678 fname
= (unet
or self
).rundir
.joinpath(fname
)
1679 stdout
= open(fname
, "w", encoding
="utf-8")
1680 stderr
= subprocess
.STDOUT
1683 # Save the current namespace info to compare against later
1687 nsdict
= {x
: os
.readlink(f
"/proc/self/ns/{x}") for x
in nslist
}
1690 x
: os
.readlink(f
"{unet.proc_path}/{unet.pid}/ns/{x}") for x
in nslist
1694 # (A) Basically we need to save the pid of the unshare call for nsenter.
1696 # For `unet is not None` (node case) the level this exists at is based on wether
1697 # unet is using a forking nsenter or not. So if unet.nsenter_fork == True then
1698 # we need the child pid of the p.pid (child of pid returned to us), otherwise
1699 # unet.nsenter_fork == False and we just use p.pid as it will be unshare after
1700 # nsenter exec's it.
1702 # For the `unet is None` (unet case) the unshare is at the top level or
1703 # non-existent so we always save the returned p.pid. If we are unshare_inline we
1704 # won't have a __pre_cmd but we can save our child_pid to kill later, otherwise
1705 # we set unet.pid to None b/c there's literally nothing to do.
1707 # ---------------------------------------------------------------------------
1708 # Breakdown for nested (non-unet) namespace creation, and what PID
1709 # to use for __pre_cmd nsenter use.
1710 # ---------------------------------------------------------------------------
1713 # - for non-inline unshare: Use BBB with pid_for_children, unless none/none
1714 # #then (AAA) returned
1715 # - for inline unshare: use returned pid (AAA) with pid_for_children
1717 # All commands use unet.popen to launch the unshare of mutini or cat.
1718 # mutini for PID unshare, otherwise cat. AAA is the returned pid BBB is the
1719 # child of the returned.
1724 # Here we are running mutini if we are creating new pid namespace workspace,
1727 # [PID+PID] pid tree looks like this:
1729 # PID NSPID PPID PGID
1730 # uuu - N/A uuu main unet process
1731 # AAA - uuu AAA nsenter (forking, from unet) (in unet namespaces -pid)
1732 # BBB - AAA AAA unshare --fork --kill-child (forking)
1733 # CCC 1 BBB CCC mutini (non-forking since it is pid 1 in new namespace)
1735 # Use BBB if we use pid_for_children, CCC for all
1737 # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid
1738 # tree looks like this:
1741 # uuu N/A uuu main unet process
1742 # AAA uuu AAA nsenter (forking) (in unet namespaces -pid)
1743 # BBB AAA AAA unshare -> cat (from unshare non-forking)
1747 # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid
1748 # tree looks like this:
1750 # PID NSPID PPID PGID
1751 # uuu - N/A uuu main unet process
1752 # AAA - uuu AAA nsenter -> unshare --fork --kill-child
1753 # BBB 1 AAA AAA mutini (non-forking since it is pid 1 in new namespace)
1755 # Use AAA if we use pid_for_children, BBB for all
1757 # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree
1761 # uuu N/A uuu main unet process
1762 # AAA uuu AAA nsenter -> unshare -> cat
1764 # Use AAA for all, there's no BBB
1766 # Inline-Unshare Variant
1767 # ----------------------
1769 # For unshare_inline and new PID namespace we have unshared all but our PID
1770 # namespace, but our children end up in the new namespace so the fork popen
1771 # does is good enough.
1773 # [PID+PID] pid tree looks like this:
1775 # PID NSPID PPID PGID
1776 # uuu - N/A uuu main unet process
1777 # AAA - uuu AAA unshare --fork --kill-child (forking)
1778 # BBB 1 AAA BBB mutini
1780 # Use AAA if we use pid_for_children, BBB for all
1782 # [PID+none] For non-pid workspace creation (but unet pid) we use cat and pid
1783 # tree looks like this:
1786 # uuu N/A uuu main unet process
1787 # AAA uuu AAA unshare -> cat
1791 # [none+PID] For pid workspace creation (but NOT unet pid) we use mutini and pid
1792 # tree looks like this:
1794 # PID NSPID PPID PGID
1795 # uuu - N/A uuu main unet process
1796 # AAA - uuu AAA unshare --fork --kill-child
1797 # BBB 1 AAA BBB mutini
1799 # Use AAA if we use pid_for_children, BBB for all
1801 # [none+none] For non-pid workspace and non-pid unet we use cat and pid tree
1805 # uuu N/A uuu main unet process
1806 # AAA uuu AAA unshare -> cat
1811 # ---------------------------------------------------------------------------
1812 # Breakdown for unet namespace creation, and what PID to use for __pre_cmd
1813 # ---------------------------------------------------------------------------
1815 # tl;dr: save returned PID or nothing.
1816 # - for non-inline unshare: Use AAA with pid_for_children (returned pid)
1817 # - for inline unshare: no __precmd as the fork in popen is enough.
1819 # Use commander to launch the unshare mutini/cat (for PID/none
1820 # workspace PID) for non-inline case. AAA is the returned pid BBB is the child
1826 # Here we are running mutini if we are creating new pid namespace workspace,
1829 # [PID] for unet pid creation pid tree looks like this:
1831 # PID NSPID PPID PGID
1832 # uuu - N/A uuu main unet process
1833 # AAA - uuu AAA unshare --fork --kill-child (forking)
1834 # BBB 1 AAA BBB mutini
1836 # Use AAA if we use pid_for_children, BBB for all
1838 # [none] for unet non-pid, pid tree looks like this:
1841 # uuu N/A uuu main unet process
1842 # AAA uuu AAA unshare -> cat
1846 # Inline-Unshare Variant
1847 # -----------------------
1849 # For unshare_inline and new PID namespace we have unshared all but our PID
1850 # namespace, but our children end up in the new namespace so the fork in popen
1851 # does is good enough.
1853 # [PID] for unet pid creation pid tree looks like this:
1855 # PID NSPID PPID PGID
1856 # uuu - N/A uuu main unet process
1857 # AAA 1 uuu AAA mutini
1859 # Save p / p.pid, but don't configure any nsenter, uneeded.
1861 # Use nothing as the fork when doing a popen is enough to be in all the right
1864 # [none] for unet non-pid, pid tree looks like this:
1867 # uuu N/A uuu main unet process
1869 # Nothing, no __pre_cmd.
1873 self
.ppid
= os
.getppid()
1874 self
.unshare_inline
= unshare_inline
1877 self
.uflags
= uflags
1879 # Open file descriptors for current namespaces for later resotration.
1882 kversion
= [int(x
) for x
in platform
.release().split("-")[0].split(".")]
1883 kvok
= kversion
[0] > 5 or (kversion
[0] == 5 and kversion
[1] >= 8)
1888 or sys
.version_info
[0] < 3
1889 or (sys
.version_info
[0] == 3 and sys
.version_info
[1] < 9)
1891 # get list of namespace file descriptors before we unshare
1893 self
.p_ns_fnames
= []
1895 for i
in range(0, 64):
1897 if (tmpflags
& v
) == 0:
1900 if v
in linux
.namespace_files
:
1901 path
= os
.path
.join("/proc/self", linux
.namespace_files
[v
])
1902 if os
.path
.exists(path
):
1903 self
.p_ns_fds
.append(os
.open(path
, 0))
1904 self
.p_ns_fnames
.append(f
"{path} -> {os.readlink(path)}")
1906 "%s: saving old namespace fd %s (%s)",
1908 self
.p_ns_fnames
[-1],
1914 self
.p_ns_fds
= None
1915 self
.p_ns_fnames
= None
1916 self
.ppid_fd
= linux
.pidfd_open(self
.ppid
)
1919 "%s: unshare to new namespaces %s",
1921 linux
.clone_flag_string(uflags
),
1924 linux
.unshare(uflags
)
1929 self
.nsenter_fork
= False
1931 # Need to fork to create the PID namespace, but we need to continue
1932 # running from the parent so that things like pytest work. We'll execute
1933 # a mutini process to manage the child init 1 duties.
1935 # We (the parent pid) can no longer create threads, due to that being
1936 # restricted by the kernel. See EINVAL in clone(2).
1938 p
= commander
.popen(
1939 [mutini_path
, "-v"],
1940 stdin
=subprocess
.PIPE
,
1944 # new session/pgid so signals don't propagate
1945 start_new_session
=True,
1949 self
.nsenter_fork
= False
1951 # Using cat and a stdin PIPE is nice as it will exit when we do. However,
1952 # we also detach it from the pgid so that signals do not propagate to it.
1953 # This is b/c it would exit early (e.g., ^C) then, at least the main munet
1954 # proc which has no other processes like frr daemons running, will take the
1955 # main network namespace with it, which will remove the bridges and the
1956 # veth pair (because the bridge side veth is deleted).
1957 self
.logger
.debug("%s: creating namespace process: %s", self
, cmd
)
1959 # Use the parent unet process if we have one this will cause us to inherit
1960 # the namespaces correctly even in the non-inline case.
1961 parent
= self
.unet
if self
.unet
else commander
1965 stdin
=subprocess
.PIPE
,
1969 start_new_session
=not unet
,
1973 # The pid number returned is in the global pid namespace. For unshare_inline
1974 # this can be unfortunate b/c our /proc has been remounted in our new pid
1975 # namespace and won't contain global pid namespace pids. To solve for this
1976 # we get all the pid values for the process below.
1978 # See (A) above for when we need the child pid.
1979 self
.logger
.debug("%s: namespace process: %s", self
, proc_str(p
))
1981 if unet
and unet
.nsenter_fork
:
1982 assert not unet
.unshare_inline
1983 # Need child pid of p.pid
1984 pgrep
= unet
.rootcmd
.get_exec_path("pgrep")
1985 # a sing fork was done
1986 child_pid
= unet
.rootcmd
.cmd_raises([pgrep
, "-o", "-P", str(p
.pid
)])
1987 self
.pid
= int(child_pid
.strip())
1988 self
.logger
.debug("%s: child of namespace process: %s", self
, pid
)
1992 # Let's always have a valid value.
1993 if self
.pid
is None:
1997 # Let's find all our pids in the nested PID namespaces
2000 proc_path
= unet
.proc_path
2002 proc_path
= self
.proc_path
if hasattr(self
, "proc_path") else "/proc"
2003 proc_path
= f
"{proc_path}/{self.pid}"
2005 pid_status
= open(f
"{proc_path}/status", "r", encoding
="ascii").read()
2006 m
= re
.search(r
"\nNSpid:((?:\t[0-9]+)+)\n", pid_status
)
2007 self
.pids
= [int(x
) for x
in m
.group(1).strip().split("\t")]
2008 assert self
.pids
[0] == self
.pid
2010 self
.logger
.debug("%s: namespace scoped pids: %s", self
, self
.pids
)
2012 # -----------------------------------------------
2013 # Now let's wait until unshare completes it's job
2014 # -----------------------------------------------
2015 timeout
= Timeout(30)
2016 if self
.pid
is not None and self
.pid
!= our_pid
:
2017 while (not p
or not p
.poll()) and not timeout
.is_expired():
2018 # check new namespace values against old (nsdict), unshare
2019 # can actually take a bit to complete.
2020 for fname
in tuple(nslist
):
2021 # self.pid will be the global pid b/c we didn't unshare_inline
2022 nspath
= f
"{proc_path}/ns/{fname}"
2024 nsf
= os
.readlink(nspath
)
2025 except OSError as error
:
2027 "unswitched: error (ok) checking %s: %s", nspath
, error
2030 if nsdict
[fname
] != nsf
:
2032 "switched: original %s current %s", nsdict
[fname
], nsf
2034 nslist
.remove(fname
)
2035 elif unshare_inline
:
2037 "unshare_inline not unshared %s == %s", nsdict
[fname
], nsf
2041 "unswitched: current %s elapsed: %s", nsf
, timeout
.elapsed()
2045 "all done waiting for unshare after %s", timeout
.elapsed()
2049 elapsed
= int(timeout
.elapsed())
2054 "%s: unshare taking more than %ss: %s", self
, elapsed
, nslist
2058 if p
is not None and p
.poll():
2059 self
.logger
.error("%s: namespace process failed: %s", self
, comm_error(p
))
2060 assert p
.poll() is None, "unshare failed"
2063 # Setup the pre-command to enter the target namespace from the running munet
2064 # process using self.pid
2069 elif unet
and unet
.nsenter_fork
:
2070 # if unet created a pid namespace we need to enter it since we aren't
2071 # entering a child pid namespace we created for the node. Otherwise
2072 # we have a /proc remounted under unet, but our process is running in
2073 # the root pid namepsace
2074 nselm
= "pid_for_children"
2075 nsflags
.append(f
"--pid={pp / nselm}")
2078 # We dont need a fork.
2079 nsflags
.append("-F")
2080 nsenter_fork
= False
2082 # Save nsenter values if running from root namespace
2083 # we need this for the unshare_inline case when run externally (e.g., from
2084 # within tmux server).
2085 root_nsflags
= [x
.replace("%P%", str(self
.pid
)) for x
in nsflags
]
2086 self
.__root
_base
_pre
_cmd
= ["/usr/bin/nsenter", *root_nsflags
]
2087 self
.__root
_pre
_cmd
= list(self
.__root
_base
_pre
_cmd
)
2091 # We have nothing to do here since our process is now in the correct
2092 # namespaces and children will inherit from us, even the PID namespace will
2093 # be corrent b/c commands are run by first forking.
2094 self
.nsenter_fork
= False
2096 self
.__base
_pre
_cmd
= []
2098 # We will use nsenter
2099 self
.nsenter_fork
= nsenter_fork
2100 self
.nsflags
= nsflags
2101 self
.__base
_pre
_cmd
= list(self
.__root
_base
_pre
_cmd
)
2103 self
.__pre
_cmd
= list(self
.__base
_pre
_cmd
)
2105 # Always mark new mount namespaces as recursive private
2107 # if self.p is None and not pid:
2108 self
.cmd_raises_nsonly("mount --make-rprivate /")
2110 # We need to remount the procfs for the new PID namespace, since we aren't using
2111 # unshare(1) which does that for us.
2112 if pid
and unshare_inline
:
2114 self
.cmd_raises_nsonly("mount -t proc proc /proc")
2116 # We do not want cmd_status in child classes (e.g., container) for
2117 # the remaining setup calls in this __init__ function.
2120 # Remount /sys to pickup any changes in the network, but keep root
2121 # /sys/fs/cgroup. This pattern could be made generic and supported for any
2122 # overlapping mounts
2124 tmpmnt
= f
"/tmp/cgm-{self.pid}"
2125 self
.cmd_status_nsonly(
2126 f
"mkdir {tmpmnt} && mount --rbind /sys/fs/cgroup {tmpmnt}"
2129 for i
in range(0, 10):
2130 rc
, o
, e
= self
.cmd_status_nsonly(
2131 "mount -t sysfs sysfs /sys", warn
=False
2136 "got error mounting new sysfs will retry: %s",
2137 cmd_error(rc
, o
, e
),
2141 raise Exception(cmd_error(rc
, o
, e
))
2143 self
.cmd_status_nsonly(
2144 f
"mount --move {tmpmnt} /sys/fs/cgroup && rmdir {tmpmnt}"
2147 # Original micronet code
2148 # self.cmd_raises_nsonly("mount -t sysfs sysfs /sys")
2149 # self.cmd_raises_nsonly(
2150 # "mount -o rw,nosuid,nodev,noexec,relatime "
2151 # "-t cgroup2 cgroup /sys/fs/cgroup"
2154 # Set the hostname to the namespace name
2155 if uts
and set_hostname
:
2156 self
.cmd_status_nsonly("hostname " + self
.name
)
2157 nroot
= subprocess
.check_output("hostname")
2158 if unshare_inline
or (unet
and unet
.unshare_inline
):
2160 root_hostname
!= nroot
2161 ), f
'hostname unchanged from "{nroot}" wanted "{self.name}"'
2163 # Assert that we didn't just change the host hostname
2165 root_hostname
== nroot
2166 ), f
'root hostname "{root_hostname}" changed to "{nroot}"!'
2169 if isinstance(private_mounts
, str):
2170 private_mounts
= [private_mounts
]
2171 for m
in private_mounts
:
2174 self
.tmpfs_mount(s
[0])
2176 self
.bind_mount(s
[0], s
[1])
2178 # this will fail if running inside the namespace with PID
2180 o
= self
.cmd_nostatus_nsonly("ls -l /proc/1/ns")
2182 o
= self
.cmd_nostatus_nsonly("ls -l /proc/self/ns")
2184 self
.logger
.debug("namespaces:\n %s", o
)
2186 # will cache the path, which is important in delete to avoid running a shell
2187 # which can hang during cleanup
2188 self
.ip_path
= get_exec_path_host("ip")
2190 self
.cmd_status_nsonly([self
.ip_path
, "link", "set", "lo", "up"])
2192 self
.logger
.info("%s: created", self
)
2194 def _get_pre_cmd(self
, use_str
, use_pty
, ns_only
=False, root_level
=False, **kwargs
):
2195 """Get the pre-user-command values.
2197 The values returned here should be what is required to cause the user's command
2198 to execute in the correct context (e.g., namespace, container, sshremote).
2203 pre_cmd
= self
.__root
_pre
_cmd
if root_level
else self
.__pre
_cmd
2204 return shlex
.join(pre_cmd
) if use_str
else list(pre_cmd
)
2206 def tmpfs_mount(self
, inner
):
2207 self
.logger
.debug("Mounting tmpfs on %s", inner
)
2208 self
.cmd_raises("mkdir -p " + inner
)
2209 self
.cmd_raises("mount -n -t tmpfs tmpfs " + inner
)
2211 def bind_mount(self
, outer
, inner
):
2212 self
.logger
.debug("Bind mounting %s on %s", outer
, inner
)
2213 if commander
.test("-f", outer
):
2214 self
.cmd_raises(f
"mkdir -p {os.path.dirname(inner)} && touch {inner}")
2216 if not commander
.test("-e", outer
):
2217 commander
.cmd_raises_nsonly(f
"mkdir -p {outer}")
2218 self
.cmd_raises(f
"mkdir -p {inner}")
2219 self
.cmd_raises("mount --rbind {} {} ".format(outer
, inner
))
2221 def add_netns(self
, ns
):
2222 self
.logger
.debug("Adding network namespace %s", ns
)
2224 if os
.path
.exists("/run/netns/{}".format(ns
)):
2225 self
.logger
.warning("%s: Removing existing nsspace %s", self
, ns
)
2227 self
.delete_netns(ns
)
2228 except Exception as ex
:
2229 self
.logger
.warning(
2230 "%s: Couldn't remove existing nsspace %s: %s",
2236 self
.cmd_raises_nsonly([self
.ip_path
, "netns", "add", ns
])
2238 def delete_netns(self
, ns
):
2239 self
.logger
.debug("Deleting network namespace %s", ns
)
2240 self
.cmd_raises_nsonly([self
.ip_path
, "netns", "delete", ns
])
2242 def set_intf_netns(self
, intf
, ns
, up
=False):
2243 # In case a user hard-codes 1 thinking it "resets"
2248 self
.logger
.debug("Moving interface %s to namespace %s", intf
, ns
)
2250 cmd
= [self
.ip_path
, "link", "set", intf
, "netns", ns
]
2253 self
.intf_ip_cmd(intf
, cmd
)
2254 if ns
== str(self
.pid
):
2255 # If we are returning then remove from dict
2256 if intf
in self
.ifnetns
:
2257 del self
.ifnetns
[intf
]
2259 self
.ifnetns
[intf
] = ns
2261 def reset_intf_netns(self
, intf
):
2262 self
.logger
.debug("Moving interface %s to default namespace", intf
)
2263 self
.set_intf_netns(intf
, str(self
.pid
))
2265 def intf_ip_cmd(self
, intf
, cmd
):
2266 """Run an ip command, considering an interface's possible namespace."""
2267 if intf
in self
.ifnetns
:
2268 if isinstance(cmd
, list):
2269 assert cmd
[0].endswith("ip")
2270 cmd
[1:1] = ["-n", self
.ifnetns
[intf
]]
2272 assert cmd
.startswith("ip ")
2273 cmd
= "ip -n " + self
.ifnetns
[intf
] + cmd
[2:]
2274 self
.cmd_raises_nsonly(cmd
)
2276 def intf_tc_cmd(self
, intf
, cmd
):
2277 """Run a tc command, considering an interface's possible namespace."""
2278 if intf
in self
.ifnetns
:
2279 if isinstance(cmd
, list):
2280 assert cmd
[0].endswith("tc")
2281 cmd
[1:1] = ["-n", self
.ifnetns
[intf
]]
2283 assert cmd
.startswith("tc ")
2284 cmd
= "tc -n " + self
.ifnetns
[intf
] + cmd
[2:]
2285 self
.cmd_raises_nsonly(cmd
)
2287 def set_ns_cwd(self
, cwd
: Union
[str, Path
]):
2288 """Common code for changing pre_cmd and pre_nscmd."""
2289 self
.logger
.debug("%s: new CWD %s", self
, cwd
)
2290 self
.__root
_pre
_cmd
= self
.__root
_base
_pre
_cmd
+ ["--wd=" + str(cwd
)]
2292 self
.__pre
_cmd
= self
.__base
_pre
_cmd
+ ["--wd=" + str(cwd
)]
2293 elif self
.unshare_inline
:
2296 async def _async_delete(self
):
2297 if type(self
) == LinuxNamespace
: # pylint: disable=C0123
2298 self
.logger
.info("%s: deleting", self
)
2300 self
.logger
.debug("%s: LinuxNamespace sub-class deleting", self
)
2302 # Signal pid namespace proc to exit
2304 (self
.p
is None or self
.p
.pid
!= self
.pid
)
2306 and self
.pid
!= our_pid
2309 "cleanup pid on separate pid %s from proc pid %s",
2311 self
.p
.pid
if self
.p
else None,
2313 await self
.cleanup_pid(self
.pid
)
2315 if self
.p
is not None:
2316 self
.logger
.debug("cleanup proc pid %s", self
.p
.pid
)
2317 await self
.async_cleanup_proc(self
.p
)
2319 # return to the previous namespace, need to do this in case anothe munet
2320 # is being created, especially when it plans to inherit the parent's (host)
2323 logging
.info("restoring from inline unshare: cwd: %s", os
.getcwd())
2324 # This only works in linux>=5.8
2325 if self
.p_ns_fds
is None:
2327 "%s: restoring namespaces %s",
2329 linux
.clone_flag_string(self
.uflags
),
2331 # fd = linux.pidfd_open(self.ppid)
2334 for i
in range(0, retry
):
2336 linux
.setns(fd
, self
.uflags
)
2337 except OSError as error
:
2338 self
.logger
.warning(
2339 "%s: could not reset to old namespace fd %s: %s",
2349 while self
.p_ns_fds
:
2350 fd
= self
.p_ns_fds
.pop()
2351 fname
= self
.p_ns_fnames
.pop()
2353 "%s: restoring namespace from fd %s (%s)", self
, fname
, fd
2356 for i
in range(0, retry
):
2360 except OSError as error
:
2361 self
.logger
.warning(
2362 "%s: could not reset to old namespace fd %s (%s): %s",
2372 self
.p_ns_fds
= None
2373 self
.p_ns_fnames
= None
2374 logging
.info("restored from unshare: cwd: %s", os
.getcwd())
2376 self
.__root
_base
_pre
_cmd
= ["/bin/false"]
2377 self
.__base
_pre
_cmd
= ["/bin/false"]
2378 self
.__root
_pre
_cmd
= ["/bin/false"]
2379 self
.__pre
_cmd
= ["/bin/false"]
2381 await super()._async
_delete
()
2384 class SharedNamespace(Commander
):
2385 """Share another namespace.
2387 An object that executes commands in an existing pid's linux namespace
2390 def __init__(self
, name
, pid
=None, nsflags
=None, **kwargs
):
2391 """Share a linux namespace.
2394 name: Internal name for the namespace.
2395 pid: PID of the process to share with.
2396 nsflags: nsenter flags to pass to inherit namespaces from
2398 super().__init
__(name
, **kwargs
)
2400 self
.logger
.debug("%s: Creating", self
)
2402 self
.cwd
= os
.path
.abspath(os
.getcwd())
2403 self
.pid
= pid
if pid
is not None else our_pid
2405 nsflags
= (x
.replace("%P%", str(self
.pid
)) for x
in nsflags
) if nsflags
else []
2406 self
.__base
_pre
_cmd
= ["/usr/bin/nsenter", *nsflags
] if nsflags
else []
2407 self
.__pre
_cmd
= self
.__base
_pre
_cmd
2408 self
.ip_path
= self
.get_exec_path("ip")
2410 def _get_pre_cmd(self
, use_str
, use_pty
, ns_only
=False, root_level
=False, **kwargs
):
2411 """Get the pre-user-command values.
2413 The values returned here should be what is required to cause the user's command
2414 to execute in the correct context (e.g., namespace, container, sshremote).
2419 assert not root_level
2420 return shlex
.join(self
.__pre
_cmd
) if use_str
else list(self
.__pre
_cmd
)
2422 def set_ns_cwd(self
, cwd
: Union
[str, Path
]):
2423 """Common code for changing pre_cmd and pre_nscmd."""
2424 self
.logger
.debug("%s: new CWD %s", self
, cwd
)
2425 self
.__pre
_cmd
= self
.__base
_pre
_cmd
+ ["--wd=" + str(cwd
)]
2428 class Bridge(SharedNamespace
, InterfaceMixin
):
2429 """A linux bridge."""
2434 def _get_next_id(cls
):
2435 # Do not use `cls` here b/c that makes the variable class specific
2437 Bridge
.next_ord
= n
+ 1
2440 def __init__(self
, name
=None, mtu
=None, unet
=None, **kwargs
):
2441 """Create a linux Bridge."""
2442 self
.id = self
._get
_next
_id
()
2444 name
= "br{}".format(self
.id)
2446 unet_pid
= our_pid
if unet
.pid
is None else unet
.pid
2448 super().__init
__(name
, pid
=unet_pid
, nsflags
=unet
.nsflags
, unet
=unet
, **kwargs
)
2450 self
.set_intf_basename(self
.name
+ "-e")
2454 self
.logger
.debug("Bridge: Creating")
2456 assert len(self
.name
) <= 16 # Make sure fits in IFNAMSIZE
2457 self
.cmd_raises(f
"ip link delete {name} || true")
2458 self
.cmd_raises(f
"ip link add {name} type bridge")
2460 self
.cmd_raises(f
"ip link set {name} mtu {self.mtu}")
2461 self
.cmd_raises(f
"ip link set {name} up")
2463 self
.logger
.debug("%s: Created, Running", self
)
2465 def get_ifname(self
, netname
):
2466 return self
.net_intfs
[netname
] if netname
in self
.net_intfs
else None
2468 async def _async_delete(self
):
2469 """Stop the bridge (i.e., delete the linux resources)."""
2470 if type(self
) == Bridge
: # pylint: disable=C0123
2471 self
.logger
.info("%s: deleting", self
)
2473 self
.logger
.debug("%s: Bridge sub-class deleting", self
)
2475 rc
, o
, e
= await self
.async_cmd_status(
2476 [self
.ip_path
, "link", "show", self
.name
],
2477 stdin
=subprocess
.DEVNULL
,
2478 start_new_session
=True,
2482 rc
, o
, e
= await self
.async_cmd_status(
2483 [self
.ip_path
, "link", "delete", self
.name
],
2484 stdin
=subprocess
.DEVNULL
,
2485 start_new_session
=True,
2490 "%s: error deleting bridge %s: %s",
2493 cmd_error(rc
, o
, e
),
2495 await super()._async
_delete
()
2498 class BaseMunet(LinuxNamespace
):
2510 """Create a Munet."""
2511 # logging.warning("BaseMunet: %s", name)
2518 self
.isolated
= isolated
2520 self
.cli_server
= None
2521 self
.cli_sockpath
= None
2522 self
.cli_histfile
= None
2523 self
.cli_in_window_cmds
= {}
2524 self
.cli_run_cmds
= {}
2527 # We need a directory for various files
2530 rundir
= "/tmp/munet"
2531 self
.rundir
= Path(rundir
)
2534 # Always having a global /proc is required to keep things from exploding
2535 # complexity with nested new pid namespaces..
2538 self
.proc_path
= Path(tempfile
.mkdtemp(suffix
="-proc", prefix
="mu-"))
2539 logging
.debug("%s: mounting /proc on proc_path %s", name
, self
.proc_path
)
2540 linux
.mount("proc", str(self
.proc_path
), "proc")
2542 self
.proc_path
= Path("/proc")
2545 # Now create a root level commander that works regardless of whether we inline
2546 # unshare or not. Save it in the global variable as well
2549 if not self
.isolated
:
2550 self
.rootcmd
= commander
2553 f
"--mount={self.proc_path / '1/ns/mnt'}",
2554 f
"--net={self.proc_path / '1/ns/net'}",
2555 f
"--uts={self.proc_path / '1/ns/uts'}",
2556 # f"--ipc={self.proc_path / '1/ns/ipc'}",
2557 # f"--time={self.proc_path / '1/ns/time'}",
2558 # f"--cgroup={self.proc_path / '1/ns/cgroup'}",
2560 self
.rootcmd
= SharedNamespace("root", pid
=1, nsflags
=nsflags
)
2564 # XXX Backing up PID namespace just doesn't work.
2565 # f"--pid={self.proc_path / '1/ns/pid_for_children'}",
2566 f
"--mount={self.proc_path / '1/ns/mnt'}",
2567 f
"--net={self.proc_path / '1/ns/net'}",
2568 f
"--uts={self.proc_path / '1/ns/uts'}",
2569 # f"--ipc={self.proc_path / '1/ns/ipc'}",
2570 # f"--time={self.proc_path / '1/ns/time'}",
2571 # f"--cgroup={self.proc_path / '1/ns/cgroup'}",
2573 self
.rootcmd
= SharedNamespace("root", pid
=1, nsflags
=nsflags
)
2574 global roothost
# pylint: disable=global-statement
2576 roothost
= self
.rootcmd
2578 self
.cfgopt
= munet_config
.ConfigOptionsProxy(pytestconfig
)
2581 name
, mount
=True, net
=isolated
, uts
=isolated
, pid
=pid
, unet
=None, **kwargs
2584 # This allows us to cleanup any leftover running munet's
2585 if "MUNET_PID" in os
.environ
:
2586 if os
.environ
["MUNET_PID"] != str(our_pid
):
2588 "Found env MUNET_PID != our pid %s, instead its %s, changing",
2590 os
.environ
["MUNET_PID"],
2592 os
.environ
["MUNET_PID"] = str(our_pid
)
2594 # this is for testing purposes do not use
2595 if not BaseMunet
.g_unet
:
2596 BaseMunet
.g_unet
= self
2598 self
.logger
.debug("%s: Creating", self
)
2600 def __getitem__(self
, key
):
2601 if key
in self
.switches
:
2602 return self
.switches
[key
]
2603 return self
.hosts
[key
]
2605 def add_host(self
, name
, cls
=LinuxNamespace
, **kwargs
):
2606 """Add a host to munet."""
2607 self
.logger
.debug("%s: add_host %s(%s)", self
, cls
.__name
__, name
)
2609 self
.hosts
[name
] = cls(name
, unet
=self
, **kwargs
)
2611 # Create a new mounted FS for tracking nested network namespaces creatd by the
2612 # user with `ip netns add`
2614 # XXX why is this failing with podman???
2615 # self.hosts[name].tmpfs_mount("/run/netns")
2617 return self
.hosts
[name
]
2619 def add_link(self
, node1
, node2
, if1
, if2
, mtu
=None, **intf_constraints
):
2620 """Add a link between switch and node or 2 nodes.
2622 If constraints are given they are applied to each endpoint. See
2623 `InterfaceMixin::set_intf_constraints()` for more info.
2629 except AttributeError:
2630 if node1
in self
.switches
:
2631 node1
= self
.switches
[node1
]
2633 node1
= self
.hosts
[node1
]
2638 except AttributeError:
2639 if node2
in self
.switches
:
2640 node2
= self
.switches
[node2
]
2642 node2
= self
.hosts
[node2
]
2645 if name1
in self
.switches
:
2646 assert name2
in self
.hosts
2647 elif name2
in self
.switches
:
2648 assert name1
in self
.hosts
2649 name1
, name2
= name2
, name1
2653 assert name1
in self
.hosts
2654 assert name2
in self
.hosts
2657 lname
= "{}:{}-{}:{}".format(name1
, if1
, name2
, if2
)
2658 self
.logger
.debug("%s: add_link %s%s", self
, lname
, " p2p" if isp2p
else "")
2659 self
.links
[lname
] = (name1
, if1
, name2
, if2
)
2661 # And create the veth now.
2663 lhost
, rhost
= self
.hosts
[name1
], self
.hosts
[name2
]
2664 lifname
= "i1{:x}".format(lhost
.pid
)
2666 # Done at root level
2667 nsif1
= lhost
.get_ns_ifname(if1
)
2668 nsif2
= rhost
.get_ns_ifname(if2
)
2670 # Use pids[-1] to get the unet scoped pid for hosts
2671 self
.cmd_raises_nsonly(
2672 f
"ip link add {lifname} type veth peer name {nsif2}"
2673 f
" netns {rhost.pids[-1]}"
2675 self
.cmd_raises_nsonly(f
"ip link set {lifname} netns {lhost.pids[-1]}")
2677 lhost
.cmd_raises_nsonly("ip link set {} name {}".format(lifname
, nsif1
))
2679 lhost
.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1
, mtu
))
2680 lhost
.cmd_raises_nsonly("ip link set {} up".format(nsif1
))
2681 lhost
.register_interface(if1
)
2684 rhost
.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2
, mtu
))
2685 rhost
.cmd_raises_nsonly("ip link set {} up".format(nsif2
))
2686 rhost
.register_interface(if2
)
2688 switch
= self
.switches
[name1
]
2689 rhost
= self
.hosts
[name2
]
2691 nsif1
= switch
.get_ns_ifname(if1
)
2692 nsif2
= rhost
.get_ns_ifname(if2
)
2698 self
.logger
.error('"%s" len %s > 16', nsif1
, len(nsif1
))
2699 elif len(nsif2
) > 16:
2700 self
.logger
.error('"%s" len %s > 16', nsif2
, len(nsif2
))
2701 assert len(nsif1
) <= 16 and len(nsif2
) <= 16 # Make sure fits in IFNAMSIZE
2703 self
.logger
.debug("%s: Creating veth pair for link %s", self
, lname
)
2705 # Use pids[-1] to get the unet scoped pid for hosts
2706 # switch is already in our namespace so nothing to convert.
2707 self
.cmd_raises_nsonly(
2708 f
"ip link add {nsif1} type veth peer name {nsif2}"
2709 f
" netns {rhost.pids[-1]}"
2714 # # the switch interface should match the switch config
2715 # switch.cmd_raises_nsonly(
2716 # "ip link set {} mtu {}".format(if1, switch.mtu)
2718 switch
.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif1
, mtu
))
2719 rhost
.cmd_raises_nsonly("ip link set {} mtu {}".format(nsif2
, mtu
))
2721 switch
.register_interface(if1
)
2722 rhost
.register_interface(if2
)
2723 rhost
.register_network(switch
.name
, if2
)
2725 switch
.cmd_raises_nsonly(f
"ip link set {nsif1} master {switch.name}")
2727 switch
.cmd_raises_nsonly(f
"ip link set {nsif1} up")
2728 rhost
.cmd_raises_nsonly(f
"ip link set {nsif2} up")
2730 # Cache the MAC values, and reverse mapping
2731 self
.get_mac(name1
, nsif1
)
2732 self
.get_mac(name2
, nsif2
)
2734 # Setup interface constraints if provided
2735 if intf_constraints
:
2736 node1
.set_intf_constraints(if1
, **intf_constraints
)
2737 node2
.set_intf_constraints(if2
, **intf_constraints
)
2739 def add_switch(self
, name
, cls
=Bridge
, **kwargs
):
2740 """Add a switch to munet."""
2741 self
.logger
.debug("%s: add_switch %s(%s)", self
, cls
.__name
__, name
)
2742 self
.switches
[name
] = cls(name
, unet
=self
, **kwargs
)
2743 return self
.switches
[name
]
2745 def get_mac(self
, name
, ifname
):
2746 if name
in self
.hosts
:
2747 dev
= self
.hosts
[name
]
2749 dev
= self
.switches
[name
]
2751 nsifname
= self
.get_ns_ifname(ifname
)
2753 if (name
, ifname
) not in self
.macs
:
2754 _
, output
, _
= dev
.cmd_status_nsonly("ip -o link show " + nsifname
)
2755 m
= re
.match(".*link/(loopback|ether) ([0-9a-fA-F:]+) .*", output
)
2757 self
.macs
[(name
, ifname
)] = mac
2758 self
.rmacs
[mac
] = (name
, ifname
)
2760 return self
.macs
[(name
, ifname
)]
2762 async def _delete_link(self
, lname
):
2763 rname
, rif
= self
.links
[lname
][2:4]
2764 host
= self
.hosts
[rname
]
2765 nsrif
= host
.get_ns_ifname(rif
)
2767 self
.logger
.debug("%s: Deleting veth pair for link %s", self
, lname
)
2768 rc
, o
, e
= await host
.async_cmd_status_nsonly(
2769 [self
.ip_path
, "link", "delete", nsrif
],
2770 stdin
=subprocess
.DEVNULL
,
2771 start_new_session
=True,
2775 self
.logger
.error("Err del veth pair %s: %s", lname
, cmd_error(rc
, o
, e
))
2777 async def _delete_links(self
):
2778 # for x in self.links:
2779 # await self._delete_link(x)
2780 return await asyncio
.gather(*[self
._delete
_link
(x
) for x
in self
.links
])
2782 async def _async_delete(self
):
2783 """Delete the munet topology."""
2784 # logger = self.logger if False else logging
2785 logger
= self
.logger
2786 if type(self
) == BaseMunet
: # pylint: disable=C0123
2787 logger
.info("%s: deleting.", self
)
2789 logger
.debug("%s: BaseMunet sub-class deleting.", self
)
2791 logger
.debug("Deleting links")
2793 await self
._delete
_links
()
2794 except Exception as error
:
2795 logger
.error("%s: error deleting links: %s", self
, error
, exc_info
=True)
2797 logger
.debug("Deleting hosts and bridges")
2799 # Delete hosts and switches, wait for them all to complete
2800 # even if there is an exception.
2801 htask
= [x
.async_delete() for x
in self
.hosts
.values()]
2802 stask
= [x
.async_delete() for x
in self
.switches
.values()]
2803 await asyncio
.gather(*htask
, *stask
, return_exceptions
=True)
2804 except Exception as error
:
2806 "%s: error deleting hosts and switches: %s", self
, error
, exc_info
=True
2815 self
.cli_server
.cancel()
2816 self
.cli_server
= None
2817 if self
.cli_sockpath
:
2818 await self
.async_cmd_status(
2819 "rm -rf " + os
.path
.dirname(self
.cli_sockpath
)
2821 self
.cli_sockpath
= None
2822 except Exception as error
:
2824 "%s: error cli server or sockpaths: %s", self
, error
, exc_info
=True
2828 if self
.cli_histfile
:
2829 readline
.write_history_file(self
.cli_histfile
)
2830 self
.cli_histfile
= None
2831 except Exception as error
:
2833 "%s: error saving history file: %s", self
, error
, exc_info
=True
2836 # XXX for some reason setns during the delete is changing our dir to /.
2840 await super()._async
_delete
()
2841 except Exception as error
:
2843 "%s: error deleting parent classes: %s", self
, error
, exc_info
=True
2848 if self
.proc_path
and str(self
.proc_path
) != "/proc":
2849 logger
.debug("%s: umount, remove proc_path %s", self
, self
.proc_path
)
2850 linux
.umount(str(self
.proc_path
), 0)
2851 os
.rmdir(self
.proc_path
)
2852 except Exception as error
:
2854 "%s: error umount and removing proc_path %s: %s",
2861 linux
.umount(str(self
.proc_path
), linux
.MNT_DETACH
)
2862 except Exception as error2
:
2864 "%s: error umount with detach proc_path %s: %s",
2871 if BaseMunet
.g_unet
== self
:
2872 BaseMunet
.g_unet
= None
2875 BaseMunet
.g_unet
= None
2877 if True: # pylint: disable=using-constant-test
2880 """A Read-Execute-Print-Loop (REPL) interface.
2882 A newline or prompt changing command should be sent to the
2883 spawned child prior to creation as the `prompt` will be `expect`ed
2890 continuation_prompt
=None,
2891 extra_init_cmd
=None,
2895 self
.echo
= will_echo
2897 re
.compile(r
"(\x9B|\x1B\[)[0-?]*[ -\/]*[@-~]") if escape_ansi
else None
2901 'ShellWraper: XXX prompt "%s" will_echo %s child.echo %s',
2909 logging
.info("Setting child to echo")
2910 self
.child
.setecho(False)
2911 self
.child
.waitnoecho()
2912 assert not self
.child
.echo
2914 self
.prompt
= prompt
2915 self
.cont_prompt
= continuation_prompt
2917 # Use expect_exact if we can as it should be faster
2918 self
.expects
= [prompt
]
2919 if re
.escape(prompt
) == prompt
and hasattr(self
.child
, "expect_exact"):
2920 self
._expectf
= self
.child
.expect_exact
2922 self
._expectf
= self
.child
.expect
2923 if continuation_prompt
:
2924 self
.expects
.append(continuation_prompt
)
2925 if re
.escape(continuation_prompt
) != continuation_prompt
:
2926 self
._expectf
= self
.child
.expect
2929 self
.expect_prompt()
2930 self
.child
.sendline(extra_init_cmd
)
2931 self
.expect_prompt()
2933 def expect_prompt(self
, timeout
=-1):
2934 return self
._expectf
(self
.expects
, timeout
=timeout
)
2936 def run_command(self
, command
, timeout
=-1):
2937 """Pexpect REPLWrapper compatible run_command.
2939 This will split `command` into lines and feed each one to the shell.
2942 command: string of commands separated by newlines, a trailing
2943 newline will cause and empty line to be sent.
2944 timeout: pexpect timeout value.
2946 lines
= command
.splitlines()
2947 if command
[-1] == "\n":
2952 self
.child
.sendline(line
)
2953 index
= self
.expect_prompt(timeout
=timeout
)
2954 output
+= self
.child
.before
2957 if hasattr(self
.child
, "kill"):
2958 self
.child
.kill(signal
.SIGINT
)
2960 self
.child
.send("\x03")
2961 self
.expect_prompt(timeout
=30 if self
.child
.timeout
is None else -1)
2962 raise ValueError("Continuation prompt found at end of commands")
2965 output
= self
.escape
.sub("", output
)
2969 def cmd_nostatus(self
, cmd
, timeout
=-1):
2970 r
"""Execute a shell command.
2973 (strip/cleaned \r) output
2975 output
= self
.run_command(cmd
, timeout
)
2976 output
= output
.replace("\r\n", "\n")
2978 # remove the command
2979 idx
= output
.find(cmd
)
2982 "Didn't find command ('%s') in expected output ('%s')",
2987 # Remove up to and including the command from the output stream
2988 output
= output
[idx
+ len(cmd
) :]
2990 return output
.replace("\r", "").strip()
2992 def cmd_status(self
, cmd
, timeout
=-1):
2993 r
"""Execute a shell command.
2996 status and (strip/cleaned \r) output
2998 # Run the command getting the output
2999 output
= self
.cmd_nostatus(cmd
, timeout
)
3001 # Now get the status
3003 rcstr
= self
.run_command(scmd
)
3004 rcstr
= rcstr
.replace("\r\n", "\n")
3006 # remove the command
3007 idx
= rcstr
.find(scmd
)
3011 "Didn't find status ('%s') in expected output ('%s')",
3020 rcstr
= rcstr
[idx
+ len(scmd
) :].strip()
3023 except ValueError as error
:
3025 "%s: error with expected status output: %s: %s",
3034 def cmd_raises(self
, cmd
, timeout
=-1):
3035 r
"""Execute a shell command.
3038 (strip/cleaned \r) ouptut
3041 CalledProcessError: on non-zero exit status
3043 rc
, output
= self
.cmd_status(cmd
, timeout
)
3045 raise CalledProcessError(rc
, cmd
, output
)
3049 # ---------------------------
3050 # Root level utility function
3051 # ---------------------------
3054 def get_exec_path(binary
):
3055 return commander
.get_exec_path(binary
)
3058 def get_exec_path_host(binary
):
3059 return commander
.get_exec_path(binary
)
3062 def get_our_script_path(script
):
3063 # would be nice to find this w/o using a path lookup
3064 sdir
= os
.path
.dirname(os
.path
.abspath(__file__
))
3065 spath
= os
.path
.join(sdir
, script
)
3066 if os
.path
.exists(spath
):
3068 return get_exec_path(script
)
3071 commander
= Commander("munet")