proxmox-file-restore/src/qemu_helper.rs

   1 //! Helper to start a QEMU VM for single file restore.
   2 use std::fs::{File, OpenOptions};
   3 use std::io::prelude::*;
   4 use std::os::unix::io::AsRawFd;
   5 use std::path::PathBuf;
   6 use std::time::Duration;
   7
   8 use anyhow::{bail, format_err, Error};
   9 use tokio::time;
  10
  11 use nix::sys::signal::{kill, Signal};
  12 use nix::unistd::Pid;
  13
  14 use proxmox::tools::fs::{create_path, file_read_string, make_tmp_file, CreateOptions};
  15 use proxmox::tools::fd::fd_change_cloexec;
  16 use proxmox_sys::logrotate::LogRotate;
  17
  18 use pbs_client::{VsockClient, DEFAULT_VSOCK_PORT};
  19
  20 use crate::{cpio, backup_user};
  21 use super::SnapRestoreDetails;
  22
  23 const PBS_VM_NAME: &str = "pbs-restore-vm";
  24 const MAX_CID_TRIES: u64 = 32;
  25
  26 fn create_restore_log_dir() -> Result<String, Error> {
  27     let logpath = format!("{}/file-restore", pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR);
  28
  29     proxmox_lang::try_block!({
  30         let backup_user = backup_user()?;
  31         let opts = CreateOptions::new()
  32             .owner(backup_user.uid)
  33             .group(backup_user.gid);
  34
  35         let opts_root = CreateOptions::new()
  36             .owner(nix::unistd::ROOT)
  37             .group(nix::unistd::Gid::from_raw(0));
  38
  39         create_path(pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR, None, Some(opts))?;
  40         create_path(&logpath, None, Some(opts_root))?;
  41         Ok(())
  42     })
  43     .map_err(|err: Error| format_err!("unable to create file-restore log dir - {}", err))?;
  44
  45     Ok(logpath)
  46 }
  47
  48 fn validate_img_existance(debug: bool) -> Result<(), Error> {
  49     let kernel = PathBuf::from(pbs_buildcfg::PROXMOX_BACKUP_KERNEL_FN);
  50     let initramfs = PathBuf::from(if debug {
  51         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_DBG_FN
  52     } else {
  53         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_FN
  54     });
  55     if !kernel.exists() || !initramfs.exists() {
  56         bail!("cannot run file-restore VM: package 'proxmox-backup-restore-image' is not (correctly) installed");
  57     }
  58     Ok(())
  59 }
  60
  61 pub fn try_kill_vm(pid: i32) -> Result<(), Error> {
  62     let pid = Pid::from_raw(pid);
  63     if let Ok(()) = kill(pid, None) {
  64         // process is running (and we could kill it), check if it is actually ours
  65         // (if it errors assume we raced with the process's death and ignore it)
  66         if let Ok(cmdline) = file_read_string(format!("/proc/{}/cmdline", pid)) {
  67             if cmdline.split('\0').any(|a| a == PBS_VM_NAME) {
  68                 // yes, it's ours, kill it brutally with SIGKILL, no reason to take
  69                 // any chances - in this state it's most likely broken anyway
  70                 if let Err(err) = kill(pid, Signal::SIGKILL) {
  71                     bail!(
  72                         "reaping broken VM (pid {}) with SIGKILL failed: {}",
  73                         pid,
  74                         err
  75                     );
  76                 }
  77             }
  78         }
  79     }
  80
  81     Ok(())
  82 }
  83
  84 async fn create_temp_initramfs(ticket: &str, debug: bool) -> Result<(File, String), Error> {
  85     use std::ffi::CString;
  86     use tokio::fs::File;
  87
  88     let (tmp_file, tmp_path) =
  89         make_tmp_file("/tmp/file-restore-qemu.initramfs.tmp", CreateOptions::new())?;
  90     nix::unistd::unlink(&tmp_path)?;
  91     fd_change_cloexec(tmp_file.as_raw_fd(), false)?;
  92
  93     let initramfs = if debug {
  94         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_DBG_FN
  95     } else {
  96         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_FN
  97     };
  98
  99     let mut f = File::from_std(tmp_file);
 100     let mut base = File::open(initramfs).await?;
 101
 102     tokio::io::copy(&mut base, &mut f).await?;
 103
 104     let name = CString::new("ticket").unwrap();
 105     cpio::append_file(
 106         &mut f,
 107         ticket.as_bytes(),
 108         &name,
 109         0,
 110         (libc::S_IFREG | 0o400) as u16,
 111         0,
 112         0,
 113         0,
 114         ticket.len() as u32,
 115     )
 116     .await?;
 117     cpio::append_trailer(&mut f).await?;
 118
 119     let tmp_file = f.into_std().await;
 120     let path = format!("/dev/fd/{}", &tmp_file.as_raw_fd());
 121
 122     Ok((tmp_file, path))
 123 }
 124
 125 pub async fn start_vm(
 126     // u16 so we can do wrapping_add without going too high
 127     mut cid: u16,
 128     details: &SnapRestoreDetails,
 129     files: impl Iterator<Item = String>,
 130     ticket: &str,
 131 ) -> Result<(i32, i32), Error> {
 132     if let Err(_) = std::env::var("PBS_PASSWORD") {
 133         bail!("environment variable PBS_PASSWORD has to be set for QEMU VM restore");
 134     }
 135
 136     let debug = if let Ok(val) = std::env::var("PBS_QEMU_DEBUG") {
 137         !val.is_empty()
 138     } else {
 139         false
 140     };
 141
 142     validate_img_existance(debug)?;
 143
 144     let pid;
 145     let (mut pid_file, pid_path) = make_tmp_file("/tmp/file-restore-qemu.pid.tmp", CreateOptions::new())?;
 146     nix::unistd::unlink(&pid_path)?;
 147     fd_change_cloexec(pid_file.as_raw_fd(), false)?;
 148
 149     let (_ramfs_pid, ramfs_path) = create_temp_initramfs(ticket, debug).await?;
 150
 151     let logpath = create_restore_log_dir()?;
 152     let logfile = &format!("{}/qemu.log", logpath);
 153     let mut logrotate = LogRotate::new(logfile, false, Some(16), None)?;
 154
 155     if let Err(err) = logrotate.do_rotate() {
 156         eprintln!("warning: logrotate for QEMU log file failed - {}", err);
 157     }
 158
 159     let mut logfd = OpenOptions::new()
 160         .append(true)
 161         .create_new(true)
 162         .open(logfile)?;
 163     fd_change_cloexec(logfd.as_raw_fd(), false)?;
 164
 165     // preface log file with start timestamp so one can see how long QEMU took to start
 166     writeln!(logfd, "[{}] PBS file restore VM log", {
 167         let now = proxmox_time::epoch_i64();
 168         proxmox_time::epoch_to_rfc3339(now)?
 169     },)?;
 170
 171     let base_args = [
 172         "-chardev",
 173         &format!(
 174             "file,id=log,path=/dev/null,logfile=/dev/fd/{},logappend=on",
 175             logfd.as_raw_fd()
 176         ),
 177         "-serial",
 178         "chardev:log",
 179         "-vnc",
 180         "none",
 181         "-enable-kvm",
 182         "-kernel",
 183         pbs_buildcfg::PROXMOX_BACKUP_KERNEL_FN,
 184         "-initrd",
 185         &ramfs_path,
 186         "-append",
 187         &format!(
 188             "{} panic=1 zfs_arc_min=0 zfs_arc_max=0",
 189             if debug { "debug" } else { "quiet" }
 190         ),
 191         "-daemonize",
 192         "-pidfile",
 193         &format!("/dev/fd/{}", pid_file.as_raw_fd()),
 194         "-name",
 195         PBS_VM_NAME,
 196     ];
 197
 198     // Generate drive arguments for all fidx files in backup snapshot
 199     let mut drives = Vec::new();
 200     let mut id = 0;
 201     for file in files {
 202         if !file.ends_with(".img.fidx") {
 203             continue;
 204         }
 205         drives.push("-drive".to_owned());
 206         let keyfile = if let Some(ref keyfile) = details.keyfile {
 207             format!(",,keyfile={}", keyfile)
 208         } else {
 209             "".to_owned()
 210         };
 211         drives.push(format!(
 212             "file=pbs:repository={},,snapshot={},,archive={}{},read-only=on,if=none,id=drive{}",
 213             details.repo, details.snapshot, file, keyfile, id
 214         ));
 215
 216         // a PCI bus can only support 32 devices, so add a new one every 32
 217         let bus = (id / 32) + 2;
 218         if id % 32 == 0 {
 219             drives.push("-device".to_owned());
 220             drives.push(format!("pci-bridge,id=bridge{},chassis_nr={}", bus, bus));
 221         }
 222
 223         drives.push("-device".to_owned());
 224         // drive serial is used by VM to map .fidx files to /dev paths
 225         let serial = file.strip_suffix(".img.fidx").unwrap_or(&file);
 226         drives.push(format!(
 227             "virtio-blk-pci,drive=drive{},serial={},bus=bridge{}",
 228             id, serial, bus
 229         ));
 230         id += 1;
 231     }
 232
 233     let ram = if debug {
 234         1024
 235     } else {
 236         // add more RAM if many drives are given
 237         match id {
 238             f if f < 10 => 192,
 239             f if f < 20 => 256,
 240             _ => 384,
 241         }
 242     };
 243
 244     // Try starting QEMU in a loop to retry if we fail because of a bad 'cid' value
 245     let mut attempts = 0;
 246     loop {
 247         let mut qemu_cmd = std::process::Command::new("qemu-system-x86_64");
 248         qemu_cmd.args(base_args.iter());
 249         qemu_cmd.arg("-m");
 250         qemu_cmd.arg(ram.to_string());
 251         qemu_cmd.args(&drives);
 252         qemu_cmd.arg("-device");
 253         qemu_cmd.arg(format!(
 254             "vhost-vsock-pci,guest-cid={},disable-legacy=on",
 255             cid
 256         ));
 257
 258         if debug {
 259             let debug_args = [
 260                 "-chardev",
 261                 &format!(
 262                     "socket,id=debugser,path=/run/proxmox-backup/file-restore-serial-{}.sock,server,nowait",
 263                     cid
 264                 ),
 265                 "-serial",
 266                 "chardev:debugser",
 267             ];
 268             qemu_cmd.args(debug_args.iter());
 269         }
 270
 271         qemu_cmd.stdout(std::process::Stdio::null());
 272         qemu_cmd.stderr(std::process::Stdio::piped());
 273
 274         let res = tokio::task::block_in_place(|| qemu_cmd.spawn()?.wait_with_output())?;
 275
 276         if res.status.success() {
 277             // at this point QEMU is already daemonized and running, so if anything fails we
 278             // technically leave behind a zombie-VM... this shouldn't matter, as it will stop
 279             // itself soon enough (timer), and the following operations are unlikely to fail
 280             let mut pidstr = String::new();
 281             pid_file.read_to_string(&mut pidstr)?;
 282             pid = pidstr.trim_end().parse().map_err(|err| {
 283                 format_err!("cannot parse PID returned by QEMU ('{}'): {}", &pidstr, err)
 284             })?;
 285             break;
 286         } else {
 287             let out = String::from_utf8_lossy(&res.stderr);
 288             if out.contains("unable to set guest cid: Address already in use") {
 289                 attempts += 1;
 290                 if attempts >= MAX_CID_TRIES {
 291                     bail!("CID '{}' in use, but max attempts reached, aborting", cid);
 292                 }
 293                 // CID in use, try next higher one
 294                 eprintln!("CID '{}' in use by other VM, attempting next one", cid);
 295                 // skip special-meaning low values
 296                 cid = cid.wrapping_add(1).max(10);
 297             } else {
 298                 eprint!("{}", out);
 299                 bail!("Starting VM failed. See output above for more information.");
 300             }
 301         }
 302     }
 303
 304     // QEMU has started successfully, now wait for virtio socket to become ready
 305     let pid_t = Pid::from_raw(pid);
 306     for _ in 0..60 {
 307         let client = VsockClient::new(cid as i32, DEFAULT_VSOCK_PORT, Some(ticket.to_owned()));
 308         if let Ok(Ok(_)) =
 309             time::timeout(Duration::from_secs(2), client.get("api2/json/status", None)).await
 310         {
 311             if debug {
 312                 eprintln!(
 313                     "Connect to '/run/proxmox-backup/file-restore-serial-{}.sock' for shell access",
 314                     cid
 315                 )
 316             }
 317             return Ok((pid, cid as i32));
 318         }
 319         if kill(pid_t, None).is_err() { // check if QEMU process exited in between
 320             bail!("VM exited before connection could be established");
 321         }
 322         time::sleep(Duration::from_millis(200)).await;
 323     }
 324
 325     // start failed
 326     if let Err(err) = try_kill_vm(pid) {
 327         eprintln!("killing failed VM failed: {}", err);
 328     }
 329     bail!("starting VM timed out");
 330 }