src/bin/proxmox_file_restore/qemu_helper.rs

   1 //! Helper to start a QEMU VM for single file restore.
   2 use std::fs::{File, OpenOptions};
   3 use std::io::prelude::*;
   4 use std::os::unix::io::AsRawFd;
   5 use std::path::PathBuf;
   6 use std::time::Duration;
   7
   8 use anyhow::{bail, format_err, Error};
   9 use tokio::time;
  10
  11 use nix::sys::signal::{kill, Signal};
  12 use nix::unistd::Pid;
  13
  14 use proxmox::tools::fs::{create_path, file_read_string, make_tmp_file, CreateOptions};
  15
  16 use pbs_client::{VsockClient, DEFAULT_VSOCK_PORT};
  17
  18 use proxmox_backup::backup::backup_user;
  19 use proxmox_backup::tools;
  20
  21 use super::SnapRestoreDetails;
  22
  23 const PBS_VM_NAME: &str = "pbs-restore-vm";
  24 const MAX_CID_TRIES: u64 = 32;
  25
  26 fn create_restore_log_dir() -> Result<String, Error> {
  27     let logpath = format!("{}/file-restore", pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR);
  28
  29     proxmox::try_block!({
  30         let backup_user = backup_user()?;
  31         let opts = CreateOptions::new()
  32             .owner(backup_user.uid)
  33             .group(backup_user.gid);
  34
  35         let opts_root = CreateOptions::new()
  36             .owner(nix::unistd::ROOT)
  37             .group(nix::unistd::Gid::from_raw(0));
  38
  39         create_path(pbs_buildcfg::PROXMOX_BACKUP_LOG_DIR, None, Some(opts))?;
  40         create_path(&logpath, None, Some(opts_root))?;
  41         Ok(())
  42     })
  43     .map_err(|err: Error| format_err!("unable to create file-restore log dir - {}", err))?;
  44
  45     Ok(logpath)
  46 }
  47
  48 fn validate_img_existance(debug: bool) -> Result<(), Error> {
  49     let kernel = PathBuf::from(pbs_buildcfg::PROXMOX_BACKUP_KERNEL_FN);
  50     let initramfs = PathBuf::from(if debug {
  51         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_DBG_FN
  52     } else {
  53         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_FN
  54     });
  55     if !kernel.exists() || !initramfs.exists() {
  56         bail!("cannot run file-restore VM: package 'proxmox-backup-restore-image' is not (correctly) installed");
  57     }
  58     Ok(())
  59 }
  60
  61 pub fn try_kill_vm(pid: i32) -> Result<(), Error> {
  62     let pid = Pid::from_raw(pid);
  63     if let Ok(()) = kill(pid, None) {
  64         // process is running (and we could kill it), check if it is actually ours
  65         // (if it errors assume we raced with the process's death and ignore it)
  66         if let Ok(cmdline) = file_read_string(format!("/proc/{}/cmdline", pid)) {
  67             if cmdline.split('\0').any(|a| a == PBS_VM_NAME) {
  68                 // yes, it's ours, kill it brutally with SIGKILL, no reason to take
  69                 // any chances - in this state it's most likely broken anyway
  70                 if let Err(err) = kill(pid, Signal::SIGKILL) {
  71                     bail!(
  72                         "reaping broken VM (pid {}) with SIGKILL failed: {}",
  73                         pid,
  74                         err
  75                     );
  76                 }
  77             }
  78         }
  79     }
  80
  81     Ok(())
  82 }
  83
  84 async fn create_temp_initramfs(ticket: &str, debug: bool) -> Result<(File, String), Error> {
  85     use std::ffi::CString;
  86     use tokio::fs::File;
  87
  88     let (tmp_file, tmp_path) =
  89         make_tmp_file("/tmp/file-restore-qemu.initramfs.tmp", CreateOptions::new())?;
  90     nix::unistd::unlink(&tmp_path)?;
  91     tools::fd_change_cloexec(tmp_file.as_raw_fd(), false)?;
  92
  93     let initramfs = if debug {
  94         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_DBG_FN
  95     } else {
  96         pbs_buildcfg::PROXMOX_BACKUP_INITRAMFS_FN
  97     };
  98
  99     let mut f = File::from_std(tmp_file);
 100     let mut base = File::open(initramfs).await?;
 101
 102     tokio::io::copy(&mut base, &mut f).await?;
 103
 104     let name = CString::new("ticket").unwrap();
 105     tools::cpio::append_file(
 106         &mut f,
 107         ticket.as_bytes(),
 108         &name,
 109         0,
 110         (libc::S_IFREG | 0o400) as u16,
 111         0,
 112         0,
 113         0,
 114         ticket.len() as u32,
 115     )
 116     .await?;
 117     tools::cpio::append_trailer(&mut f).await?;
 118
 119     let tmp_file = f.into_std().await;
 120     let path = format!("/dev/fd/{}", &tmp_file.as_raw_fd());
 121
 122     Ok((tmp_file, path))
 123 }
 124
 125 pub async fn start_vm(
 126     // u16 so we can do wrapping_add without going too high
 127     mut cid: u16,
 128     details: &SnapRestoreDetails,
 129     files: impl Iterator<Item = String>,
 130     ticket: &str,
 131 ) -> Result<(i32, i32), Error> {
 132     if let Err(_) = std::env::var("PBS_PASSWORD") {
 133         bail!("environment variable PBS_PASSWORD has to be set for QEMU VM restore");
 134     }
 135
 136     let debug = if let Ok(val) = std::env::var("PBS_QEMU_DEBUG") {
 137         !val.is_empty()
 138     } else {
 139         false
 140     };
 141
 142     validate_img_existance(debug)?;
 143
 144     let pid;
 145     let (mut pid_file, pid_path) = make_tmp_file("/tmp/file-restore-qemu.pid.tmp", CreateOptions::new())?;
 146     nix::unistd::unlink(&pid_path)?;
 147     tools::fd_change_cloexec(pid_file.as_raw_fd(), false)?;
 148
 149     let (_ramfs_pid, ramfs_path) = create_temp_initramfs(ticket, debug).await?;
 150
 151     let logpath = create_restore_log_dir()?;
 152     let logfile = &format!("{}/qemu.log", logpath);
 153     let mut logrotate = tools::logrotate::LogRotate::new(logfile, false)
 154         .ok_or_else(|| format_err!("could not get QEMU log file names"))?;
 155
 156     if let Err(err) = logrotate.do_rotate(CreateOptions::default(), Some(16)) {
 157         eprintln!("warning: logrotate for QEMU log file failed - {}", err);
 158     }
 159
 160     let mut logfd = OpenOptions::new()
 161         .append(true)
 162         .create_new(true)
 163         .open(logfile)?;
 164     tools::fd_change_cloexec(logfd.as_raw_fd(), false)?;
 165
 166     // preface log file with start timestamp so one can see how long QEMU took to start
 167     writeln!(logfd, "[{}] PBS file restore VM log", {
 168         let now = proxmox::tools::time::epoch_i64();
 169         proxmox::tools::time::epoch_to_rfc3339(now)?
 170     },)?;
 171
 172     let base_args = [
 173         "-chardev",
 174         &format!(
 175             "file,id=log,path=/dev/null,logfile=/dev/fd/{},logappend=on",
 176             logfd.as_raw_fd()
 177         ),
 178         "-serial",
 179         "chardev:log",
 180         "-vnc",
 181         "none",
 182         "-enable-kvm",
 183         "-kernel",
 184         pbs_buildcfg::PROXMOX_BACKUP_KERNEL_FN,
 185         "-initrd",
 186         &ramfs_path,
 187         "-append",
 188         &format!(
 189             "{} panic=1 zfs_arc_min=0 zfs_arc_max=0",
 190             if debug { "debug" } else { "quiet" }
 191         ),
 192         "-daemonize",
 193         "-pidfile",
 194         &format!("/dev/fd/{}", pid_file.as_raw_fd()),
 195         "-name",
 196         PBS_VM_NAME,
 197     ];
 198
 199     // Generate drive arguments for all fidx files in backup snapshot
 200     let mut drives = Vec::new();
 201     let mut id = 0;
 202     for file in files {
 203         if !file.ends_with(".img.fidx") {
 204             continue;
 205         }
 206         drives.push("-drive".to_owned());
 207         let keyfile = if let Some(ref keyfile) = details.keyfile {
 208             format!(",,keyfile={}", keyfile)
 209         } else {
 210             "".to_owned()
 211         };
 212         drives.push(format!(
 213             "file=pbs:repository={},,snapshot={},,archive={}{},read-only=on,if=none,id=drive{}",
 214             details.repo, details.snapshot, file, keyfile, id
 215         ));
 216
 217         // a PCI bus can only support 32 devices, so add a new one every 32
 218         let bus = (id / 32) + 2;
 219         if id % 32 == 0 {
 220             drives.push("-device".to_owned());
 221             drives.push(format!("pci-bridge,id=bridge{},chassis_nr={}", bus, bus));
 222         }
 223
 224         drives.push("-device".to_owned());
 225         // drive serial is used by VM to map .fidx files to /dev paths
 226         let serial = file.strip_suffix(".img.fidx").unwrap_or(&file);
 227         drives.push(format!(
 228             "virtio-blk-pci,drive=drive{},serial={},bus=bridge{}",
 229             id, serial, bus
 230         ));
 231         id += 1;
 232     }
 233
 234     let ram = if debug {
 235         1024
 236     } else {
 237         // add more RAM if many drives are given
 238         match id {
 239             f if f < 10 => 192,
 240             f if f < 20 => 256,
 241             _ => 384,
 242         }
 243     };
 244
 245     // Try starting QEMU in a loop to retry if we fail because of a bad 'cid' value
 246     let mut attempts = 0;
 247     loop {
 248         let mut qemu_cmd = std::process::Command::new("qemu-system-x86_64");
 249         qemu_cmd.args(base_args.iter());
 250         qemu_cmd.arg("-m");
 251         qemu_cmd.arg(ram.to_string());
 252         qemu_cmd.args(&drives);
 253         qemu_cmd.arg("-device");
 254         qemu_cmd.arg(format!(
 255             "vhost-vsock-pci,guest-cid={},disable-legacy=on",
 256             cid
 257         ));
 258
 259         if debug {
 260             let debug_args = [
 261                 "-chardev",
 262                 &format!(
 263                     "socket,id=debugser,path=/run/proxmox-backup/file-restore-serial-{}.sock,server,nowait",
 264                     cid
 265                 ),
 266                 "-serial",
 267                 "chardev:debugser",
 268             ];
 269             qemu_cmd.args(debug_args.iter());
 270         }
 271
 272         qemu_cmd.stdout(std::process::Stdio::null());
 273         qemu_cmd.stderr(std::process::Stdio::piped());
 274
 275         let res = tokio::task::block_in_place(|| qemu_cmd.spawn()?.wait_with_output())?;
 276
 277         if res.status.success() {
 278             // at this point QEMU is already daemonized and running, so if anything fails we
 279             // technically leave behind a zombie-VM... this shouldn't matter, as it will stop
 280             // itself soon enough (timer), and the following operations are unlikely to fail
 281             let mut pidstr = String::new();
 282             pid_file.read_to_string(&mut pidstr)?;
 283             pid = pidstr.trim_end().parse().map_err(|err| {
 284                 format_err!("cannot parse PID returned by QEMU ('{}'): {}", &pidstr, err)
 285             })?;
 286             break;
 287         } else {
 288             let out = String::from_utf8_lossy(&res.stderr);
 289             if out.contains("unable to set guest cid: Address already in use") {
 290                 attempts += 1;
 291                 if attempts >= MAX_CID_TRIES {
 292                     bail!("CID '{}' in use, but max attempts reached, aborting", cid);
 293                 }
 294                 // CID in use, try next higher one
 295                 eprintln!("CID '{}' in use by other VM, attempting next one", cid);
 296                 // skip special-meaning low values
 297                 cid = cid.wrapping_add(1).max(10);
 298             } else {
 299                 eprint!("{}", out);
 300                 bail!("Starting VM failed. See output above for more information.");
 301             }
 302         }
 303     }
 304
 305     // QEMU has started successfully, now wait for virtio socket to become ready
 306     let pid_t = Pid::from_raw(pid);
 307     for _ in 0..60 {
 308         let client = VsockClient::new(cid as i32, DEFAULT_VSOCK_PORT, Some(ticket.to_owned()));
 309         if let Ok(Ok(_)) =
 310             time::timeout(Duration::from_secs(2), client.get("api2/json/status", None)).await
 311         {
 312             if debug {
 313                 eprintln!(
 314                     "Connect to '/run/proxmox-backup/file-restore-serial-{}.sock' for shell access",
 315                     cid
 316                 )
 317             }
 318             return Ok((pid, cid as i32));
 319         }
 320         if kill(pid_t, None).is_err() { // check if QEMU process exited in between
 321             bail!("VM exited before connection could be established");
 322         }
 323         time::sleep(Duration::from_millis(200)).await;
 324     }
 325
 326     // start failed
 327     if let Err(err) = try_kill_vm(pid) {
 328         eprintln!("killing failed VM failed: {}", err);
 329     }
 330     bail!("starting VM timed out");
 331 }