PVE/QemuServer/PCI.pm

   1 package PVE::QemuServer::PCI;
   2
   3 use PVE::JSONSchema;
   4 use PVE::SysFSTools;
   5
   6 use base 'Exporter';
   7
   8 our @EXPORT_OK = qw(
   9 print_pci_addr
  10 print_pcie_addr
  11 print_pcie_root_port
  12 parse_hostpci
  13 );
  14
  15 our $MAX_HOSTPCI_DEVICES = 16;
  16
  17 my $PCIRE = qr/([a-f0-9]{4}:)?[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
  18 my $hostpci_fmt = {
  19     host => {
  20         default_key => 1,
  21         type => 'string',
  22         pattern => qr/$PCIRE(;$PCIRE)*/,
  23         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
  24         description => <<EODESCR,
  25 Host PCI device pass through. The PCI ID of a host's PCI device or a list
  26 of PCI virtual functions of the host. HOSTPCIID syntax is:
  27
  28 'bus:dev.func' (hexadecimal numbers)
  29
  30 You can us the 'lspci' command to list existing PCI devices.
  31 EODESCR
  32     },
  33     rombar => {
  34         type => 'boolean',
  35         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
  36         optional => 1,
  37         default => 1,
  38     },
  39     romfile => {
  40         type => 'string',
  41         pattern => '[^,;]+',
  42         format_description => 'string',
  43         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
  44         optional => 1,
  45     },
  46     pcie => {
  47         type => 'boolean',
  48         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
  49         optional => 1,
  50         default => 0,
  51     },
  52     'x-vga' => {
  53         type => 'boolean',
  54         description =>  "Enable vfio-vga device support.",
  55         optional => 1,
  56         default => 0,
  57     },
  58     'mdev' => {
  59         type => 'string',
  60         format_description => 'string',
  61         pattern => '[^/\.:]+',
  62         optional => 1,
  63         description => <<EODESCR
  64 The type of mediated device to use.
  65 An instance of this type will be created on startup of the VM and
  66 will be cleaned up when the VM stops.
  67 EODESCR
  68     }
  69 };
  70 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
  71
  72 our $hostpcidesc = {
  73         optional => 1,
  74         type => 'string', format => 'pve-qm-hostpci',
  75         description => "Map host PCI devices into guest.",
  76         verbose_description =>  <<EODESCR,
  77 Map host PCI devices into guest.
  78
  79 NOTE: This option allows direct access to host hardware. So it is no longer
  80 possible to migrate such machines - use with special care.
  81
  82 CAUTION: Experimental! User reported problems with this option.
  83 EODESCR
  84 };
  85 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
  86
  87 my $pci_addr_map;
  88 sub get_pci_addr_map {
  89     $pci_addr_map = {
  90         piix3 => { bus => 0, addr => 1, conflict_ok => qw(ehci)  },
  91         ehci => { bus => 0, addr => 1, conflict_ok => qw(piix3) }, # instead of piix3 on arm
  92         vga => { bus => 0, addr => 2 },
  93         balloon0 => { bus => 0, addr => 3 },
  94         watchdog => { bus => 0, addr => 4 },
  95         scsihw0 => { bus => 0, addr => 5, conflict_ok => qw(pci.3) },
  96         'pci.3' => { bus => 0, addr => 5, conflict_ok => qw(scsihw0) }, # also used for virtio-scsi-single bridge
  97         scsihw1 => { bus => 0, addr => 6 },
  98         ahci0 => { bus => 0, addr => 7 },
  99         qga0 => { bus => 0, addr => 8 },
 100         spice => { bus => 0, addr => 9 },
 101         virtio0 => { bus => 0, addr => 10 },
 102         virtio1 => { bus => 0, addr => 11 },
 103         virtio2 => { bus => 0, addr => 12 },
 104         virtio3 => { bus => 0, addr => 13 },
 105         virtio4 => { bus => 0, addr => 14 },
 106         virtio5 => { bus => 0, addr => 15 },
 107         hostpci0 => { bus => 0, addr => 16 },
 108         hostpci1 => { bus => 0, addr => 17 },
 109         net0 => { bus => 0, addr => 18 },
 110         net1 => { bus => 0, addr => 19 },
 111         net2 => { bus => 0, addr => 20 },
 112         net3 => { bus => 0, addr => 21 },
 113         net4 => { bus => 0, addr => 22 },
 114         net5 => { bus => 0, addr => 23 },
 115         vga1 => { bus => 0, addr => 24 },
 116         vga2 => { bus => 0, addr => 25 },
 117         vga3 => { bus => 0, addr => 26 },
 118         hostpci2 => { bus => 0, addr => 27 },
 119         hostpci3 => { bus => 0, addr => 28 },
 120         #addr29 : usb-host (pve-usb.cfg)
 121         'pci.1' => { bus => 0, addr => 30 },
 122         'pci.2' => { bus => 0, addr => 31 },
 123         'net6' => { bus => 1, addr => 1 },
 124         'net7' => { bus => 1, addr => 2 },
 125         'net8' => { bus => 1, addr => 3 },
 126         'net9' => { bus => 1, addr => 4 },
 127         'net10' => { bus => 1, addr => 5 },
 128         'net11' => { bus => 1, addr => 6 },
 129         'net12' => { bus => 1, addr => 7 },
 130         'net13' => { bus => 1, addr => 8 },
 131         'net14' => { bus => 1, addr => 9 },
 132         'net15' => { bus => 1, addr => 10 },
 133         'net16' => { bus => 1, addr => 11 },
 134         'net17' => { bus => 1, addr => 12 },
 135         'net18' => { bus => 1, addr => 13 },
 136         'net19' => { bus => 1, addr => 14 },
 137         'net20' => { bus => 1, addr => 15 },
 138         'net21' => { bus => 1, addr => 16 },
 139         'net22' => { bus => 1, addr => 17 },
 140         'net23' => { bus => 1, addr => 18 },
 141         'net24' => { bus => 1, addr => 19 },
 142         'net25' => { bus => 1, addr => 20 },
 143         'net26' => { bus => 1, addr => 21 },
 144         'net27' => { bus => 1, addr => 22 },
 145         'net28' => { bus => 1, addr => 23 },
 146         'net29' => { bus => 1, addr => 24 },
 147         'net30' => { bus => 1, addr => 25 },
 148         'net31' => { bus => 1, addr => 26 },
 149         'xhci' => { bus => 1, addr => 27 },
 150         'pci.4' => { bus => 1, addr => 28 },
 151         'rng0' => { bus => 1, addr => 29 },
 152         'virtio6' => { bus => 2, addr => 1 },
 153         'virtio7' => { bus => 2, addr => 2 },
 154         'virtio8' => { bus => 2, addr => 3 },
 155         'virtio9' => { bus => 2, addr => 4 },
 156         'virtio10' => { bus => 2, addr => 5 },
 157         'virtio11' => { bus => 2, addr => 6 },
 158         'virtio12' => { bus => 2, addr => 7 },
 159         'virtio13' => { bus => 2, addr => 8 },
 160         'virtio14' => { bus => 2, addr => 9 },
 161         'virtio15' => { bus => 2, addr => 10 },
 162         'ivshmem' => { bus => 2, addr => 11 },
 163         'audio0' => { bus => 2, addr => 12 },
 164         hostpci4 => { bus => 2, addr => 13 },
 165         hostpci5 => { bus => 2, addr => 14 },
 166         hostpci6 => { bus => 2, addr => 15 },
 167         hostpci7 => { bus => 2, addr => 16 },
 168         hostpci8 => { bus => 2, addr => 17 },
 169         hostpci9 => { bus => 2, addr => 18 },
 170         hostpci10 => { bus => 2, addr => 19 },
 171         hostpci11 => { bus => 2, addr => 20 },
 172         hostpci12 => { bus => 2, addr => 21 },
 173         hostpci13 => { bus => 2, addr => 22 },
 174         hostpci14 => { bus => 2, addr => 23 },
 175         hostpci15 => { bus => 2, addr => 24 },
 176         'virtioscsi0' => { bus => 3, addr => 1 },
 177         'virtioscsi1' => { bus => 3, addr => 2 },
 178         'virtioscsi2' => { bus => 3, addr => 3 },
 179         'virtioscsi3' => { bus => 3, addr => 4 },
 180         'virtioscsi4' => { bus => 3, addr => 5 },
 181         'virtioscsi5' => { bus => 3, addr => 6 },
 182         'virtioscsi6' => { bus => 3, addr => 7 },
 183         'virtioscsi7' => { bus => 3, addr => 8 },
 184         'virtioscsi8' => { bus => 3, addr => 9 },
 185         'virtioscsi9' => { bus => 3, addr => 10 },
 186         'virtioscsi10' => { bus => 3, addr => 11 },
 187         'virtioscsi11' => { bus => 3, addr => 12 },
 188         'virtioscsi12' => { bus => 3, addr => 13 },
 189         'virtioscsi13' => { bus => 3, addr => 14 },
 190         'virtioscsi14' => { bus => 3, addr => 15 },
 191         'virtioscsi15' => { bus => 3, addr => 16 },
 192         'virtioscsi16' => { bus => 3, addr => 17 },
 193         'virtioscsi17' => { bus => 3, addr => 18 },
 194         'virtioscsi18' => { bus => 3, addr => 19 },
 195         'virtioscsi19' => { bus => 3, addr => 20 },
 196         'virtioscsi20' => { bus => 3, addr => 21 },
 197         'virtioscsi21' => { bus => 3, addr => 22 },
 198         'virtioscsi22' => { bus => 3, addr => 23 },
 199         'virtioscsi23' => { bus => 3, addr => 24 },
 200         'virtioscsi24' => { bus => 3, addr => 25 },
 201         'virtioscsi25' => { bus => 3, addr => 26 },
 202         'virtioscsi26' => { bus => 3, addr => 27 },
 203         'virtioscsi27' => { bus => 3, addr => 28 },
 204         'virtioscsi28' => { bus => 3, addr => 29 },
 205         'virtioscsi29' => { bus => 3, addr => 30 },
 206         'virtioscsi30' => { bus => 3, addr => 31 },
 207         'scsihw2' => { bus => 4, addr => 1 },
 208         'scsihw3' => { bus => 4, addr => 2 },
 209         'scsihw4' => { bus => 4, addr => 3 },
 210     } if !defined($pci_addr_map);
 211     return $pci_addr_map;
 212 }
 213
 214 my $get_addr_mapping_from_id = sub {
 215     my ($map, $id) = @_;
 216
 217     my $d = $map->{$id};
 218     return undef if !defined($d) || !defined($d->{bus}) || !defined($d->{addr});
 219
 220     return { bus => $d->{bus}, addr => sprintf("0x%x", $d->{addr}) };
 221 };
 222
 223 sub print_pci_addr {
 224     my ($id, $bridges, $arch, $machine) = @_;
 225
 226     my $res = '';
 227
 228     # using same bus slots on all HW, so we need to check special cases here:
 229     my $busname = 'pci';
 230     if ($arch eq 'aarch64' && $machine =~ /^virt/) {
 231         die "aarch64/virt cannot use IDE devices\n" if $id =~ /^ide/;
 232         $busname = 'pcie';
 233     }
 234
 235     my $map = get_pci_addr_map();
 236     if (my $d = $get_addr_mapping_from_id->($map, $id)) {
 237         $res = ",bus=$busname.$d->{bus},addr=$d->{addr}";
 238         $bridges->{$d->{bus}} = 1 if $bridges;
 239     }
 240
 241     return $res;
 242 }
 243
 244 my $pcie_addr_map;
 245 sub get_pcie_addr_map {
 246     $pcie_addr_map = {
 247         vga => { bus => 'pcie.0', addr => 1 },
 248         hostpci0 => { bus => "ich9-pcie-port-1", addr => 0 },
 249         hostpci1 => { bus => "ich9-pcie-port-2", addr => 0 },
 250         hostpci2 => { bus => "ich9-pcie-port-3", addr => 0 },
 251         hostpci3 => { bus => "ich9-pcie-port-4", addr => 0 },
 252         hostpci4 => { bus => "ich9-pcie-port-5", addr => 0 },
 253         hostpci5 => { bus => "ich9-pcie-port-6", addr => 0 },
 254         hostpci6 => { bus => "ich9-pcie-port-7", addr => 0 },
 255         hostpci7 => { bus => "ich9-pcie-port-8", addr => 0 },
 256         hostpci8 => { bus => "ich9-pcie-port-9", addr => 0 },
 257         hostpci9 => { bus => "ich9-pcie-port-10", addr => 0 },
 258         hostpci10 => { bus => "ich9-pcie-port-11", addr => 0 },
 259         hostpci11 => { bus => "ich9-pcie-port-12", addr => 0 },
 260         hostpci12 => { bus => "ich9-pcie-port-13", addr => 0 },
 261         hostpci13 => { bus => "ich9-pcie-port-14", addr => 0 },
 262         hostpci14 => { bus => "ich9-pcie-port-15", addr => 0 },
 263         hostpci15 => { bus => "ich9-pcie-port-16", addr => 0 },
 264         # win7 is picky about pcie assignments
 265         hostpci0bus0 => { bus => "pcie.0", addr => 16 },
 266         hostpci1bus0 => { bus => "pcie.0", addr => 17 },
 267         hostpci2bus0 => { bus => "pcie.0", addr => 18 },
 268         hostpci3bus0 => { bus => "pcie.0", addr => 19 },
 269         ivshmem => { bus => 'pcie.0', addr => 20 },
 270         hostpci4bus0 => { bus => "pcie.0", addr => 9 },
 271         hostpci5bus0 => { bus => "pcie.0", addr => 10 },
 272         hostpci6bus0 => { bus => "pcie.0", addr => 11 },
 273         hostpci7bus0 => { bus => "pcie.0", addr => 12 },
 274         hostpci8bus0 => { bus => "pcie.0", addr => 13 },
 275         hostpci9bus0 => { bus => "pcie.0", addr => 14 },
 276         hostpci10bus0 => { bus => "pcie.0", addr => 15 },
 277         hostpci11bus0 => { bus => "pcie.0", addr => 21 },
 278         hostpci12bus0 => { bus => "pcie.0", addr => 22 },
 279         hostpci13bus0 => { bus => "pcie.0", addr => 23 },
 280         hostpci14bus0 => { bus => "pcie.0", addr => 24 },
 281         hostpci15bus0 => { bus => "pcie.0", addr => 25 },
 282     } if !defined($pcie_addr_map);
 283
 284     return $pcie_addr_map;
 285 }
 286
 287 sub print_pcie_addr {
 288     my ($id) = @_;
 289
 290     my $res = '';
 291
 292     my $map = get_pcie_addr_map($id);
 293     if (my $d = $get_addr_mapping_from_id->($map, $id)) {
 294         $res = ",bus=$d->{bus},addr=$d->{addr}";
 295     }
 296
 297     return $res;
 298 }
 299
 300 # Generates the device strings for additional pcie root ports. The first 4 pcie
 301 # root ports are defined in the pve-q35*.cfg files.
 302 sub print_pcie_root_port {
 303     my ($i) = @_;
 304     my $res = '';
 305
 306     my $root_port_addresses = {
 307          4 => "10.0",
 308          5 => "10.1",
 309          6 => "10.2",
 310          7 => "10.3",
 311          8 => "10.4",
 312          9 => "10.5",
 313         10 => "10.6",
 314         11 => "10.7",
 315         12 => "11.0",
 316         13 => "11.1",
 317         14 => "11.2",
 318         15 => "11.3",
 319     };
 320
 321     if (defined($root_port_addresses->{$i})) {
 322         my $id = $i + 1;
 323         $res = "pcie-root-port,id=ich9-pcie-port-${id}";
 324         $res .= ",addr=$root_port_addresses->{$i}";
 325         $res .= ",x-speed=16,x-width=32,multifunction=on,bus=pcie.0";
 326         $res .= ",port=${id},chassis=${id}";
 327     }
 328
 329     return $res;
 330 }
 331
 332 sub parse_hostpci {
 333     my ($value) = @_;
 334
 335     return undef if !$value;
 336
 337     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
 338
 339     my @idlist = split(/;/, $res->{host});
 340     delete $res->{host};
 341     foreach my $id (@idlist) {
 342         my $devs = PVE::SysFSTools::lspci($id);
 343         die "no PCI device found for '$id'\n" if !scalar(@$devs);
 344         push @{$res->{pciid}}, @$devs;
 345     }
 346     return $res;
 347 }
 348
 349 sub print_hostpci_devices {
 350     my ($conf, $devices, $winversion, $q35, $bridges, $arch, $machine_type) = @_;
 351
 352     my $kvm_off = 0;
 353     my $gpu_passthrough = 0;
 354
 355     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
 356         my $id = "hostpci$i";
 357         my $d = parse_hostpci($conf->{$id});
 358         next if !$d;
 359
 360         if (my $pcie = $d->{pcie}) {
 361             die "q35 machine model is not enabled" if !$q35;
 362             # win7 wants to have the pcie devices directly on the pcie bus
 363             # instead of in the root port
 364             if ($winversion == 7) {
 365                 $pciaddr = print_pcie_addr("${id}bus0");
 366             } else {
 367                 # add more root ports if needed, 4 are present by default
 368                 # by pve-q35 cfgs, rest added here on demand.
 369                 if ($i > 3) {
 370                     push @$devices, '-device', print_pcie_root_port($i);
 371                 }
 372                 $pciaddr = print_pcie_addr($id);
 373             }
 374         } else {
 375             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
 376         }
 377
 378         my $xvga = '';
 379         if ($d->{'x-vga'}) {
 380             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
 381             $kvm_off = 1;
 382             $vga->{type} = 'none' if !defined($conf->{vga});
 383             $gpu_passthrough = 1;
 384         }
 385
 386         my $pcidevices = $d->{pciid};
 387         my $multifunction = 1 if @$pcidevices > 1;
 388
 389         my $sysfspath;
 390         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
 391             my $pci_id = $pcidevices->[0]->{id};
 392             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
 393             $sysfspath = "/sys/bus/pci/devices/$pci_id/$uuid";
 394         } elsif ($d->{mdev}) {
 395             warn "ignoring mediated device '$id' with multifunction device\n";
 396         }
 397
 398         my $j=0;
 399         foreach my $pcidevice (@$pcidevices) {
 400             my $devicestr = "vfio-pci";
 401
 402             if ($sysfspath) {
 403                 $devicestr .= ",sysfsdev=$sysfspath";
 404             } else {
 405                 $devicestr .= ",host=$pcidevice->{id}";
 406             }
 407
 408             my $mf_addr = $multifunction ? ".$j" : '';
 409             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
 410
 411             if ($j == 0) {
 412                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
 413                 $devicestr .= "$xvga";
 414                 $devicestr .= ",multifunction=on" if $multifunction;
 415                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
 416             }
 417
 418             push @$devices, '-device', $devicestr;
 419             $j++;
 420         }
 421     }
 422
 423     return ($kvm_off, $gpu_passthrough);
 424 }
 425
 426 1;