src/PVE/SysFSTools.pm

   1 package PVE::SysFSTools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use IO::File;
   7
   8 use PVE::Tools qw(file_read_firstline dir_glob_foreach);
   9
  10 my $pcisysfs = "/sys/bus/pci";
  11 my $domainregex = "[a-f0-9]{4,}";
  12 my $pciregex = "($domainregex):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])";
  13
  14 my $parse_pci_ids = sub {
  15     my $ids = {};
  16
  17     open(my $fh, '<', "/usr/share/misc/pci.ids")
  18         or return $ids;
  19
  20     my $curvendor;
  21     my $curdevice;
  22     while (my $line = <$fh>) {
  23         if ($line =~ m/^([0-9a-fA-F]{4})\s+(.*)$/) {
  24             $curvendor = ($ids->{"0x$1"} = {});
  25             $curvendor->{name} = $2;
  26         } elsif ($line =~ m/^\t([0-9a-fA-F]{4})\s+(.*)$/) {
  27             $curdevice = ($curvendor->{devices}->{"0x$1"} = {});
  28             $curdevice->{name} = $2;
  29         } elsif ($line =~ m/^\t\t([0-9a-fA-F]{4}) ([0-9a-fA-F]{4})\s+(.*)$/) {
  30             $curdevice->{subs}->{"0x$1"}->{"0x$2"} = $3;
  31         }
  32     }
  33
  34     return $ids;
  35 };
  36
  37 my $fixup_missing_domain = sub {
  38     my ($id) = @_;
  39
  40     $id = "0000:$id" if $id !~ m/^${domainregex}:/;
  41
  42     return $id;
  43 };
  44
  45 # returns a list of pci devices
  46 #
  47 # filter is either a string (then it tries to match to the id)
  48 # or a sub ref (then it adds the device if the sub returns truthy)
  49 #
  50 # verbose also returns iommu groups, subvendor/device and the
  51 # human readable names from /usr/share/misc/pci.ids
  52 #
  53 # return format:
  54 # [
  55 #     {
  56 #         id => '00:00.0',
  57 #         vendor => '0xabab',
  58 #         device => '0xefef',
  59 #         class => '0x012345',
  60 #
  61 #         # optional
  62 #         iommugroup => '14',
  63 #         mdev => 1,
  64 #         vendor_name => 'Foo Inc.',
  65 #         device_name => 'Bar 9000AF',
  66 #         subsystem_vendor => '0xacac',
  67 #         subsystem_device => '0xfefe',
  68 #         subsystem_vendor_name => 'Foo Europe GmbH',
  69 #         subsystem_device_name => 'Bar 9001AF OC',
  70 #     },
  71 #     ...
  72 # ]
  73 #
  74 sub lspci {
  75     my ($filter, $verbose) = @_;
  76
  77     my $devices = [];
  78     my $ids = {};
  79     if ($verbose) {
  80         $ids = $parse_pci_ids->();
  81     }
  82
  83     dir_glob_foreach("$pcisysfs/devices", $pciregex, sub {
  84         my ($fullid, $domain, $bus, $slot, $function) = @_;
  85         my $id = "$domain:$bus:$slot.$function";
  86
  87         if (defined($filter) && !ref($filter) && $id !~ m/^(0000:)?\Q$filter\E/) {
  88             return; # filter ids early
  89         }
  90
  91         my $devdir = "$pcisysfs/devices/$fullid";
  92
  93         my $vendor = file_read_firstline("$devdir/vendor");
  94         my $device = file_read_firstline("$devdir/device");
  95         my $class = file_read_firstline("$devdir/class");
  96
  97         my $res = {
  98             id => $id,
  99             vendor => $vendor,
 100             device => $device,
 101             class => $class,
 102         };
 103
 104         if (defined($filter) && ref($filter) eq 'CODE' && !$filter->($res)) {
 105             return;
 106         }
 107
 108         if ($verbose) {
 109             $res->{iommugroup} = -1;
 110             if (-e "$devdir/iommu_group") {
 111                 my ($iommugroup) = (readlink("$devdir/iommu_group") =~ m/\/(\d+)$/);
 112                 $res->{iommugroup} = int($iommugroup);
 113             }
 114
 115             if (-d "$devdir/mdev_supported_types") {
 116                 $res->{mdev} = 1;
 117             }
 118
 119             my $device_hash = $ids->{$vendor}->{devices}->{$device} // {};
 120
 121             my $sub_vendor = file_read_firstline("$devdir/subsystem_vendor");
 122             my $sub_device = file_read_firstline("$devdir/subsystem_device");
 123
 124             my $vendor_name = $ids->{$vendor}->{name};
 125             my $device_name = $device_hash->{name};
 126             my $sub_vendor_name = $ids->{$sub_vendor}->{name};
 127             my $sub_device_name = $device_hash->{subs}->{$sub_vendor}->{$sub_device};
 128
 129             $res->{vendor_name} = $vendor_name if defined($vendor_name);
 130             $res->{device_name} = $device_name if defined($device_name);
 131             $res->{subsystem_vendor} = $sub_vendor if defined($sub_vendor);
 132             $res->{subsystem_device} = $sub_device if defined($sub_device);
 133             $res->{subsystem_vendor_name} = $sub_vendor_name if defined($sub_vendor_name);
 134             $res->{subsystem_device_name} = $sub_device_name if defined($sub_device_name);
 135         }
 136
 137         push @$devices, $res;
 138     });
 139
 140     # Entries should be sorted by ids
 141     $devices = [ sort { $a->{id} cmp $b->{id} } @$devices ];
 142
 143     return $devices;
 144 }
 145
 146 #
 147 # return format:
 148 # [
 149 #     {
 150 #         type => 'FooType_1',
 151 #         description => "a longer description with custom format\nand newlines",
 152 #         available => 5,
 153 #     },
 154 #     ...
 155 # ]
 156 #
 157 sub get_mdev_types {
 158     my ($id) = @_;
 159
 160     $id = $fixup_missing_domain->($id);
 161
 162     my $types = [];
 163
 164     my $mdev_path = "$pcisysfs/devices/$id/mdev_supported_types";
 165     if (!-d $mdev_path) {
 166         return $types;
 167     }
 168
 169     dir_glob_foreach($mdev_path, '[^\.].*', sub {
 170         my ($type) = @_;
 171
 172         my $type_path = "$mdev_path/$type";
 173
 174         my $available = int(file_read_firstline("$type_path/available_instances"));
 175         my $description = PVE::Tools::file_get_contents("$type_path/description");
 176
 177         push @$types, {
 178             type => $type,
 179             description => $description,
 180             available => $available,
 181         };
 182     });
 183
 184     return $types;
 185 }
 186
 187 sub check_iommu_support{
 188     # we have IOMMU support if /sys/class/iommu/ is populated
 189     return PVE::Tools::dir_glob_regex('/sys/class/iommu/', "[^\.].*");
 190 }
 191
 192 sub file_write {
 193     my ($filename, $buf) = @_;
 194
 195     my $fh = IO::File->new($filename, "w");
 196     return undef if !$fh;
 197
 198     my $res = print $fh $buf;
 199
 200     $fh->close();
 201
 202     return $res;
 203 }
 204
 205 sub pci_device_info {
 206     my ($name, $verbose) = @_;
 207
 208     my $res;
 209
 210     return undef if $name !~ m/^${pciregex}$/;
 211     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
 212
 213     my $devdir = "$pcisysfs/devices/$name";
 214
 215     my $irq = file_read_firstline("$devdir/irq");
 216     return undef if !defined($irq) || $irq !~ m/^\d+$/;
 217
 218     my $vendor = file_read_firstline("$devdir/vendor");
 219     return undef if !defined($vendor) || $vendor !~ s/^0x//;
 220
 221     my $product = file_read_firstline("$devdir/device");
 222     return undef if !defined($product) || $product !~ s/^0x//;
 223
 224     $res = {
 225         name => $name,
 226         vendor => $vendor,
 227         device => $product,
 228         domain => $domain,
 229         bus => $bus,
 230         slot => $slot,
 231         func => $func,
 232         irq => $irq,
 233         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
 234     };
 235
 236     if ($verbose) {
 237         my $sub_vendor = file_read_firstline("$devdir/subsystem_vendor");
 238         $sub_vendor =~ s/^0x// if defined($sub_vendor);
 239         my $sub_device = file_read_firstline("$devdir/subsystem_device");
 240         $sub_device =~ s/^0x// if defined($sub_device);
 241
 242         $res->{subsystem_vendor} = $sub_vendor if defined($sub_vendor);
 243         $res->{subsystem_device} = $sub_device if defined($sub_device);
 244
 245         if (-e "$devdir/iommu_group") {
 246             my ($iommugroup) = (readlink("$devdir/iommu_group") =~ m/\/(\d+)$/);
 247             $res->{iommugroup} = int($iommugroup);
 248         }
 249
 250         if (-d "$devdir/mdev_supported_types") {
 251             $res->{mdev} = 1;
 252         }
 253     }
 254
 255     return $res;
 256 }
 257
 258 sub pci_dev_reset {
 259     my ($dev) = @_;
 260
 261     my $name = $dev->{name};
 262
 263     my $fn = "$pcisysfs/devices/$name/reset";
 264
 265     return file_write($fn, "1");
 266 }
 267
 268 sub pci_dev_bind_to_vfio {
 269     my ($dev) = @_;
 270
 271     my $name = $dev->{name};
 272
 273     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
 274
 275     if (!-d $vfio_basedir) {
 276         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
 277     }
 278     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
 279
 280     my $testdir = "$vfio_basedir/$name";
 281     return 1 if -d $testdir;
 282
 283     my $data = "$dev->{vendor} $dev->{device}";
 284     return undef if !file_write("$vfio_basedir/new_id", $data);
 285
 286     my $fn = "$pcisysfs/devices/$name/driver/unbind";
 287     if (!file_write($fn, $name)) {
 288         return undef if -f $fn;
 289     }
 290
 291     $fn = "$vfio_basedir/bind";
 292     if (! -d $testdir) {
 293         return undef if !file_write($fn, $name);
 294     }
 295
 296     return -d $testdir;
 297 }
 298
 299 sub pci_dev_group_bind_to_vfio {
 300     my ($pciid) = @_;
 301
 302     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
 303
 304     if (!-d $vfio_basedir) {
 305         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
 306     }
 307     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
 308
 309     $pciid = $fixup_missing_domain->($pciid);
 310
 311     # get IOMMU group devices
 312     opendir(my $D, "$pcisysfs/devices/$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
 313     my @devs = grep /^${domainregex}:/, readdir($D);
 314     closedir($D);
 315
 316     foreach my $pciid (@devs) {
 317         $pciid =~ m/^([:\.0-9a-f]+)$/ or die "PCI ID $pciid not valid!\n";
 318
 319         # PCI bridges, switches or root-ports aren't supported and all have a pci_bus dir we can test
 320         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
 321
 322         my $info = pci_device_info($1);
 323         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
 324     }
 325
 326     return 1;
 327 }
 328
 329 sub pci_create_mdev_device {
 330     my ($pciid, $uuid, $type) = @_;
 331
 332     $pciid = $fixup_missing_domain->($pciid);
 333
 334     my $basedir = "$pcisysfs/devices/$pciid";
 335     my $mdev_dir = "$basedir/mdev_supported_types";
 336
 337     die "pci device '$pciid' does not support mediated devices \n"
 338         if !-d $mdev_dir;
 339
 340     die "pci device '$pciid' has no type '$type'\n"
 341         if !-d "$mdev_dir/$type";
 342
 343     if (-d "$basedir/$uuid") {
 344         # it already exists, checking type
 345         my $typelink = readlink("$basedir/$uuid/mdev_type");
 346         my ($existingtype) = $typelink =~ m|/([^/]+)$|;
 347         die "mdev instance '$uuid' already exits, but type is not '$type'\n"
 348             if $type ne $existingtype;
 349
 350         # instance exists, so use it but warn the user
 351         warn "mdev instance '$uuid' already existed, using it.\n";
 352         return undef;
 353     }
 354
 355     my $instances = file_read_firstline("$mdev_dir/$type/available_instances");
 356     my ($avail) = $instances =~ m/^(\d+)$/;
 357     die "pci device '$pciid' has no available instances of '$type'\n"
 358         if $avail < 1;
 359
 360     die "could not create 'type' for pci devices '$pciid'\n"
 361         if !file_write("$mdev_dir/$type/create", $uuid);
 362
 363     return undef;
 364 }
 365
 366 sub pci_cleanup_mdev_device {
 367     my ($pciid, $uuid) = @_;
 368
 369     $pciid = $fixup_missing_domain->($pciid);
 370
 371     my $basedir = "$pcisysfs/devices/$pciid/$uuid";
 372
 373     if (! -e $basedir) {
 374         return 1; # no cleanup necessary if it does not exist
 375     }
 376
 377     return file_write("$basedir/remove", "1");
 378 }
 379
 380 # encode the hostpci index and vmid into the uuid
 381 sub generate_mdev_uuid {
 382     my ($vmid, $index) = @_;
 383
 384     my $string = sprintf("%08d-0000-0000-0000-%012d", $index, $vmid);
 385
 386     return $string;
 387 }
 388
 389 # idea is from usbutils package (/usr/bin/usb-devices) script
 390 sub __scan_usb_device {
 391     my ($res, $devpath, $parent, $level) = @_;
 392
 393     return if ! -d $devpath;
 394     return if $level && $devpath !~ m/^.*[-.](\d+)$/;
 395     my $port = $level ? int($1 - 1) : 0;
 396
 397     my $busnum = int(file_read_firstline("$devpath/busnum"));
 398     my $devnum = int(file_read_firstline("$devpath/devnum"));
 399
 400     my $d = {
 401         port => $port,
 402         level => $level,
 403         busnum => $busnum,
 404         devnum => $devnum,
 405         speed => file_read_firstline("$devpath/speed"),
 406         class => hex(file_read_firstline("$devpath/bDeviceClass")),
 407         vendid => file_read_firstline("$devpath/idVendor"),
 408         prodid => file_read_firstline("$devpath/idProduct"),
 409     };
 410
 411     if ($level) {
 412         my $usbpath = $devpath;
 413         $usbpath =~ s|^.*/\d+\-||;
 414         $d->{usbpath} = $usbpath;
 415     }
 416
 417     my $product = file_read_firstline("$devpath/product");
 418     $d->{product} = $product if $product;
 419
 420     my $manu = file_read_firstline("$devpath/manufacturer");
 421     $d->{manufacturer} = $manu if $manu;
 422
 423     my $serial => file_read_firstline("$devpath/serial");
 424     $d->{serial} = $serial if $serial;
 425
 426     push @$res, $d;
 427
 428     foreach my $subdev (<$devpath/$busnum-*>) {
 429         next if $subdev !~ m|/$busnum-[0-9]+(\.[0-9]+)*$|;
 430         __scan_usb_device($res, $subdev, $devnum, $level + 1);
 431     }
 432
 433 };
 434
 435 sub scan_usb {
 436
 437     my $devlist = [];
 438
 439     foreach my $device (</sys/bus/usb/devices/usb*>) {
 440         __scan_usb_device($devlist, $device, 0, 0);
 441     }
 442
 443     return $devlist;
 444 }
 445
 446 1;