src/PVE/SysFSTools.pm

   1 package PVE::SysFSTools;
   2
   3 use strict;
   4 use warnings;
   5
   6 use IO::File;
   7
   8 use PVE::Tools qw(file_read_firstline dir_glob_foreach);
   9
  10 my $pcisysfs = "/sys/bus/pci";
  11 my $pciregex = "([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])";
  12
  13 my $parse_pci_ids = sub {
  14     my $ids = {};
  15
  16     open(my $fh, '<', "/usr/share/misc/pci.ids")
  17         or return $ids;
  18
  19     my $curvendor;
  20     my $curdevice;
  21     while (my $line = <$fh>) {
  22         if ($line =~ m/^([0-9a-fA-F]{4})\s+(.*)$/) {
  23             $curvendor = ($ids->{"0x$1"} = {});
  24             $curvendor->{name} = $2;
  25         } elsif ($line =~ m/^\t([0-9a-fA-F]{4})\s+(.*)$/) {
  26             $curdevice = ($curvendor->{devices}->{"0x$1"} = {});
  27             $curdevice->{name} = $2;
  28         } elsif ($line =~ m/^\t\t([0-9a-fA-F]{4}) ([0-9a-fA-F]{4})\s+(.*)$/) {
  29             $curdevice->{subs}->{"0x$1"}->{"0x$2"} = $3;
  30         }
  31     }
  32
  33     return $ids;
  34 };
  35
  36 # returns a list of pci devices
  37 #
  38 # filter is either a string (then it tries to match to the id)
  39 # or a sub ref (then it adds the device if the sub returns truthy)
  40 #
  41 # verbose also returns iommu groups, subvendor/device and the
  42 # human readable names from /usr/share/misc/pci.ids
  43 #
  44 # return format:
  45 # [
  46 #     {
  47 #         id => '00:00.0',
  48 #         vendor => '0xabab',
  49 #         device => '0xefef',
  50 #         class => '0x012345',
  51 #
  52 #         # optional
  53 #         iommugroup => '14',
  54 #         mdev => 1,
  55 #         vendor_name => 'Foo Inc.',
  56 #         device_name => 'Bar 9000AF',
  57 #         subsystem_vendor => '0xacac',
  58 #         subsystem_device => '0xfefe',
  59 #         subsystem_vendor_name => 'Foo Europe GmbH',
  60 #         subsystem_device_name => 'Bar 9001AF OC',
  61 #     },
  62 #     ...
  63 # ]
  64 #
  65 sub lspci {
  66     my ($filter, $verbose) = @_;
  67
  68     my $devices = [];
  69     my $ids = {};
  70     if ($verbose) {
  71         $ids = $parse_pci_ids->();
  72     }
  73
  74     dir_glob_foreach("$pcisysfs/devices", $pciregex, sub {
  75         my ($fullid, $domain, $bus, $slot, $function) = @_;
  76         my $id = "$bus:$slot.$function";
  77
  78         if (defined($filter) && !ref($filter) && $id !~ m/^\Q$filter\E/) {
  79             return; # filter ids early
  80         }
  81
  82         my $devdir = "$pcisysfs/devices/$fullid";
  83
  84         my $vendor = file_read_firstline("$devdir/vendor");
  85         my $device = file_read_firstline("$devdir/device");
  86         my $class = file_read_firstline("$devdir/class");
  87
  88         my $res = {
  89             id => $id,
  90             vendor => $vendor,
  91             device => $device,
  92             class => $class,
  93         };
  94
  95         if (defined($filter) && ref($filter) eq 'CODE' && !$filter->($res)) {
  96             return;
  97         }
  98
  99         if ($verbose) {
 100             $res->{iommugroup} = -1;
 101             if (-e "$devdir/iommu_group") {
 102                 my ($iommugroup) = (readlink("$devdir/iommu_group") =~ m/\/(\d+)$/);
 103                 $res->{iommugroup} = int($iommugroup);
 104             }
 105
 106             if (-d "$devdir/mdev_supported_types") {
 107                 $res->{mdev} = 1;
 108             }
 109
 110             my $device_hash = $ids->{$vendor}->{devices}->{$device} // {};
 111
 112             my $sub_vendor = file_read_firstline("$devdir/subsystem_vendor");
 113             my $sub_device = file_read_firstline("$devdir/subsystem_device");
 114
 115             my $vendor_name = $ids->{$vendor}->{name};
 116             my $device_name = $device_hash->{name};
 117             my $sub_vendor_name = $ids->{$sub_vendor}->{name};
 118             my $sub_device_name = $device_hash->{subs}->{$sub_vendor}->{$sub_device};
 119
 120             $res->{vendor_name} = $vendor_name if defined($vendor_name);
 121             $res->{device_name} = $device_name if defined($device_name);
 122             $res->{subsystem_vendor} = $sub_vendor if defined($sub_vendor);
 123             $res->{subsystem_device} = $sub_device if defined($sub_device);
 124             $res->{subsystem_vendor_name} = $sub_vendor_name if defined($sub_vendor_name);
 125             $res->{subsystem_device_name} = $sub_device_name if defined($sub_device_name);
 126         }
 127
 128         push @$devices, $res;
 129     });
 130
 131     # Entries should be sorted by ids
 132     $devices = [ sort { $a->{id} cmp $b->{id} } @$devices ];
 133
 134     return $devices;
 135 }
 136
 137 #
 138 # return format:
 139 # [
 140 #     {
 141 #         type => 'FooType_1',
 142 #         description => "a longer description with custom format\nand newlines",
 143 #         available => 5,
 144 #     },
 145 #     ...
 146 # ]
 147 #
 148 sub get_mdev_types {
 149     my ($id) = @_;
 150
 151     my $fullid = $id;
 152     if ($id !~ m/^[0-9a-fA-f]{4}:/) {
 153         $fullid = "0000:$id";
 154     }
 155
 156     my $types = [];
 157
 158     my $mdev_path = "$pcisysfs/devices/$fullid/mdev_supported_types";
 159     if (!-d $mdev_path) {
 160         return $types;
 161     }
 162
 163     dir_glob_foreach($mdev_path, '[^\.].*', sub {
 164         my ($type) = @_;
 165
 166         my $type_path = "$mdev_path/$type";
 167
 168         my $available = int(file_read_firstline("$type_path/available_instances"));
 169         my $description = PVE::Tools::file_get_contents("$type_path/description");
 170
 171         push @$types, {
 172             type => $type,
 173             description => $description,
 174             available => $available,
 175         };
 176     });
 177
 178     return $types;
 179 }
 180
 181 sub check_iommu_support{
 182     # we have IOMMU support if /sys/class/iommu/ is populated
 183     return PVE::Tools::dir_glob_regex('/sys/class/iommu/', "[^\.].*");
 184 }
 185
 186 sub file_write {
 187     my ($filename, $buf) = @_;
 188
 189     my $fh = IO::File->new($filename, "w");
 190     return undef if !$fh;
 191
 192     my $res = print $fh $buf;
 193
 194     $fh->close();
 195
 196     return $res;
 197 }
 198
 199 sub pci_device_info {
 200     my ($name) = @_;
 201
 202     my $res;
 203
 204     return undef if $name !~ m/^${pciregex}$/;
 205     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
 206
 207     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
 208     return undef if !defined($irq) || $irq !~ m/^\d+$/;
 209
 210     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
 211     return undef if !defined($vendor) || $vendor !~ s/^0x//;
 212
 213     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
 214     return undef if !defined($product) || $product !~ s/^0x//;
 215
 216     $res = {
 217         name => $name,
 218         vendor => $vendor,
 219         product => $product,
 220         domain => $domain,
 221         bus => $bus,
 222         slot => $slot,
 223         func => $func,
 224         irq => $irq,
 225         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
 226     };
 227
 228     return $res;
 229 }
 230
 231 sub pci_dev_reset {
 232     my ($dev) = @_;
 233
 234     my $name = $dev->{name};
 235
 236     my $fn = "$pcisysfs/devices/$name/reset";
 237
 238     return file_write($fn, "1");
 239 }
 240
 241 sub pci_dev_bind_to_vfio {
 242     my ($dev) = @_;
 243
 244     my $name = $dev->{name};
 245
 246     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
 247
 248     if (!-d $vfio_basedir) {
 249         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
 250     }
 251     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
 252
 253     my $testdir = "$vfio_basedir/$name";
 254     return 1 if -d $testdir;
 255
 256     my $data = "$dev->{vendor} $dev->{product}";
 257     return undef if !file_write("$vfio_basedir/new_id", $data);
 258
 259     my $fn = "$pcisysfs/devices/$name/driver/unbind";
 260     if (!file_write($fn, $name)) {
 261         return undef if -f $fn;
 262     }
 263
 264     $fn = "$vfio_basedir/bind";
 265     if (! -d $testdir) {
 266         return undef if !file_write($fn, $name);
 267     }
 268
 269     return -d $testdir;
 270 }
 271
 272 sub pci_dev_group_bind_to_vfio {
 273     my ($pciid) = @_;
 274
 275     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
 276
 277     if (!-d $vfio_basedir) {
 278         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
 279     }
 280     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
 281
 282     # get IOMMU group devices
 283     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
 284       my @devs = grep /^0000:/, readdir($D);
 285     closedir($D);
 286
 287     foreach my $pciid (@devs) {
 288         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
 289
 290         # pci bridges, switches or root ports are not supported
 291         # they have a pci_bus subdirectory so skip them
 292         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
 293
 294         my $info = pci_device_info($1);
 295         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
 296     }
 297
 298     return 1;
 299 }
 300
 301 sub pci_create_mdev_device {
 302     my ($pciid, $uuid, $type) = @_;
 303
 304     my $basedir = "$pcisysfs/devices/0000:$pciid";
 305     my $mdev_dir = "$basedir/mdev_supported_types";
 306
 307     die "pci device '$pciid' does not support mediated devices \n"
 308         if !-d $mdev_dir;
 309
 310     die "pci device '$pciid' has no type '$type'\n"
 311         if !-d "$mdev_dir/$type";
 312
 313     if (-d "$basedir/$uuid") {
 314         # it already exists, checking type
 315         my $typelink = readlink("$basedir/$uuid/mdev_type");
 316         my ($existingtype) = $typelink =~ m|/([^/]+)$|;
 317         die "mdev instance '$uuid' already exits, but type is not '$type'\n"
 318             if $type ne $existingtype;
 319
 320         # instance exists, so use it but warn the user
 321         warn "mdev instance '$uuid' already existed, using it.\n";
 322         return undef;
 323     }
 324
 325     my $instances = file_read_firstline("$mdev_dir/$type/available_instances");
 326     my ($avail) = $instances =~ m/^(\d+)$/;
 327     die "pci device '$pciid' has no available instances of '$type'\n"
 328         if $avail < 1;
 329
 330     die "could not create 'type' for pci devices '$pciid'\n"
 331         if !file_write("$mdev_dir/$type/create", $uuid);
 332
 333     return undef;
 334 }
 335
 336 sub pci_cleanup_mdev_device {
 337     my ($pciid, $uuid) = @_;
 338
 339     my $basedir = "$pcisysfs/devices/0000:$pciid/$uuid";
 340
 341     if (! -e $basedir) {
 342         return 1; # no cleanup necessary if it does not exist
 343     }
 344
 345     return file_write("$basedir/remove", "1");
 346 }
 347
 348 # encode the hostpci index and vmid into the uuid
 349 sub generate_mdev_uuid {
 350     my ($vmid, $index) = @_;
 351
 352     my $string = sprintf("%08d-0000-0000-0000-%012d", $index, $vmid);
 353
 354     return $string;
 355 }
 356
 357 # idea is from usbutils package (/usr/bin/usb-devices) script
 358 sub __scan_usb_device {
 359     my ($res, $devpath, $parent, $level) = @_;
 360
 361     return if ! -d $devpath;
 362     return if $level && $devpath !~ m/^.*[-.](\d+)$/;
 363     my $port = $level ? int($1 - 1) : 0;
 364
 365     my $busnum = int(file_read_firstline("$devpath/busnum"));
 366     my $devnum = int(file_read_firstline("$devpath/devnum"));
 367
 368     my $d = {
 369         port => $port,
 370         level => $level,
 371         busnum => $busnum,
 372         devnum => $devnum,
 373         speed => file_read_firstline("$devpath/speed"),
 374         class => hex(file_read_firstline("$devpath/bDeviceClass")),
 375         vendid => file_read_firstline("$devpath/idVendor"),
 376         prodid => file_read_firstline("$devpath/idProduct"),
 377     };
 378
 379     if ($level) {
 380         my $usbpath = $devpath;
 381         $usbpath =~ s|^.*/\d+\-||;
 382         $d->{usbpath} = $usbpath;
 383     }
 384
 385     my $product = file_read_firstline("$devpath/product");
 386     $d->{product} = $product if $product;
 387
 388     my $manu = file_read_firstline("$devpath/manufacturer");
 389     $d->{manufacturer} = $manu if $manu;
 390
 391     my $serial => file_read_firstline("$devpath/serial");
 392     $d->{serial} = $serial if $serial;
 393
 394     push @$res, $d;
 395
 396     foreach my $subdev (<$devpath/$busnum-*>) {
 397         next if $subdev !~ m|/$busnum-[0-9]+(\.[0-9]+)*$|;
 398         __scan_usb_device($res, $subdev, $devnum, $level + 1);
 399     }
 400
 401 };
 402
 403 sub scan_usb {
 404
 405     my $devlist = [];
 406
 407     foreach my $device (</sys/bus/usb/devices/usb*>) {
 408         __scan_usb_device($devlist, $device, 0, 0);
 409     }
 410
 411     return $devlist;
 412 }
 413
 414 1;