DAB.pm

   1 package PVE::DAB;
   2
   3 use strict;
   4 use warnings;
   5 use IO::File;
   6 use File::Path;
   7 use File::Basename;
   8 use IO::Select;
   9 use IPC::Open2;
  10 use IPC::Open3;
  11 use POSIX qw (LONG_MAX);
  12 use UUID;
  13 use Cwd;
  14
  15 # fixme: lock container ?
  16
  17 my $dablibdir = "/usr/lib/dab";
  18 my $devicetar = "$dablibdir/devices.tar.gz";
  19 my $default_env = "$dablibdir/scripts/defenv";
  20 my $fake_init = "$dablibdir/scripts/init.pl";
  21 my $script_ssh_init = "$dablibdir/scripts/ssh_gen_host_keys";
  22 my $script_mysql_randompw = "$dablibdir/scripts/mysql_randompw";
  23 my $script_init_urandom = "$dablibdir/scripts/init_urandom";
  24
  25 my $postfix_main_cf = <<EOD;
  26 # See /usr/share/postfix/main.cf.dist for a commented, more complete version
  27
  28 smtpd_banner = \$myhostname ESMTP \$mail_name (Debian/GNU)
  29 biff = no
  30
  31 # appending .domain is the MUA's job.
  32 append_dot_mydomain = no
  33
  34 # Uncomment the next line to generate "delayed mail" warnings
  35 #delay_warning_time = 4h
  36
  37 alias_maps = hash:/etc/aliases
  38 alias_database = hash:/etc/aliases
  39 mydestination = \$myhostname, localhost.\$mydomain, localhost
  40 relayhost =
  41 mynetworks = 127.0.0.0/8
  42 inet_interfaces = loopback-only
  43 recipient_delimiter = +
  44
  45 compatibility_level = 2
  46
  47 EOD
  48
  49 # produce apt compatible filenames (/var/lib/apt/lists)
  50 sub __url_to_filename {
  51     my $url = shift;
  52
  53     $url =~ s|^\S+://||;
  54     $url =~ s|_|%5f|g;
  55     $url =~ s|/|_|g;
  56
  57     return $url;
  58 }
  59
  60 # defaults:
  61 #  origin: debian
  62 my $supported_suites = {
  63     'bookworm' => {
  64         ostype => "debian-12",
  65     },
  66     'bullseye' => {
  67         ostype => "debian-11",
  68     },
  69     'buster' => {
  70         ostype => "debian-10",
  71     },
  72     'stretch' => {
  73         ostype => "debian-9.0",
  74     },
  75     'jessie' => {
  76         ostype => "debian-8.0",
  77     },
  78     'wheezy' => {
  79         ostype => "debian-7.0",
  80     },
  81     'squeeze' => {
  82         ostype => "debian-6.0",
  83     },
  84     'lenny' => {
  85         ostype => "debian-5.0",
  86     },
  87     'etch' => {
  88         ostype => "debian-4.0",
  89     },
  90
  91 # DEVUAN
  92     'devuan-jessie' => {
  93         suite => 'jessie',
  94         ostype => "devuan-1.0",
  95     },
  96     'devuan-ascii' => {
  97         suite => 'ascii',
  98         ostype => "devuan-2.0",
  99     },
 100     'ascii' => {
 101         ostype => "devuan-2.0",
 102     },
 103     'beowulf' => {
 104         ostype => "devuan-3.0",
 105     },
 106     'chimaera' => {
 107         ostype => "devuan-4.0",
 108     },
 109
 110 # UBUNTU
 111     'hardy' => {
 112         ostype => "ubuntu-8.04",
 113         origin => 'ubuntu',
 114     },
 115     'intrepid' => {
 116         ostype => "ubuntu-8.10",
 117         origin => 'ubuntu',
 118     },
 119     'jaunty' => {
 120         ostype => "ubuntu-9.04",
 121         origin => 'ubuntu',
 122     },
 123     'precise' => {
 124         ostype => "ubuntu-12.04",
 125         origin => 'ubuntu',
 126     },
 127     'trusty' => {
 128         ostype => "ubuntu-14.04",
 129         origin => 'ubuntu',
 130     },
 131     'vivid' => {
 132         ostype => "ubuntu-15.04",
 133         origin => 'ubuntu',
 134     },
 135     'wily' => {
 136         ostype => "ubuntu-15.10",
 137         origin => 'ubuntu',
 138     },
 139     'xenial' => {
 140         ostype => "ubuntu-16.04",
 141         origin => 'ubuntu',
 142     },
 143     'yakkety' => {
 144         ostype => "ubuntu-16.10",
 145         origin => 'ubuntu',
 146     },
 147     'zesty' => {
 148         ostype => "ubuntu-17.04",
 149         origin => 'ubuntu',
 150     },
 151     'artful' => {
 152         ostype => "ubuntu-17.10",
 153         origin => 'ubuntu',
 154     },
 155     'bionic' => {
 156         ostype => "ubuntu-18.04",
 157         origin => 'ubuntu',
 158     },
 159     'cosmic' => {
 160         ostype => "ubuntu-18.10",
 161         origin => 'ubuntu',
 162     },
 163     'disco' => {
 164         ostype => "ubuntu-19.04",
 165         origin => 'ubuntu',
 166     },
 167     'eoan' => {
 168         ostype => "ubuntu-19.10",
 169         origin => 'ubuntu',
 170     },
 171     'focal' => {
 172         ostype => "ubuntu-20.04",
 173         origin => 'ubuntu',
 174     },
 175     'groovy' => {
 176         ostype => "ubuntu-20.10",
 177         origin => 'ubuntu',
 178     },
 179     'hirsute' => {
 180         ostype => "ubuntu-21.04",
 181         origin => 'ubuntu',
 182     },
 183     'impish' => {
 184         ostype => "ubuntu-21.10",
 185         origin => 'ubuntu',
 186     },
 187 };
 188
 189 sub get_suite_info {
 190     my ($suite) = @_;
 191
 192     my $suiteinfo = $supported_suites->{$suite} || die "unsupported suite '$suite'!\n";
 193
 194     # set defaults
 195     $suiteinfo->{origin} //= 'debian';
 196     $suiteinfo->{suite} //= $suite;
 197
 198     return $suiteinfo;
 199 }
 200
 201 sub download {
 202     my ($self, $url, $path) = @_;
 203
 204     $self->logmsg ("download: $url\n");
 205     my $tmpfn = "$path.tmp$$";
 206     eval {
 207         $self->run_command ("wget -q '$url'  -O '$tmpfn'");
 208     };
 209
 210     my $err = $@;
 211     if ($err) {
 212         unlink $tmpfn;
 213         die $err;
 214     }
 215
 216     rename ($tmpfn, $path);
 217 }
 218
 219 sub write_file {
 220     my ($data, $file, $perm) = @_;
 221
 222     die "no filename" if !$file;
 223
 224     unlink $file;
 225
 226     my $fh = IO::File->new ($file, O_WRONLY | O_CREAT, $perm) ||
 227         die "unable to open file '$file'";
 228
 229     print $fh $data;
 230
 231     $fh->close;
 232 }
 233
 234 sub read_file {
 235     my ($file) = @_;
 236
 237     die "no filename" if !$file;
 238
 239     my $fh = IO::File->new ($file) ||
 240         die "unable to open file '$file'";
 241
 242     local $/; # slurp mode
 243
 244     my $data = <$fh>;
 245
 246     $fh->close;
 247
 248     return $data;
 249 }
 250
 251 sub read_config {
 252     my ($filename) = @_;
 253
 254     my $res = {};
 255
 256     my $fh = IO::File->new ("<$filename") || return $res;
 257     my $rec = '';
 258
 259     while (defined (my $line = <$fh>)) {
 260         next if $line =~ m/^\#/;
 261         next if $line =~ m/^\s*$/;
 262         $rec .= $line;
 263     };
 264
 265     close ($fh);
 266
 267     chomp $rec;
 268     $rec .= "\n";
 269
 270     while ($rec) {
 271         if ($rec =~ s/^Description:\s*([^\n]*)(\n\s+.*)*$//si) {
 272             $res->{headline} = $1;
 273             chomp $res->{headline};
 274             my $long = $2;
 275             $long =~ s/^\s+/ /;
 276             $res->{description} = $long;
 277             chomp $res->{description};
 278         } elsif ($rec =~ s/^([^:]+):\s*(.*\S)\s*\n//) {
 279             my ($key, $value) = (lc ($1), $2);
 280             if ($key eq 'source' || $key eq 'mirror') {
 281                 push @{$res->{$key}}, $value;
 282             } else {
 283                 die "duplicate key '$key'\n" if defined ($res->{$key});
 284                 $res->{$key} = $value;
 285             }
 286         } else {
 287             die "unable to parse config file: $rec";
 288         }
 289     }
 290
 291     die "unable to parse config file" if $rec;
 292
 293     return $res;
 294 }
 295
 296 sub run_command {
 297     my ($self, $cmd, $input, $getoutput) = @_;
 298
 299     my $reader = IO::File->new();
 300     my $writer = IO::File->new();
 301     my $error  = IO::File->new();
 302
 303     my $orig_pid = $$;
 304
 305     my $cmdstr = ref ($cmd) eq 'ARRAY' ? join (' ', @$cmd) : $cmd;
 306
 307     my $pid;
 308     eval {
 309         if (ref ($cmd) eq 'ARRAY') {
 310             $pid = open3 ($writer, $reader, $error, @$cmd) || die $!;
 311         } else {
 312             $pid = open3 ($writer, $reader, $error, $cmdstr) || die $!;
 313         }
 314     };
 315
 316     my $err = $@;
 317
 318     # catch exec errors
 319     if ($orig_pid != $$) {
 320         $self->logmsg ("ERROR: command '$cmdstr' failed - fork failed\n");
 321         POSIX::_exit (1);
 322         kill ('KILL', $$);
 323     }
 324
 325     die $err if $err;
 326
 327     print $writer $input if defined $input;
 328     close $writer;
 329
 330     my $select = new IO::Select;
 331     $select->add ($reader);
 332     $select->add ($error);
 333
 334     my $res = '';
 335     my $logfd = $self->{logfd};
 336
 337     while ($select->count) {
 338         my @handles = $select->can_read ();
 339
 340         foreach my $h (@handles) {
 341             my $buf = '';
 342             my $count = sysread ($h, $buf, 4096);
 343             if (!defined ($count)) {
 344                 waitpid ($pid, 0);
 345                 die "command '$cmdstr' failed: $!";
 346             }
 347             $select->remove ($h) if !$count;
 348
 349             print $logfd $buf;
 350
 351             $res .= $buf if $getoutput;
 352         }
 353     }
 354
 355     waitpid ($pid, 0);
 356     my $ec = ($? >> 8);
 357
 358     die "command '$cmdstr' failed with exit code $ec\n" if $ec;
 359
 360     return $res;
 361 }
 362
 363 sub logmsg {
 364     my $self = shift;
 365     print STDERR @_;
 366     $self->writelog (@_);
 367 }
 368
 369 sub writelog {
 370     my $self = shift;
 371     my $fd = $self->{logfd};
 372     print $fd @_;
 373 }
 374
 375 sub __sample_config {
 376     my ($self) = @_;
 377
 378     my $data = '';
 379     my $arch = $self->{config}->{architecture};
 380
 381     my $ostype = $self->{config}->{ostype};
 382
 383     if ($ostype =~ m/^de(bi|vu)an-/) {
 384         $data .= "lxc.include = /usr/share/lxc/config/debian.common.conf\n";
 385     } elsif ($ostype =~ m/^ubuntu-/) {
 386         $data .= "lxc.include = /usr/share/lxc/config/ubuntu.common.conf\n";
 387     } else {
 388         die "unknown os type '$ostype'\n";
 389     }
 390     $data .= "lxc.uts.name = localhost\n";
 391     $data .= "lxc.rootfs.path = $self->{rootfs}\n";
 392
 393     return $data;
 394 }
 395
 396 sub __allocate_ve {
 397     my ($self) = @_;
 398
 399     my $cid;
 400     if (my $fd = IO::File->new (".veid")) {
 401         $cid = <$fd>;
 402         chomp $cid;
 403         close ($fd);
 404     }
 405
 406
 407     $self->{working_dir} = getcwd;
 408     $self->{veconffile} = "$self->{working_dir}/config";
 409     $self->{rootfs} = "$self->{working_dir}/rootfs";
 410
 411     if ($cid) {
 412         $self->{veid} = $cid;
 413         return $cid;
 414     }
 415
 416     my $uuid;
 417     my $uuid_str;
 418     UUID::generate($uuid);
 419     UUID::unparse($uuid, $uuid_str);
 420     $self->{veid} = $uuid_str;
 421
 422     my $fd = IO::File->new (">.veid") ||
 423         die "unable to write '.veid'\n";
 424     print $fd "$self->{veid}\n";
 425     close ($fd);
 426
 427     my $cdata = $self->__sample_config();
 428
 429     my $fh = IO::File->new ($self->{veconffile}, O_WRONLY|O_CREAT|O_EXCL) ||
 430         die "unable to write lxc config file '$self->{veconffile}' - $!";
 431     print $fh $cdata;
 432     close ($fh);
 433
 434     mkdir $self->{rootfs} || die "unable to create rootfs - $!";
 435
 436     $self->logmsg ("allocated VE $self->{veid}\n");
 437
 438     return $self->{veid};
 439 }
 440
 441 sub new {
 442     my ($class, $config) = @_;
 443
 444     $class = ref ($class) || $class;
 445
 446     my $self = {};
 447
 448     $config = read_config ('dab.conf') if !$config;
 449
 450     $self->{config} = $config;
 451
 452     bless $self, $class;
 453
 454     $self->{logfile} = "logfile";
 455     $self->{logfd} = IO::File->new (">>$self->{logfile}") ||
 456         die "unable to open log file";
 457
 458     my $arch = $config->{architecture};
 459     die "no 'architecture' specified\n" if !$arch;
 460
 461     die "unsupported architecture '$arch'\n"
 462         if $arch !~ m/^(i386|amd64)$/;
 463
 464     my $suite = $config->{suite} || die "no 'suite' specified\n";
 465
 466     my $suiteinfo = get_suite_info($suite);
 467     $suite = $suiteinfo->{suite};
 468     $config->{ostype} = $suiteinfo->{ostype};
 469
 470     my $name = $config->{name} || die "no 'name' specified\n";
 471
 472     $name =~ m/^[a-z][0-9a-z\-\*\.]+$/ ||
 473         die "illegal characters in name '$name'\n";
 474
 475     my $version = $config->{version};
 476     die "no 'version' specified\n" if !$version;
 477     die "no 'section' specified\n" if !$config->{section};
 478     die "no 'description' specified\n" if !$config->{headline};
 479     die "no 'maintainer' specified\n" if !$config->{maintainer};
 480
 481     if ($name =~ m/^$config->{ostype}/) {
 482         $self->{targetname} = "${name}_${version}_$config->{architecture}";
 483     } else {
 484         $self->{targetname} = "$config->{ostype}-${name}_" .
 485             "${version}_$config->{architecture}";
 486     }
 487
 488     if (!$config->{source}) {
 489         if (lc($suiteinfo->{origin}) eq 'debian') {
 490             if ($suite eq 'etch' || $suite eq 'lenny') {
 491                 push @{$config->{source}}, (
 492                     'http://ftp.debian.org/debian SUITE main contrib',
 493                     'http://security.debian.org SUITE/updates main contrib',
 494                 );
 495             } elsif ($suite eq 'bullseye' || $suite eq 'bookworm') {
 496                 push @{$config->{source}}, (
 497                     "http://ftp.debian.org/debian SUITE main contrib",
 498                     "http://ftp.debian.org/debian SUITE-updates main contrib",
 499                     "http://security.debian.org SUITE-security main contrib",
 500                 );
 501             } else {
 502                 push @{$config->{source}}, (
 503                     "http://ftp.debian.org/debian SUITE main contrib",
 504                     "http://ftp.debian.org/debian SUITE-updates main contrib",
 505                     "http://security.debian.org SUITE/updates main contrib",
 506                 );
 507             }
 508         } elsif (lc($suiteinfo->{origin}) eq 'ubuntu') {
 509             my $comp = "main restricted universe multiverse";
 510             push @{$config->{source}}, (
 511                 "http://archive.ubuntu.com/ubuntu SUITE $comp",
 512                 "http://archive.ubuntu.com/ubuntu SUITE-updates $comp",
 513                 "http://archive.ubuntu.com/ubuntu SUITE-security $comp",
 514             );
 515         } else {
 516             die "implement me";
 517         }
 518     }
 519
 520     my $sources = undef;
 521
 522     foreach my $s (@{$config->{source}}) {
 523         if ($s =~ m@^\s*((http|ftp)://\S+)\s+(\S+)((\s+(\S+))+)$@) {
 524             my ($url, $su, $components) = ($1, $3, $4);
 525             $su =~ s/SUITE/$suite/;
 526             $components =~ s/^\s+//;
 527             $components =~ s/\s+$//;
 528             my $ca;
 529             foreach my $co (split (/\s+/, $components)) {
 530                 push @$ca, $co;
 531             }
 532             $ca = ['main'] if !$ca;
 533
 534             push @$sources, {
 535                 source => $url,
 536                 comp => $ca,
 537                 suite => $su,
 538             };
 539         } else {
 540             die "syntax error in source spezification '$s'\n";
 541         }
 542     }
 543
 544     foreach my $m (@{$config->{mirror}}) {
 545         if ($m =~ m@^\s*((http|ftp)://\S+)\s*=>\s*((http|ftp)://\S+)\s*$@) {
 546             my ($ms, $md) = ($1, $3);
 547             my $found;
 548             foreach my $ss (@$sources) {
 549                 if ($ss->{source} eq $ms) {
 550                     $found = 1;
 551                     $ss->{mirror} = $md;
 552                     last;
 553                 }
 554             }
 555             die "unusable mirror $ms\n" if !$found;
 556         } else {
 557             die "syntax error in mirror spezification '$m'\n";
 558         }
 559     }
 560
 561     $self->{sources} = $sources;
 562
 563     $self->{infodir} = "info";
 564
 565     $self->__allocate_ve ();
 566
 567     $self->{cachedir} = ($config->{cachedir} || 'cache')  . "/$suite";;
 568
 569     my $incl = [qw (less ssh openssh-server logrotate)];
 570
 571     my $excl = [qw (modutils reiserfsprogs ppp pppconfig pppoe
 572                     pppoeconf nfs-common mtools ntp)];
 573
 574     # ubuntu has too many dependencies on udev, so
 575     # we cannot exclude it (instead we disable udevd)
 576
 577     if ($suite eq 'vivid' || $suite eq 'wily' || $suite eq 'xenial' ||
 578         $suite eq 'yakkety' || $suite eq 'zesty' || $suite eq 'artful' ||
 579         $suite eq 'bionic' || $suite eq 'cosmic' || $suite eq 'disco' ||
 580         $suite eq 'eoan' || $suite eq 'focal' || $suite eq 'groovy'
 581         || $suite eq 'hirsute' || $suite eq 'impish'
 582     ) {
 583         push @$incl, 'isc-dhcp-client';
 584         push @$excl, qw(libmodule-build-perl);
 585     } elsif ($suite eq 'trusty') {
 586         push @$excl, qw(systemd systemd-services libpam-systemd libsystemd-daemon0 memtest86+);
 587    } elsif ($suite eq 'precise') {
 588         push @$excl, qw(systemd systemd-services libpam-systemd libsystemd-daemon0 memtest86+ ubuntu-standard);
 589     } elsif ($suite eq 'hardy') {
 590         push @$excl, qw(kbd);
 591         push @$excl, qw(apparmor apparmor-utils ntfs-3g
 592                         friendly-recovery);
 593     } elsif ($suite eq 'intrepid' || $suite eq 'jaunty') {
 594         push @$excl, qw(apparmor apparmor-utils libapparmor1 libapparmor-perl
 595                         libntfs-3g28 ntfs-3g friendly-recovery);
 596     } elsif ($suite eq 'jessie') {
 597         push @$incl, 'sysvinit-core'; # avoid systemd and udev
 598         push @$incl, 'libperl4-corelibs-perl'; # to make lsof happy
 599         push @$excl, qw(systemd systemd-sysv udev module-init-tools pciutils hdparm memtest86+ parted);
 600     } elsif ($suite eq 'stretch' || $suite eq 'buster' || $suite eq 'bullseye' || $suite eq 'bookworm') {
 601         push @$excl, qw(module-init-tools pciutils hdparm memtest86+ parted);
 602      } else {
 603         push @$excl, qw(udev module-init-tools pciutils hdparm memtest86+ parted);
 604     }
 605
 606     $self->{incl} = $incl;
 607     $self->{excl} = $excl;
 608
 609     return $self;
 610 }
 611
 612 sub initialize {
 613     my ($self) = @_;
 614
 615     my $infodir = $self->{infodir};
 616     my $arch = $self->{config}->{architecture};
 617
 618     rmtree $infodir;
 619     mkpath $infodir;
 620
 621     # truncate log
 622     my $logfd = $self->{logfd} = IO::File->new (">$self->{logfile}") ||
 623         die "unable to open log file";
 624
 625     my $COMPRESSORS = [
 626         {
 627             ext => 'xz',
 628             decomp => 'xz -d',
 629         },
 630         {
 631             ext => 'gz',
 632             decomp => 'gzip -d',
 633         },
 634     ];
 635
 636     foreach my $ss (@{$self->{sources}}) {
 637         my $src = $ss->{mirror} || $ss->{source};
 638         my $path = "dists/$ss->{suite}/Release";
 639         my $url = "$src/$path";
 640         my $target = __url_to_filename ("$ss->{source}/$path");
 641         eval {
 642             $self->download ($url, "$infodir/$target");
 643             $self->download ("$url.gpg", "$infodir/$target.gpg");
 644             # fixme: impl. verify (needs --keyring option)
 645         };
 646         if (my $err = $@) {
 647             print $logfd $@;
 648             warn "Release info ignored\n";
 649         };
 650
 651         foreach my $comp (@{$ss->{comp}}) {
 652             foreach my $compressor (@$COMPRESSORS) {
 653                 $path = "dists/$ss->{suite}/$comp/binary-$arch/Packages.$compressor->{ext}";
 654                 $target = "$infodir/" . __url_to_filename ("$ss->{source}/$path");
 655                 my $pkgsrc = "$src/$path";
 656                 eval {
 657                     $self->download ($pkgsrc, $target);
 658                     $self->run_command ("$compressor->{decomp} '$target'");
 659                 };
 660                 if (my $err = $@) {
 661                     print $logfd "could not download Packages.$compressor->{ext}\n";
 662                 } else {
 663                     last;
 664                 }
 665             }
 666         }
 667     }
 668 }
 669
 670 sub write_config {
 671     my ($self, $filename, $size) = @_;
 672
 673     my $config = $self->{config};
 674
 675     my $data = '';
 676
 677     $data .= "Name: $config->{name}\n";
 678     $data .= "Version: $config->{version}\n";
 679     $data .= "Type: lxc\n";
 680     $data .= "OS: $config->{ostype}\n";
 681     $data .= "Section: $config->{section}\n";
 682     $data .= "Maintainer: $config->{maintainer}\n";
 683     $data .= "Architecture: $config->{architecture}\n";
 684     $data .= "Installed-Size: $size\n";
 685
 686     # optional
 687     $data .= "Infopage: $config->{infopage}\n" if $config->{infopage};
 688     $data .= "ManageUrl: $config->{manageurl}\n" if $config->{manageurl};
 689     $data .= "Certified: $config->{certified}\n" if $config->{certified};
 690
 691     # description
 692     $data .= "Description: $config->{headline}\n";
 693     $data .= "$config->{description}\n" if $config->{description};
 694
 695     write_file ($data, $filename, 0644);
 696 }
 697
 698 sub finalize {
 699     my ($self, $opts) = @_;
 700
 701     my $suite = $self->{config}->{suite};
 702     my $infodir = $self->{infodir};
 703     my $arch = $self->{config}->{architecture};
 704
 705     my $instpkgs = $self->read_installed ();
 706     my $pkginfo = $self->pkginfo();
 707     my $veid = $self->{veid};
 708     my $conffile = $self->{veconffile};
 709     my $rootdir = $self->{rootfs};
 710
 711     my $vestat = $self->ve_status();
 712     die "ve not running - unable to finalize\n" if !$vestat->{running};
 713
 714     # cleanup mysqld
 715     if (-f "$rootdir/etc/init.d/mysql") {
 716         $self->ve_command ("/etc/init.d/mysql stop");
 717     }
 718
 719     if (!($opts->{keepmycnf} || (-f "$rootdir/etc/init.d/mysql_randompw"))) {
 720         unlink "$rootdir/root/.my.cnf";
 721     }
 722
 723     if ($suite eq 'etch') {
 724         # enable apache2 startup
 725         if ($instpkgs->{apache2}) {
 726             write_file ("NO_START=0\n", "$rootdir/etc/default/apache2");
 727         } else {
 728             unlink "$rootdir/etc/default/apache2";
 729         }
 730     }
 731     $self->logmsg ("cleanup package status\n");
 732     # prevent auto selection of all standard, required or important
 733     # packages which are not installed
 734     foreach my $pkg (keys %$pkginfo) {
 735         my $pri = $pkginfo->{$pkg}->{priority};
 736         if ($pri && ($pri eq 'required' || $pri eq 'important'
 737                      || $pri eq 'standard')) {
 738             if (!$instpkgs->{$pkg}) {
 739                 $self->ve_dpkg_set_selection ($pkg, 'purge');
 740             }
 741         }
 742     }
 743
 744     $self->ve_command ("apt-get clean");
 745
 746     $self->logmsg ("update available package list\n");
 747
 748     $self->ve_command ("dpkg --clear-avail");
 749     foreach my $ss (@{$self->{sources}}) {
 750         my $relsrc = __url_to_filename ("$ss->{source}/dists/$ss->{suite}/Release");
 751         if (-f "$infodir/$relsrc" && -f "$infodir/$relsrc.gpg") {
 752             $self->run_command ("cp '$infodir/$relsrc' '$rootdir/var/lib/apt/lists/$relsrc'");
 753             $self->run_command ("cp '$infodir/$relsrc.gpg' '$rootdir/var/lib/apt/lists/$relsrc.gpg'");
 754         }
 755         foreach my $comp (@{$ss->{comp}}) {
 756             my $src = __url_to_filename ("$ss->{source}/dists/$ss->{suite}/" .
 757                                          "$comp/binary-$arch/Packages");
 758             my $target = "/var/lib/apt/lists/$src";
 759             $self->run_command ("cp '$infodir/$src' '$rootdir/$target'");
 760             $self->ve_command ("dpkg --merge-avail '$target'");
 761         }
 762     }
 763
 764     # set dselect default method
 765     write_file ("apt apt\n", "$rootdir/var/lib/dpkg/cmethopt");
 766
 767     $self->ve_divert_remove ("/usr/sbin/policy-rc.d");
 768
 769     $self->ve_divert_remove ("/sbin/start-stop-daemon");
 770
 771     $self->ve_divert_remove ("/sbin/init");
 772
 773     # finally stop the VE
 774     $self->run_command ("lxc-stop -n $veid --rcfile $conffile --kill");
 775
 776     unlink "$rootdir/sbin/defenv";
 777
 778     unlink <$rootdir/root/dead.letter*>;
 779
 780     unlink "$rootdir/var/log/init.log";
 781
 782     unlink "$rootdir/aquota.group";
 783
 784     unlink "$rootdir/aquota.user";
 785
 786     write_file ("", "$rootdir/var/log/syslog");
 787
 788     my $get_path_size = sub {
 789         my ($path) = @_;
 790         my $sizestr = $self->run_command ("du -sm $path", undef, 1);
 791         if ($sizestr =~ m/^(\d+)\s+\Q$path\E$/) {
 792             return int($1);
 793         } else {
 794             die "unable to detect size for '$path'\n";
 795         }
 796     };
 797
 798     $self->logmsg ("detecting final appliance size: ");
 799     my $size = $get_path_size->($rootdir);
 800     $self->logmsg ("$size MB\n");
 801
 802     $self->write_config ("$rootdir/etc/appliance.info", $size);
 803
 804     $self->logmsg ("creating final appliance archive\n");
 805
 806     my $target = "$self->{targetname}.tar";
 807
 808     my $compressor = $opts->{compressor} // 'gz';
 809     my $compressor2cmd_map = {
 810         gz => 'gzip',
 811         gzip => 'gzip',
 812         zst => 'zstd -9',
 813         zstd => 'zstd -9',
 814         'zstd-max' => 'zstd -19 -T0', # maximal level where the decompressor can still run efficiently
 815     };
 816     my $compressor2ending = {
 817         gzip => 'gz',
 818         zstd => 'zst',
 819         'zstd-max' => 'zst',
 820     };
 821     my $compressor_cmd = $compressor2cmd_map->{$compressor};
 822     die "unkown compressor '$compressor', use one of: ". join(', ', sort keys %$compressor2cmd_map)
 823         if !defined($compressor_cmd);
 824
 825     my $ending = $compressor2ending->{$compressor} // $compressor;
 826     my $final_archive = "${target}.${ending}";
 827     unlink $target;
 828     unlink $final_archive;
 829
 830     $self->run_command ("tar cpf $target --numeric-owner -C '$rootdir' ./etc/appliance.info");
 831     $self->run_command ("tar rpf $target --numeric-owner -C '$rootdir' --exclude ./etc/appliance.info .");
 832     $self->run_command ("$compressor_cmd $target");
 833
 834     $self->logmsg ("detecting final commpressed appliance size: ");
 835     $size = $get_path_size->($final_archive);
 836     $self->logmsg ("$size MB\n");
 837
 838     $self->logmsg ("appliance archive: $final_archive\n");
 839 }
 840
 841 sub read_installed {
 842     my ($self) = @_;
 843
 844     my $rootdir = $self->{rootfs};
 845
 846     my $pkgfilelist = "$rootdir/var/lib/dpkg/status";
 847     local $/ = '';
 848     open (PKGLST, "<$pkgfilelist") ||
 849         die "unable to open '$pkgfilelist'";
 850
 851     my $pkglist = {};
 852
 853     while (my $rec = <PKGLST>) {
 854         chomp $rec;
 855         $rec =~ s/\n\s+/ /g;
 856         $rec .= "\n";
 857         my $res = {};
 858
 859         while ($rec =~ s/^([^:]+):\s+(.*)\s*\n//) {
 860             $res->{lc $1} = $2;
 861         }
 862
 863         my $pkg = $res->{'package'};
 864         if (my $status = $res->{status}) {
 865             my @sa = split (/\s+/, $status);
 866             my $stat = $sa[0];
 867             if ($stat && ($stat ne 'purge')) {
 868                 $pkglist->{$pkg} = $res;
 869             }
 870         }
 871     }
 872
 873     close (PKGLST);
 874
 875     return $pkglist;
 876 }
 877
 878 sub ve_status {
 879     my ($self) = @_;
 880
 881     my $veid = $self->{veid};
 882
 883     my $res = { running => 0 };
 884
 885     $res->{exist} = 1 if -d "$self->{rootfs}/usr";
 886
 887     my $filename = "/proc/net/unix";
 888
 889     # similar test is used by lcxcontainers.c: list_active_containers
 890     my $fh = IO::File->new ($filename, "r");
 891     return $res if !$fh;
 892
 893     while (defined(my $line = <$fh>)) {
 894         if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
 895             my $path = $1;
 896             if ($path =~ m!^@/\S+/$veid/command$!) {
 897                 $res->{running} = 1;
 898             }
 899         }
 900     }
 901     close($fh);
 902
 903     return $res;
 904 }
 905
 906 sub ve_command {
 907     my ($self, $cmd, $input) = @_;
 908
 909     my $veid = $self->{veid};
 910     my $conffile = $self->{veconffile};
 911
 912     if (ref ($cmd) eq 'ARRAY') {
 913         unshift @$cmd, 'lxc-attach', '-n', $veid, '--rcfile', $conffile, '--clear-env', '--', 'defenv';
 914         $self->run_command ($cmd, $input);
 915     } else {
 916         $self->run_command ("lxc-attach -n $veid --rcfile $conffile --clear-env -- defenv $cmd", $input);
 917     }
 918 }
 919
 920 # like ve_command, but pipes stdin correctly
 921 sub ve_exec {
 922     my ($self, @cmd) = @_;
 923
 924     my $veid = $self->{veid};
 925     my $conffile = $self->{veconffile};
 926
 927     my $reader;
 928     my $pid = open2($reader, "<&STDIN", 'lxc-attach', '-n', $veid,  '--rcfile', $conffile, '--',
 929                     'defenv', @cmd) || die "unable to exec command";
 930
 931     while (defined (my $line = <$reader>)) {
 932         $self->logmsg ($line);
 933     }
 934
 935     waitpid ($pid, 0);
 936     my $rc = $? >> 8;
 937
 938     die "ve_exec failed - status $rc\n" if $rc != 0;
 939 }
 940
 941 sub ve_divert_add {
 942     my ($self, $filename) = @_;
 943
 944     $self->ve_command ("dpkg-divert --add --divert '$filename.distrib' " .
 945                        "--rename '$filename'");
 946 }
 947 sub ve_divert_remove {
 948     my ($self, $filename) = @_;
 949
 950     my $rootdir = $self->{rootfs};
 951
 952     unlink "$rootdir/$filename";
 953     $self->ve_command ("dpkg-divert --remove --rename '$filename'");
 954 }
 955
 956 sub ve_debconfig_set {
 957     my ($self, $dcdata) = @_;
 958
 959     my $rootdir = $self->{rootfs};
 960     my $cfgfile = "/tmp/debconf.txt";
 961     write_file ($dcdata, "$rootdir/$cfgfile");
 962     $self->ve_command ("debconf-set-selections $cfgfile");
 963     unlink "$rootdir/$cfgfile";
 964 }
 965
 966 sub ve_dpkg_set_selection {
 967     my ($self, $pkg, $status) = @_;
 968
 969     $self->ve_command ("dpkg --set-selections", "$pkg $status");
 970 }
 971
 972 sub ve_dpkg {
 973     my ($self, $cmd, @pkglist) = @_;
 974
 975     return if !scalar (@pkglist);
 976
 977     my $pkginfo = $self->pkginfo();
 978
 979     my $rootdir = $self->{rootfs};
 980     my $cachedir = $self->{cachedir};
 981
 982     my @files;
 983
 984     foreach my $pkg (@pkglist) {
 985         my $filename = $self->getpkgfile ($pkg);
 986         $self->run_command ("cp '$cachedir/$filename' '$rootdir/$filename'");
 987         push @files, "/$filename";
 988         $self->logmsg ("$cmd: $pkg\n");
 989     }
 990
 991     my $fl = join (' ', @files);
 992
 993     if ($cmd eq 'install') {
 994         $self->ve_command ("dpkg --force-depends --force-confold --install $fl");
 995     } elsif ($cmd eq 'unpack') {
 996         $self->ve_command ("dpkg --force-depends --unpack $fl");
 997     } else {
 998         die "internal error";
 999     }
1000
1001     foreach my $fn (@files) { unlink "$rootdir$fn"; }
1002 }
1003
1004 sub ve_destroy {
1005     my ($self) = @_;
1006
1007     my $veid = $self->{veid}; # fixme
1008     my $conffile = $self->{veconffile};
1009
1010     my $vestat = $self->ve_status();
1011     if ($vestat->{running}) {
1012         $self->run_command ("lxc-stop -n $veid --rcfile $conffile --kill");
1013     }
1014
1015     rmtree $self->{rootfs};
1016     unlink $self->{veconffile};
1017 }
1018
1019 sub ve_init {
1020     my ($self) = @_;
1021
1022     my $veid = $self->{veid};
1023     my $conffile = $self->{veconffile};
1024
1025     $self->logmsg ("initialize VE $veid\n");
1026
1027     my $vestat = $self->ve_status();
1028     if ($vestat->{running}) {
1029         $self->run_command ("lxc-stop -n $veid --rcfile $conffile --kill");
1030     }
1031
1032     rmtree $self->{rootfs};
1033     mkpath $self->{rootfs};
1034 }
1035
1036 sub __deb_version_cmp {
1037     my ($cur, $op, $new) = @_;
1038
1039     if (system("dpkg", "--compare-versions", $cur, $op, $new) == 0) {
1040         return 1;
1041     }
1042
1043     return 0;
1044 }
1045
1046 sub __parse_packages {
1047     my ($pkginfo, $filename, $src) = @_;
1048
1049     local $/ = '';
1050     open (PKGLST, "<$filename") ||
1051         die "unable to open '$filename'";
1052
1053     while (my $rec = <PKGLST>) {
1054         $rec =~ s/\n\s+/ /g;
1055         chomp $rec;
1056         $rec .= "\n";
1057
1058         my $res = {};
1059
1060         while ($rec =~ s/^([^:]+):\s+(.*)\s*\n//) {
1061             $res->{lc $1} = $2;
1062         }
1063
1064         my $pkg = $res->{'package'};
1065         if ($pkg && $res->{'filename'}) {
1066             my $cur;
1067             if (my $info = $pkginfo->{$pkg}) {
1068                 $cur = $info->{version};
1069             }
1070             my $new = $res->{version};
1071             if (!$cur || __deb_version_cmp ($cur, 'lt', $new)) {
1072                 if ($src) {
1073                     $res->{url} = "$src/$res->{'filename'}";
1074                 } else {
1075                     die "no url for package '$pkg'" if !$res->{url};
1076                 }
1077                 $pkginfo->{$pkg} = $res;
1078             }
1079         }
1080     }
1081
1082     close (PKGLST);
1083 }
1084
1085 sub pkginfo {
1086     my ($self) = @_;
1087
1088     return $self->{pkginfo} if $self->{pkginfo};
1089
1090     my $infodir = $self->{infodir};
1091     my $arch = $self->{config}->{architecture};
1092
1093     my $availfn = "$infodir/available";
1094
1095     my $pkginfo = {};
1096     my $pkgcount = 0;
1097
1098     # reading 'available' is faster, because it only contains latest version
1099     # (no need to do slow version compares)
1100     if (-f $availfn) {
1101             __parse_packages ($pkginfo, $availfn);
1102             $self->{pkginfo} = $pkginfo;
1103             return $pkginfo;
1104     }
1105
1106     $self->logmsg ("generating available package list\n");
1107
1108     foreach my $ss (@{$self->{sources}}) {
1109         foreach my $comp (@{$ss->{comp}}) {
1110             my $url = "$ss->{source}/dists/$ss->{suite}/$comp/binary-$arch/Packages";
1111             my $pkgfilelist = "$infodir/" . __url_to_filename ($url);
1112
1113             my $src = $ss->{mirror} || $ss->{source};
1114
1115             __parse_packages ($pkginfo, $pkgfilelist, $src);
1116         }
1117     }
1118
1119     if (my $dep = $self->{config}->{depends}) {
1120         foreach my $d (split (/,/, $dep)) {
1121             if ($d =~ m/^\s*(\S+)\s*(\((\S+)\s+(\S+)\)\s*)?$/) {
1122                 my ($pkg, $op, $rver) = ($1, $3, $4);
1123                 $self->logmsg ("checking dependencies: $d\n");
1124                 my $info = $pkginfo->{$pkg};
1125                 die "package '$pkg' not available\n" if !$info;
1126                 if ($op) {
1127                     my $cver = $info->{version};
1128                     if (!__deb_version_cmp ($cver, $op, $rver)) {
1129                         die "detected wrong version '$cver'\n";
1130                     }
1131                 }
1132             } else {
1133                 die "syntax error in depends field";
1134             }
1135         }
1136     }
1137
1138     $self->{pkginfo} = $pkginfo;
1139
1140     my $tmpfn = "$availfn.tmp$$";
1141     my $fd = IO::File->new (">$tmpfn");
1142     foreach my $pkg (sort keys %$pkginfo) {
1143         my $info = $pkginfo->{$pkg};
1144         print $fd "package: $pkg\n";
1145         foreach my $k (sort keys %$info) {
1146             next if $k eq 'description';
1147             next if $k eq 'package';
1148             my $v = $info->{$k};
1149             print $fd "$k: $v\n" if $v;
1150         }
1151         print $fd "description: $info->{description}\n" if $info->{description};
1152         print $fd "\n";
1153     }
1154     close ($fd);
1155
1156     rename ($tmpfn, $availfn);
1157
1158     return $pkginfo;
1159 }
1160
1161 sub __record_provides {
1162     my ($pkginfo, $closure, $list, $skipself) = @_;
1163
1164     foreach my $pname (@$list) {
1165         my $info = $pkginfo->{$pname};
1166         # fixme: if someone install packages directly using dpkg, there
1167         # is no entry in 'available', only in 'status'. In that case, we
1168         # should extract info from $instpkgs
1169         if (!$info) {
1170             warn "hint: ignoring provides for '$pname' - package not in 'available' list.\n";
1171             next;
1172         }
1173         if (my $prov = $info->{provides}) {
1174             my @pl = split (',', $prov);
1175             foreach my $p (@pl) {
1176                 $p =~ m/\s*(\S+)/;
1177                 if (!($skipself && (grep { $1 eq $_ } @$list))) {
1178                     $closure->{$1} = 1;
1179                 }
1180             }
1181         }
1182         $closure->{$pname} = 1 if !$skipself;
1183     }
1184 }
1185
1186 sub closure {
1187     my ($self, $closure, $list) = @_;
1188
1189     my $pkginfo = $self->pkginfo();
1190
1191     # first, record provided packages
1192     __record_provides ($pkginfo, $closure, $list, 1);
1193
1194     my $pkghash = {};
1195     my $pkglist = [];
1196
1197     # then resolve dependencies
1198     foreach my $pname (@$list) {
1199         __closure_single ($pkginfo, $closure, $pkghash, $pkglist, $pname, $self->{excl});
1200     }
1201
1202     return $pkglist;
1203 }
1204
1205 sub __closure_single {
1206     my ($pkginfo, $closure, $pkghash, $pkglist, $pname, $excl) = @_;
1207
1208     $pname =~ s/^\s+//;
1209     $pname =~ s/\s+$//;
1210     $pname =~ s/:any$//;
1211
1212     return if $closure->{$pname};
1213
1214     my $info = $pkginfo->{$pname} || die "no such package '$pname'";
1215
1216     my $dep = $info->{depends};
1217     my $predep = $info->{'pre-depends'};
1218
1219     my $size = $info->{size};
1220     my $url = $info->{url};
1221
1222     $url || die "$pname: no url for package '$pname'";
1223
1224     if (!$pkghash->{$pname}) {
1225         unshift @$pkglist, $pname;
1226         $pkghash->{$pname} = 1;
1227     }
1228
1229     __record_provides ($pkginfo, $closure, [$pname]) if $info->{provides};
1230
1231     $closure->{$pname} = 1;
1232
1233     #print "$url\n";
1234
1235     my @l;
1236
1237     push  @l, split (/,/, $predep) if $predep;
1238     push  @l, split (/,/, $dep) if $dep;
1239
1240   DEPEND: foreach my $p (@l) {
1241       my @l1 = split (/\|/, $p);
1242       foreach my $p1 (@l1) {
1243           if ($p1 =~ m/^\s*(\S+).*/) {
1244               #printf (STDERR "$pname: $p --> $1\n");
1245               if ($closure->{$1}) {
1246                   next DEPEND; # dependency already met
1247               }
1248           }
1249       }
1250       # search for non-excluded alternative
1251       my $found;
1252       foreach my $p1 (@l1) {
1253           if ($p1 =~ m/^\s*(\S+).*/) {
1254               next if grep { $1 eq $_ } @$excl;
1255               $found = $1;
1256               last;
1257           }
1258       }
1259       die "package '$pname' depends on exclusion '$p'\n" if !$found;
1260
1261       #printf (STDERR "$pname: $p --> $found\n");
1262
1263       __closure_single ($pkginfo, $closure, $pkghash, $pkglist, $found, $excl);
1264   }
1265 }
1266
1267 sub cache_packages {
1268     my ($self, $pkglist) = @_;
1269
1270     foreach my $pkg (@$pkglist) {
1271         $self->getpkgfile ($pkg);
1272     }
1273 }
1274
1275 sub getpkgfile {
1276     my ($self, $pkg) = @_;
1277
1278     my $pkginfo = $self->pkginfo();
1279     my $info = $pkginfo->{$pkg} || die "no such package '$pkg'";
1280     my $cachedir = $self->{cachedir};
1281
1282     my $url = $info->{url};
1283
1284     my $filename;
1285     if ($url =~ m|/([^/]+.deb)$|) {
1286         $filename = $1;
1287     } else {
1288         die "internal error";
1289     }
1290
1291     return $filename if -f "$cachedir/$filename";
1292
1293     mkpath $cachedir;
1294
1295     $self->download ($url, "$cachedir/$filename");
1296
1297     return $filename;
1298 }
1299
1300 sub install_init_script {
1301     my ($self, $script, $runlevel, $prio) = @_;
1302
1303     my $suite = $self->{config}->{suite};
1304     my $rootdir = $self->{rootfs};
1305
1306     my $base = basename ($script);
1307     my $target = "$rootdir/etc/init.d/$base";
1308
1309     $self->run_command ("install -m 0755 '$script' '$target'");
1310     if ($suite eq 'etch' || $suite eq 'lenny') {
1311         $self->ve_command ("update-rc.d $base start $prio $runlevel .");
1312     } elsif ($suite eq 'xenial' || $suite eq 'wily' || $suite eq 'vivid' ||
1313              $suite eq 'yakkety' || $suite eq 'zesty' || $suite eq 'artful' ||
1314              $suite eq 'bionic' || $suite eq 'cosmic' || $suite eq 'disco' ||
1315              $suite eq 'eoan' || $suite eq 'focal' || $suite eq 'groovy'
1316              || $suite eq 'hirsute' || $suite eq 'impish'
1317     ) {
1318         die "unable to install init script (system uses systemd)\n";
1319     } elsif ($suite eq 'trusty' || $suite eq 'precise') {
1320         die "unable to install init script (system uses upstart)\n";
1321     } else {
1322         $self->ve_command ("insserv $base");
1323     }
1324
1325     return $target;
1326 }
1327
1328 sub bootstrap {
1329     my ($self, $opts) = @_;
1330
1331     my $pkginfo = $self->pkginfo();
1332     my $veid = $self->{veid};
1333     my $suite = $self->{config}->{suite};
1334
1335     my $important = [ @{$self->{incl}} ];
1336     my $required;
1337     my $standard;
1338
1339     my $mta = $opts->{exim} ? 'exim' : 'postfix';
1340
1341     if ($mta eq 'postfix') {
1342         push @$important, "postfix";
1343     }
1344
1345     foreach my $p (sort keys %$pkginfo) {
1346         next if grep { $p eq $_ } @{$self->{excl}};
1347         my $pri = $pkginfo->{$p}->{priority};
1348         next if !$pri;
1349         next if $mta ne 'exim' && $p =~ m/exim/;
1350         next if $p =~ m/(selinux|semanage|policycoreutils)/;
1351
1352         push @$required, $p  if $pri eq 'required';
1353         push @$important, $p if $pri eq 'important';
1354         push @$standard, $p if $pri eq 'standard' && !$opts->{minimal};
1355     }
1356
1357     my $closure = {};
1358     $required = $self->closure ($closure, $required);
1359     $important = $self->closure ($closure, $important);
1360
1361     if (!$opts->{minimal}) {
1362         push @$standard, 'xbase-clients';
1363         $standard = $self->closure ($closure, $standard);
1364     }
1365
1366     # test if we have all 'ubuntu-minimal' and 'ubuntu-standard' packages
1367     # except those explicitly excluded
1368     if ($suite eq 'hardy' || $suite eq 'intrepid' || $suite eq 'jaunty') {
1369         my $mdeps = $pkginfo->{'ubuntu-minimal'}->{depends};
1370         foreach my $d (split (/,/, $mdeps)) {
1371             if ($d =~ m/^\s*(\S+)$/) {
1372                 my $pkg = $1;
1373                 next if $closure->{$pkg};
1374                 next if grep { $pkg eq $_ } @{$self->{excl}};
1375                 die "missing ubuntu-minimal package '$pkg'\n";
1376             }
1377         }
1378         if (!$opts->{minimal}) {
1379             $mdeps = $pkginfo->{'ubuntu-standard'}->{depends};
1380             foreach my $d (split (/,/, $mdeps)) {
1381                 if ($d =~ m/^\s*(\S+)$/) {
1382                     my $pkg = $1;
1383                     next if $closure->{$pkg};
1384                     next if grep { $pkg eq $_ } @{$self->{excl}};
1385                     die "missing ubuntu-standard package '$pkg'\n";
1386                 }
1387             }
1388         }
1389     }
1390
1391     # download/cache all files first
1392     $self->cache_packages ($required);
1393     $self->cache_packages ($important);
1394     $self->cache_packages ($standard);
1395
1396     my $rootdir = $self->{rootfs};
1397
1398     # extract required packages first
1399     $self->logmsg ("create basic environment\n");
1400     foreach my $p (@$required) {
1401         my $filename = $self->getpkgfile ($p);
1402         my $content = $self->run_command("ar -t '$self->{cachedir}/$filename'", undef, 1);
1403         if ($content =~ m/^data.tar.xz$/m) {
1404             $self->run_command ("ar -p '$self->{cachedir}/$filename' data.tar.xz | tar -C '$rootdir' -xJf - --keep-directory-symlink");
1405         } else {
1406             $self->run_command ("ar -p '$self->{cachedir}/$filename' data.tar.gz | tar -C '$rootdir' -xzf - --keep-directory-symlink");
1407         }
1408     }
1409
1410     # fake dpkg status
1411     my $data = "Package: dpkg\n" .
1412         "Version: $pkginfo->{dpkg}->{version}\n" .
1413         "Status: install ok installed\n";
1414
1415     write_file ($data, "$rootdir/var/lib/dpkg/status");
1416     write_file ("", "$rootdir/var/lib/dpkg/info/dpkg.list");
1417     write_file ("", "$rootdir/var/lib/dpkg/available");
1418
1419     $data = '';
1420     foreach my $ss (@{$self->{sources}}) {
1421         my $url = $ss->{source};
1422         my $comp = join (' ', @{$ss->{comp}});
1423         $data .= "deb $url $ss->{suite} $comp\n\n";
1424     }
1425
1426     write_file ($data, "$rootdir/etc/apt/sources.list");
1427
1428     $data = "# UNCONFIGURED FSTAB FOR BASE SYSTEM\n";
1429     write_file ($data, "$rootdir/etc/fstab", 0644);
1430
1431     write_file ("localhost\n", "$rootdir/etc/hostname", 0644);
1432
1433     # avoid warnings about non-existent resolv.conf
1434     write_file ("", "$rootdir/etc/resolv.conf", 0644);
1435
1436     if (
1437         $suite eq 'impish' ||
1438         $suite eq 'hirsute' || $suite eq 'groovy' || $suite eq 'focal' ||
1439         $suite eq 'eoan' || $suite eq 'disco' || $suite eq 'cosmic' ||
1440         $suite eq 'bionic' || $suite eq 'artful' ||
1441         $suite eq 'zesty' || $suite eq 'yakkety' || $suite eq 'xenial' ||
1442         $suite eq 'wily'
1443     ) {
1444         # no need to configure loopback device
1445     } else {
1446         $data = "auto lo\niface lo inet loopback\n";
1447         mkdir "$rootdir/etc/network";
1448         write_file ($data, "$rootdir/etc/network/interfaces", 0644);
1449     }
1450
1451     # setup devices
1452     $self->run_command ("tar xzf '$devicetar' -C '$rootdir'");
1453
1454     # avoid warnings about missing default locale
1455     write_file ("LANG=\"C\"\n", "$rootdir/etc/default/locale", 0644);
1456
1457     # fake init
1458     rename ("$rootdir/sbin/init", "$rootdir/sbin/init.org");
1459     $self->run_command ("cp '$fake_init' '$rootdir/sbin/init'");
1460
1461     $self->run_command ("cp '$default_env' '$rootdir/sbin/defenv'");
1462
1463     $self->run_command ("lxc-start -n $veid -f $self->{veconffile}");
1464
1465     $self->logmsg ("initialize ld cache\n");
1466     $self->ve_command ("/sbin/ldconfig");
1467     $self->run_command ("ln -sf mawk '$rootdir/usr/bin/awk'");
1468
1469     $self->logmsg ("installing packages\n");
1470
1471     $self->ve_dpkg ('install', 'base-files', 'base-passwd');
1472
1473     $self->ve_dpkg ('install', 'dpkg');
1474
1475     $self->run_command ("ln -sf /usr/share/zoneinfo/UTC '$rootdir/etc/localtime'");
1476
1477     $self->run_command ("ln -sf bash '$rootdir/bin/sh'");
1478
1479     $self->ve_dpkg ('install', 'libc6');
1480     $self->ve_dpkg ('install', 'perl-base');
1481
1482     unlink "$rootdir/usr/bin/awk";
1483
1484     $self->ve_dpkg ('install', 'mawk');
1485     $self->ve_dpkg ('install', 'debconf');
1486
1487     # unpack required packages
1488     foreach my $p (@$required) {
1489         $self->ve_dpkg ('unpack', $p);
1490     }
1491
1492     rename ("$rootdir/sbin/init.org", "$rootdir/sbin/init");
1493     $self->ve_divert_add ("/sbin/init");
1494     $self->run_command ("cp '$fake_init' '$rootdir/sbin/init'");
1495
1496     # disable service activation
1497     $self->ve_divert_add ("/usr/sbin/policy-rc.d");
1498     $data = "#!/bin/sh\nexit 101\n";
1499     write_file ($data, "$rootdir/usr/sbin/policy-rc.d", 755);
1500
1501     # disable start-stop-daemon
1502     $self->ve_divert_add ("/sbin/start-stop-daemon");
1503     $data = <<EOD;
1504 #!/bin/sh
1505 echo
1506 echo \"Warning: Fake start-stop-daemon called, doing nothing\"
1507 EOD
1508     write_file ($data, "$rootdir/sbin/start-stop-daemon", 0755);
1509
1510     # disable udevd
1511     $self->ve_divert_add ("/sbin/udevd");
1512
1513     if ($suite eq 'etch') {
1514         # disable apache2 startup
1515         write_file ("NO_START=1\n", "$rootdir/etc/default/apache2");
1516     }
1517
1518     $self->logmsg ("configure required packages\n");
1519     $self->ve_command ("dpkg --force-confold --skip-same-version --configure -a");
1520
1521     # set postfix defaults
1522     if ($mta eq 'postfix') {
1523         $data = "postfix postfix/main_mailer_type select Local only\n";
1524         $self->ve_debconfig_set ($data);
1525
1526         $data = "postmaster: root\nwebmaster: root\n";
1527         write_file ($data, "$rootdir/etc/aliases");
1528     }
1529
1530     if ($suite eq 'jaunty') {
1531         # jaunty does not create /var/run/network, so network startup fails.
1532         # so we do not use tmpfs for /var/run and /var/lock
1533         $self->run_command ("sed -e 's/RAMRUN=yes/RAMRUN=no/' -e 's/RAMLOCK=yes/RAMLOCK=no/'  -i $rootdir/etc/default/rcS");
1534         # and create the directory here
1535         $self->run_command ("mkdir $rootdir/var/run/network");
1536     }
1537
1538     # unpack base packages
1539     foreach my $p (@$important) {
1540         $self->ve_dpkg ('unpack', $p);
1541     }
1542
1543     # start loopback
1544     if (-x "$rootdir/sbin/ifconfig") {
1545         $self->ve_command ("ifconfig lo up");
1546     } else {
1547         $self->ve_command ("ip link set lo up");
1548     }
1549
1550     $self->logmsg ("configure important packages\n");
1551     $self->ve_command ("dpkg --force-confold --skip-same-version --configure -a");
1552
1553     if (-d "$rootdir/etc/event.d") {
1554         unlink <$rootdir/etc/event.d/tty*>;
1555     }
1556
1557     if (-f "$rootdir/etc/inittab") {
1558         $self->run_command ("sed -i -e '/getty\\s38400\\stty[23456]/d' '$rootdir/etc/inittab'");
1559     }
1560
1561     # Link /etc/mtab to /proc/mounts, so df and friends will work:
1562     unlink "$rootdir/etc/mtab";
1563     $self->ve_command ("ln -s /proc/mounts /etc/mtab");
1564
1565     # reset password
1566     $self->ve_command ("usermod -L root");
1567
1568     if ($mta eq 'postfix') {
1569         $data = "postfix postfix/main_mailer_type select No configuration\n";
1570         $self->ve_debconfig_set ($data);
1571
1572         unlink "$rootdir/etc/mailname";
1573         write_file ($postfix_main_cf, "$rootdir/etc/postfix/main.cf");
1574     }
1575
1576     if (!$opts->{minimal}) {
1577         # unpack standard packages
1578         foreach my $p (@$standard) {
1579             $self->ve_dpkg ('unpack', $p);
1580         }
1581
1582         $self->logmsg ("configure standard packages\n");
1583         $self->ve_command ("dpkg --force-confold --skip-same-version --configure -a");
1584     }
1585
1586     # disable HWCLOCK access
1587     $self->run_command ("echo 'HWCLOCKACCESS=no' >> '$rootdir/etc/default/rcS'");
1588
1589     # disable hald
1590     $self->ve_divert_add ("/usr/sbin/hald");
1591
1592     # disable /dev/urandom init
1593     $self->run_command ("install -m 0755 '$script_init_urandom' '$rootdir/etc/init.d/urandom'");
1594
1595     if ($suite eq 'etch' || $suite eq 'hardy' || $suite eq 'intrepid' || $suite eq 'jaunty') {
1596         # avoid klogd start
1597         $self->ve_divert_add ("/sbin/klogd");
1598     }
1599
1600     # remove unnecessays sysctl entries to avoid warnings
1601     my $cmd = 'sed';
1602     $cmd .= ' -e \'s/^\(kernel\.printk.*\)/#\1/\'';
1603     $cmd .= ' -e \'s/^\(kernel\.maps_protect.*\)/#\1/\'';
1604     $cmd .= ' -e \'s/^\(fs\.inotify\.max_user_watches.*\)/#\1/\'';
1605     $cmd .= ' -e \'s/^\(vm\.mmap_min_addr.*\)/#\1/\'';
1606     $cmd .= " -i '$rootdir/etc/sysctl.conf'";
1607     $self->run_command ($cmd);
1608
1609     my $bindv6only = "$rootdir/etc/sysctl.d/bindv6only.conf";
1610     if (-f $bindv6only) {
1611         $cmd = 'sed';
1612         $cmd .= ' -e \'s/^\(net\.ipv6\.bindv6only.*\)/#\1/\'';
1613         $cmd .= " -i '$bindv6only'";
1614         $self->run_command ($cmd);
1615     }
1616
1617     if ($suite eq 'hardy' || $suite eq 'intrepid' || $suite eq 'jaunty') {
1618         # disable tty init (console-setup)
1619         my $cmd = 'sed';
1620         $cmd .= ' -e \'s/^\(ACTIVE_CONSOLES=.*\)/ACTIVE_CONSOLES=/\'';
1621         $cmd .= " -i '$rootdir/etc/default/console-setup'";
1622         $self->run_command ($cmd);
1623     }
1624
1625     if ($suite eq 'intrepid' || $suite eq 'jaunty') {
1626         # remove sysctl setup (avoid warnings at startup)
1627         my $filelist = "$rootdir/etc/sysctl.d/10-console-messages.conf";
1628         $filelist .= " $rootdir/etc/sysctl.d/10-process-security.conf" if $suite eq 'intrepid';
1629         $filelist .= " $rootdir/etc/sysctl.d/10-network-security.conf";
1630         $self->run_command ("rm $filelist");
1631     }
1632
1633     if (-e "$rootdir/lib/systemd/system/sys-kernel-config.mount") {
1634         $self->ve_command ("ln -s /dev/null /etc/systemd/system/sys-kernel-debug.mount");
1635     }
1636 }
1637
1638 sub enter {
1639     my ($self) = @_;
1640
1641     my $veid = $self->{veid};
1642     my $conffile = $self->{veconffile};
1643
1644     my $vestat = $self->ve_status();
1645
1646     if (!$vestat->{exist}) {
1647         $self->logmsg ("Please create the appliance first (bootstrap)");
1648         return;
1649     }
1650
1651     if (!$vestat->{running}) {
1652         $self->run_command ("lxc-start -n $veid -f $conffile");
1653     }
1654
1655     system ("lxc-attach -n $veid --rcfile $conffile --clear-env");
1656 }
1657
1658 sub ve_mysql_command {
1659     my ($self, $sql, $password) = @_;
1660
1661     #my $bootstrap = "/usr/sbin/mysqld --bootstrap --user=mysql --skip-grant-tables " .
1662     #"--skip-bdb  --skip-innodb --skip-ndbcluster";
1663
1664     $self->ve_command ("mysql", $sql);
1665 }
1666
1667 sub ve_mysql_bootstrap {
1668     my ($self, $sql, $password) = @_;
1669
1670     my $cmd;
1671
1672     my $suite = $self->{config}->{suite};
1673
1674     if ($suite eq 'jessie') {
1675         my $rootdir = $self->{rootfs};
1676         $self->run_command ("sed -e 's/^key_buffer\\s*=/key_buffer_size =/' -i $rootdir/etc/mysql/my.cnf");
1677     }
1678
1679     if ($suite eq 'squeeze' || $suite eq 'wheezy' || $suite eq 'jessie') {
1680         $cmd = "/usr/sbin/mysqld --bootstrap --user=mysql --skip-grant-tables";
1681
1682     } else {
1683         $cmd = "/usr/sbin/mysqld --bootstrap --user=mysql --skip-grant-tables " .
1684             "--skip-bdb  --skip-innodb --skip-ndbcluster";
1685     }
1686
1687     $self->ve_command ($cmd, $sql);
1688 }
1689
1690 sub compute_required {
1691     my ($self, $pkglist) = @_;
1692
1693     my $pkginfo = $self->pkginfo();
1694     my $instpkgs = $self->read_installed ();
1695
1696     my $closure = {};
1697     __record_provides ($pkginfo, $closure, [keys %$instpkgs]);
1698
1699     return $self->closure ($closure, $pkglist);
1700 }
1701
1702 sub task_postgres {
1703     my ($self, $opts) = @_;
1704
1705     my @supp = ('7.4', '8.1');
1706     my $pgversion = '8.1';
1707
1708     my $suite = $self->{config}->{suite};
1709
1710     if ($suite eq 'lenny' || $suite eq 'hardy' || $suite eq 'intrepid' || $suite eq 'jaunty') {
1711         @supp = ('8.3');
1712         $pgversion = '8.3';
1713     } elsif ($suite eq 'squeeze') {
1714         @supp = ('8.4');
1715         $pgversion = '8.4';
1716     } elsif ($suite eq 'wheezy') {
1717         @supp = ('9.1');
1718         $pgversion = '9.1';
1719     } elsif ($suite eq 'jessie') {
1720         @supp = ('9.4');
1721         $pgversion = '9.4';
1722     } elsif ($suite eq 'stretch') {
1723         @supp = ('9.6');
1724         $pgversion = '9.6';
1725     } elsif ($suite eq 'buster') {
1726         @supp = ('11');
1727         $pgversion = '11';
1728     } elsif ($suite eq 'bullseye') {
1729         @supp = ('13');
1730         $pgversion = '13';
1731     } elsif ($suite eq 'bookworm') {
1732         # FIXME: update once froozen
1733         @supp = ('13');
1734         $pgversion = '13';
1735     }
1736
1737     $pgversion = $opts->{version} if $opts->{version};
1738
1739     die "unsupported postgres version '$pgversion'\n"
1740         if !grep { $pgversion eq $_; } @supp;
1741
1742     my $rootdir = $self->{rootfs};
1743
1744     my $required = $self->compute_required (["postgresql-$pgversion"]);
1745
1746     $self->cache_packages ($required);
1747
1748     $self->ve_dpkg ('install', @$required);
1749
1750     my $iscript = "postgresql-$pgversion";
1751     if ($suite eq 'squeeze' || $suite eq 'wheezy' || $suite eq 'jessie' ||
1752         $suite eq 'stretch') {
1753         $iscript = 'postgresql';
1754     }
1755
1756     $self->ve_command ("/etc/init.d/$iscript start") if $opts->{start};
1757 }
1758
1759 sub task_mysql {
1760     my ($self, $opts) = @_;
1761
1762     my $password = $opts->{password};
1763     my $rootdir = $self->{rootfs};
1764
1765     my $suite = $self->{config}->{suite};
1766
1767     my $ver = '5.0';
1768     if ($suite eq 'squeeze') {
1769       $ver = '5.1';
1770     } elsif ($suite eq 'wheezy' || $suite eq 'jessie') {
1771       $ver = '5.5';
1772     } else {
1773         die "task_mysql: unsupported suite '$suite'";
1774     }
1775
1776     my $required = $self->compute_required (['mysql-common', "mysql-server-$ver"]);
1777
1778     $self->cache_packages ($required);
1779
1780     $self->ve_dpkg ('install', @$required);
1781
1782     # fix security (see /usr/bin/mysql_secure_installation)
1783     my $sql = "DELETE FROM mysql.user WHERE User='';\n" .
1784         "DELETE FROM mysql.user WHERE User='root' AND Host!='localhost';\n" .
1785         "FLUSH PRIVILEGES;\n";
1786     $self->ve_mysql_bootstrap ($sql);
1787
1788     if ($password) {
1789
1790         my $rpw = $password eq 'random' ? 'admin' : $password;
1791
1792         my $sql = "USE mysql;\n" .
1793             "UPDATE user SET password=PASSWORD(\"$rpw\") WHERE user='root';\n" .
1794             "FLUSH PRIVILEGES;\n";
1795         $self->ve_mysql_bootstrap ($sql);
1796
1797         write_file ("[client]\nuser=root\npassword=\"$rpw\"\n", "$rootdir/root/.my.cnf", 0600);
1798         if ($password eq 'random') {
1799             $self->install_init_script ($script_mysql_randompw, 2, 20);
1800         }
1801     }
1802
1803     $self->ve_command ("/etc/init.d/mysql start") if $opts->{start};
1804 }
1805
1806 sub task_php {
1807     my ($self, $opts) = @_;
1808
1809     my $memlimit = $opts->{memlimit};
1810     my $rootdir = $self->{rootfs};
1811
1812     my $required = $self->compute_required ([qw (php5 php5-cli libapache2-mod-php5 php5-gd)]);
1813
1814     $self->cache_packages ($required);
1815
1816     $self->ve_dpkg ('install', @$required);
1817
1818     if ($memlimit) {
1819         $self->run_command ("sed -e 's/^\\s*memory_limit\\s*=.*;/memory_limit = ${memlimit}M;/' -i $rootdir/etc/php5/apache2/php.ini");
1820     }
1821 }
1822
1823 sub install {
1824     my ($self, $pkglist, $unpack) = @_;
1825
1826     my $required = $self->compute_required ($pkglist);
1827
1828     $self->cache_packages ($required);
1829
1830     $self->ve_dpkg ($unpack ? 'unpack' : 'install', @$required);
1831 }
1832
1833 sub cleanup {
1834     my ($self, $distclean) = @_;
1835
1836     unlink $self->{logfile};
1837     unlink "$self->{targetname}.tar";
1838     unlink "$self->{targetname}.tar.gz";
1839
1840     $self->ve_destroy ();
1841     unlink ".veid";
1842
1843     rmtree $self->{cachedir} if $distclean && !$self->{config}->{cachedir};
1844
1845     rmtree $self->{infodir};
1846
1847 }
1848
1849 1;