]> git.proxmox.com Git - aab.git/blob - PVE/AAB.pm
caching: cache signatures but ignore outer verify errors
[aab.git] / PVE / AAB.pm
1 package PVE::AAB;
2
3 use strict;
4 use warnings;
5
6 use File::Path;
7 use File::Copy;
8 use IO::File;
9 use IO::Select;
10 use IPC::Open2;
11 use IPC::Open3;
12 use UUID;
13 use Cwd;
14 my @BASE_PACKAGES = qw(base openssh vi nano);
15 my @BASE_EXCLUDES = qw(
16 e2fsprogs
17 jfsutils
18 linux
19 linux-firmware
20 lvm2
21 mdadm
22 netctl
23 pcmciautils
24 reiserfsprogs
25 xfsprogs
26 );
27
28 my $PKGDIR = "/var/cache/pacman/pkg";
29
30 my ($aablibdir, $fake_init);
31
32 sub setup_defaults($) {
33 my ($dir) = @_;
34 $aablibdir = $dir;
35 $fake_init = "$aablibdir/scripts/init.bash";
36 }
37
38 setup_defaults('/usr/lib/aab');
39
40 sub write_file {
41 my ($data, $file, $perm) = @_;
42
43 die "no filename" if !$file;
44 unlink $file;
45
46 my $fh = IO::File->new ($file, O_WRONLY | O_CREAT, $perm) ||
47 die "unable to open file '$file'";
48
49 print $fh $data;
50 $fh->close;
51 }
52
53 sub copy_file {
54 my ($a, $b) = @_;
55 copy($a, $b) or die "failed to copy $a => $b: $!";
56 }
57
58 sub rename_file {
59 my ($a, $b) = @_;
60 rename($a, $b) or die "failed to rename $a => $b: $!";
61 }
62
63 sub symln {
64 my ($a, $b) = @_;
65 symlink($a, $b) or die "failed to symlink $a => $b: $!";
66 }
67
68 sub logmsg {
69 my $self = shift;
70 print STDERR @_;
71 $self->writelog (@_);
72 }
73
74 sub writelog {
75 my $self = shift;
76 my $fd = $self->{logfd};
77 print $fd @_;
78 }
79
80 sub read_config {
81 my ($filename) = @_;
82
83 my $res = {};
84
85 my $fh = IO::File->new ("<$filename") || return $res;
86 my $rec = '';
87
88 while (defined (my $line = <$fh>)) {
89 next if $line =~ m/^\#/;
90 next if $line =~ m/^\s*$/;
91 $rec .= $line;
92 };
93
94 close ($fh);
95
96 chomp $rec;
97 $rec .= "\n";
98
99 while ($rec) {
100 if ($rec =~ s/^Description:\s*([^\n]*)(\n\s+.*)*$//si) {
101 $res->{headline} = $1;
102 chomp $res->{headline};
103 my $long = $2;
104 $long =~ s/^\s+/ /;
105 $res->{description} = $long;
106 chomp $res->{description};
107 } elsif ($rec =~ s/^([^:]+):\s*(.*\S)\s*\n//) {
108 my ($key, $value) = (lc ($1), $2);
109 if ($key eq 'source' || $key eq 'mirror') {
110 push @{$res->{$key}}, $value;
111 } else {
112 die "duplicate key '$key'\n" if defined ($res->{$key});
113 $res->{$key} = $value;
114 }
115 } else {
116 die "unable to parse config file: $rec";
117 }
118 }
119
120 die "unable to parse config file" if $rec;
121
122 $res->{architecture} = 'amd64' if $res->{architecture} eq 'x86_64';
123
124 return $res;
125 }
126
127 sub new {
128 my ($class, $config) = @_;
129
130 $config = read_config ('aab.conf') if !$config;
131 my $version = $config->{version};
132 die "no 'version' specified\n" if !$version;
133 die "no 'section' specified\n" if !$config->{section};
134 die "no 'description' specified\n" if !$config->{headline};
135 die "no 'maintainer' specified\n" if !$config->{maintainer};
136
137 my $name = $config->{name} || die "no 'name' specified\n";
138 $name =~ m/^[a-z][0-9a-z\-\*\.]+$/ ||
139 die "illegal characters in name '$name'\n";
140
141 my $targetname;
142 if ($name =~ m/^archlinux/) {
143 $targetname = "${name}_${version}_$config->{architecture}";
144 } else {
145 $targetname = "archlinux-${name}_${version}_$config->{architecture}";
146 }
147
148 my $self = { logfile => 'logfile',
149 config => $config,
150 targetname => $targetname,
151 incl => [@BASE_PACKAGES],
152 excl => [@BASE_EXCLUDES],
153 };
154
155 $self->{logfd} = IO::File->new($self->{logfile}, O_WRONLY | O_APPEND | O_CREAT)
156 or die "unable to open log file";
157
158 bless $self, $class;
159
160 $self->__allocate_ve();
161
162 return $self;
163 }
164
165 sub __sample_config {
166 my ($self) = @_;
167
168 my $arch = $self->{config}->{architecture};
169
170 return <<"CFG";
171 lxc.arch = $arch
172 lxc.include = /usr/share/lxc/config/archlinux.common.conf
173 lxc.uts.name = localhost
174 lxc.rootfs.path = $self->{rootfs}
175 lxc.mount.entry = $self->{pkgcache} $self->{pkgdir} none bind 0 0
176 CFG
177 }
178
179 sub __allocate_ve {
180 my ($self) = @_;
181
182 my $cid;
183 if (my $fd = IO::File->new(".veid")) {
184 $cid = <$fd>;
185 chomp $cid;
186 close ($fd);
187 }
188
189
190 $self->{working_dir} = getcwd;
191 $self->{veconffile} = "$self->{working_dir}/config";
192 $self->{rootfs} = "$self->{working_dir}/rootfs";
193 $self->{pkgdir} = "$self->{working_dir}/rootfs/$PKGDIR";
194 $self->{pkgcache} = "$self->{working_dir}/pkgcache";
195 $self->{'pacman.conf'} = "$self->{working_dir}/pacman.conf";
196
197 if ($cid) {
198 $self->{veid} = $cid;
199 return $cid;
200 }
201
202 my $uuid;
203 my $uuid_str;
204 UUID::generate($uuid);
205 UUID::unparse($uuid, $uuid_str);
206 $self->{veid} = $uuid_str;
207
208 my $fd = IO::File->new (">.veid") ||
209 die "unable to write '.veid'\n";
210 print $fd "$self->{veid}\n";
211 close ($fd);
212 $self->logmsg("allocated VE $self->{veid}\n");
213 }
214
215 sub initialize {
216 my ($self) = @_;
217
218 my $config = $self->{config};
219
220 $self->{logfd} = IO::File->new($self->{logfile}, O_WRONLY | O_TRUNC | O_CREAT)
221 or die "unable to open log file";
222
223 my $cdata = $self->__sample_config();
224
225 my $fh = IO::File->new($self->{veconffile}, O_WRONLY|O_CREAT|O_EXCL) ||
226 die "unable to write lxc config file '$self->{veconffile}' - $!";
227 print $fh $cdata;
228 close ($fh);
229
230 if (!$config->{source} && !$config->{mirror}) {
231 die "no sources/mirrors specified";
232 }
233
234 $self->write_pacman_conf();
235
236 $self->logmsg("configured VE $self->{veid}\n");
237 }
238
239 sub write_pacman_conf {
240 my ($self, $config_fn, $siglevel) = @_;
241
242 my $config = $self->{config};
243
244 $config->{source} //= [];
245 $config->{mirror} //= [];
246
247 $siglevel ||= "Never";
248 $config_fn ||= $self->{'pacman.conf'};
249
250 my $servers = "Server = ".join("\nServer = ", @{$config->{source}}, @{$config->{mirror}}) ."\n";
251
252 my $fh = IO::File->new($config_fn, O_WRONLY | O_CREAT | O_EXCL)
253 or die "unable to write pacman config file $self->{'pacman.conf'} - $!";
254
255 my $arch = $config->{architecture};
256 $arch = 'x86_64' if $arch eq 'amd64';
257
258 print $fh <<"EOF";
259 [options]
260 HoldPkg = pacman glibc
261 Architecture = $arch
262 CheckSpace
263 SigLevel = $siglevel
264
265 [core]
266 $servers
267 [extra]
268 $servers
269 [community]
270 $servers
271 EOF
272
273 print $fh "[multilib]\n$servers\n" if $config->{architecture} eq 'x86_64';
274
275 close($fh);
276 }
277
278 sub ve_status {
279 my ($self) = @_;
280
281 my $veid = $self->{veid};
282
283 my $res = { running => 0 };
284
285 $res->{exist} = 1 if -d "$self->{rootfs}/usr";
286
287 my $filename = "/proc/net/unix";
288
289 # similar test is used by lcxcontainers.c: list_active_containers
290 my $fh = IO::File->new ($filename, "r");
291 return $res if !$fh;
292
293 while (defined(my $line = <$fh>)) {
294 if ($line =~ m/^[a-f0-9]+:\s\S+\s\S+\s\S+\s\S+\s\S+\s\d+\s(\S+)$/) {
295 my $path = $1;
296 if ($path =~ m!^@/\S+/$veid/command$!) {
297 $res->{running} = 1;
298 }
299 }
300 }
301 close($fh);
302
303 return $res;
304 }
305
306 sub ve_destroy {
307 my ($self) = @_;
308
309 my $veid = $self->{veid}; # fixme
310
311 my $vestat = $self->ve_status();
312 if ($vestat->{running}) {
313 $self->stop_container();
314 }
315
316 rmtree $self->{rootfs};
317 unlink $self->{veconffile};
318 }
319
320 sub ve_init {
321 my ($self) = @_;
322
323
324 my $veid = $self->{veid};
325 my $conffile = $self->{veconffile};
326
327 $self->logmsg ("initialize VE $veid\n");
328
329 my $vestat = $self->ve_status();
330 if ($vestat->{running}) {
331 $self->run_command ("lxc-stop -n $veid --rcfile $conffile --kill");
332 }
333
334 rmtree $self->{rootfs};
335 mkpath "$self->{rootfs}/dev";
336 }
337
338 sub ve_command {
339 my ($self, $cmd, $input) = @_;
340
341 my $veid = $self->{veid};
342 my $conffile = $self->{veconffile};
343
344 if (ref ($cmd) eq 'ARRAY') {
345 unshift @$cmd, 'lxc-attach', '-n', $veid, '--rcfile', $conffile,'--clear-env', '--';
346 $self->run_command ($cmd, $input);
347 } else {
348 $self->run_command ("lxc-attach -n $veid --rcfile $conffile --clear-env -- $cmd", $input);
349 }
350 }
351
352 sub ve_exec {
353 my ($self, @cmd) = @_;
354
355 my $veid = $self->{veid};
356 my $conffile = $self->{veconffile};
357
358 my $reader;
359 my $pid = open2($reader, "<&STDIN", 'lxc-attach', '-n', $veid, '--rcfile', $conffile, '--', @cmd)
360 or die "unable to exec command";
361
362 while (defined (my $line = <$reader>)) {
363 $self->logmsg ($line);
364 }
365
366 waitpid ($pid, 0);
367 my $rc = $? >> 8;
368
369 die "ve_exec failed - status $rc\n" if $rc != 0;
370 }
371
372 sub run_command {
373 my ($self, $cmd, $input, $getoutput, $noerr) = @_;
374
375 my $reader = IO::File->new();
376 my $writer = IO::File->new();
377 my $error = IO::File->new();
378
379 my $orig_pid = $$;
380
381 my $cmdstr = ref ($cmd) eq 'ARRAY' ? join (' ', @$cmd) : $cmd;
382
383 my $pid;
384 eval {
385 if (ref ($cmd) eq 'ARRAY') {
386 $pid = open3 ($writer, $reader, $error, @$cmd) || die $!;
387 } else {
388 $pid = open3 ($writer, $reader, $error, $cmdstr) || die $!;
389 }
390 };
391
392 my $err = $@;
393
394 # catch exec errors
395 if ($orig_pid != $$) {
396 $self->logmsg ("ERROR: command '$cmdstr' failed - fork failed\n");
397 POSIX::_exit (1);
398 kill ('KILL', $$);
399 }
400
401 die $err if $err;
402
403 print $writer $input if defined $input;
404 close $writer;
405
406 my $select = new IO::Select;
407 $select->add ($reader);
408 $select->add ($error);
409
410 my $res = '';
411 my $logfd = $self->{logfd};
412
413 while ($select->count) {
414 my @handles = $select->can_read ();
415
416 foreach my $h (@handles) {
417 my $buf = '';
418 my $count = sysread ($h, $buf, 4096);
419 if (!defined ($count)) {
420 waitpid ($pid, 0);
421 die "command '$cmdstr' failed: $!";
422 }
423 $select->remove ($h) if !$count;
424
425 print $logfd $buf;
426
427 $res .= $buf if $getoutput;
428 }
429 }
430
431 waitpid ($pid, 0);
432 my $ec = ($? >> 8);
433
434 die "command '$cmdstr' failed with exit code $ec\n" if $ec && !$noerr;
435
436 return wantarray ? ($res, $ec) : $res;
437 }
438
439 sub start_container {
440 my ($self) = @_;
441 my $veid = $self->{veid};
442 $self->run_command(['lxc-start', '-n', $veid, '-f', $self->{veconffile}, '/usr/bin/aab_fake_init']);
443 }
444
445 sub stop_container {
446 my ($self) = @_;
447 my $veid = $self->{veid};
448 my $conffile = $self->{veconffile};
449 $self->run_command ("lxc-stop -n $veid --rcfile $conffile --kill");
450 }
451
452 sub pacman_command {
453 my ($self, $config_fn) = @_;
454 my $root = $self->{rootfs};
455 return (
456 '/usr/bin/pacman',
457 '--root', $root,
458 '--config', $config_fn || $self->{'pacman.conf'},
459 '--cachedir', $self->{pkgcache},
460 '--noconfirm',
461 );
462 }
463
464 sub cache_packages {
465 my ($self, $packages) = @_;
466 my $root = $self->{rootfs};
467
468 $self->write_pacman_conf('pacman.caching.conf', "Optional");
469 my @pacman = $self->pacman_command('pacman.caching.conf');
470 my ($_res, $ec) = $self->run_command([@pacman, '-Sw', '--', @$packages], undef, undef, 1);
471 $self->logmsg("ignore bad exit $ec due to unavailable keyring, the CT will verify that later.\n")
472 if $ec;
473 }
474
475 sub mask_systemd_unit {
476 my ($self, $unit) = @_;
477 my $root = $self->{rootfs};
478 symln '/dev/null', "$root/etc/systemd/system/$unit";
479 }
480
481 sub bootstrap {
482 my ($self, $include, $exclude) = @_;
483 my $root = $self->{rootfs};
484
485 my @pacman = $self->pacman_command();
486
487 print "Fetching package database...\n";
488 mkpath $self->{pkgcache};
489 mkpath $self->{pkgdir};
490 mkpath "$root/var/lib/pacman";
491 $self->run_command([@pacman, '-Syy']);
492
493 print "Figuring out what to install...\n";
494 my $incl = { map { $_ => 1 } @{$self->{incl}} };
495 my $excl = { map { $_ => 1 } @{$self->{excl}} };
496
497 foreach my $addinc (@$include) {
498 $incl->{$addinc} = 1;
499 delete $excl->{$addinc};
500 }
501 foreach my $addexc (@$exclude) {
502 $excl->{$addexc} = 1;
503 delete $incl->{$addexc};
504 }
505
506 my $expand = sub {
507 my ($lst) = @_;
508 foreach my $inc (keys %$lst) {
509 my $group;
510 eval { $group = $self->run_command([@pacman, '-Sqg', $inc], undef, 1); };
511 if ($group && !$@) {
512 # add the group
513 delete $lst->{$inc};
514 $lst->{$_} = 1 foreach split(/\s+/, $group);
515 }
516 }
517 };
518
519 $expand->($incl);
520 $expand->($excl);
521
522 my $packages = [ grep { !$excl->{$_} } keys %$incl ];
523
524 print "Setting up basic environment...\n";
525 mkpath "$root/etc";
526 mkpath "$root/usr/bin";
527
528 my $data = "# UNCONFIGURED FSTAB FOR BASE SYSTEM\n";
529 write_file ($data, "$root/etc/fstab", 0644);
530
531 write_file ("", "$root/etc/resolv.conf", 0644);
532 write_file("localhost\n", "$root/etc/hostname", 0644);
533 $self->run_command(['install', '-m0755', $fake_init, "$root/usr/bin/aab_fake_init"]);
534
535 unlink "$root/etc/localtime";
536 symln '/usr/share/zoneinfo/UTC', "$root/etc/localtime";
537
538 print "Caching packages...\n";
539 $self->cache_packages($packages);
540 #$self->copy_packages();
541
542 print "Creating device nodes for package manager...\n";
543 $self->create_dev();
544
545 print "Installing package manager and essentials...\n";
546 # inetutils for 'hostname' for our init
547 $self->run_command([@pacman, '-S', 'pacman', 'inetutils', 'archlinux-keyring']);
548
549 print "Setting up pacman for installation from cache...\n";
550 my $file = "$root/etc/pacman.d/mirrorlist";
551 my $backup = "${file}.aab_orig";
552 if (!-f $backup) {
553 rename_file($file, $backup);
554 write_file("Server = file://$PKGDIR\n", $file);
555 }
556
557 print "Populating keyring...\n";
558 $self->populate_keyring();
559
560 print "Removing device nodes...\n";
561 $self->cleanup_dev();
562
563 print "Starting container...\n";
564 $self->start_container();
565
566 print "Installing packages...\n";
567 $self->ve_command(['pacman', '-S', '--needed', '--noconfirm', '--', @$packages]);
568
569 print "Masking problematic systemd units...\n";
570 for my $unit (qw(sys-kernel-config.mount sys-kernel-debug.mount)) {
571 $self->mask_systemd_unit($unit);
572 }
573 }
574
575 # devices needed for gnupg to function:
576 my $devs = {
577 '/dev/null' => ['c', '1', '3'],
578 '/dev/random' => ['c', '1', '9'], # fake /dev/random (really urandom)
579 '/dev/urandom' => ['c', '1', '9'],
580 '/dev/tty' => ['c', '5', '0'],
581 };
582
583 sub cleanup_dev {
584 my ($self) = @_;
585 my $root = $self->{rootfs};
586
587 # remove temporary device files
588 unlink "${root}$_" foreach keys %$devs;
589 }
590
591 sub create_dev {
592 my ($self) = @_;
593 my $root = $self->{rootfs};
594
595 local $SIG{INT} = $SIG{TERM} = sub { $self->cleanup_dev; };
596
597 # we want to replace /dev/random, so delete devices first
598 $self->cleanup_dev();
599
600 foreach my $dev (keys %$devs) {
601 my ($type, $major, $minor) = @{$devs->{$dev}};
602 system('mknod', "${root}${dev}", $type, $major, $minor);
603 }
604 }
605
606 sub populate_keyring {
607 my ($self) = @_;
608 my $root = $self->{rootfs};
609
610 # generate weak master key and populate the keyring
611 system('unshare', '--fork', '--pid', 'chroot', "$root", 'pacman-key', '--init') == 0
612 or die "failed to initialize keyring: $?";
613 system('unshare', '--fork', '--pid', 'chroot', "$root", 'pacman-key', '--populate') == 0
614 or die "failed to populate keyring: $?";
615
616 }
617
618 sub install {
619 my ($self, $pkglist) = @_;
620
621 $self->cache_packages($pkglist);
622 $self->ve_command(['pacman', '-S', '--needed', '--noconfirm', '--', @$pkglist]);
623 }
624
625 sub write_config {
626 my ($self, $filename, $size) = @_;
627
628 my $config = $self->{config};
629
630 my $data = '';
631
632 $data .= "Name: $config->{name}\n";
633 $data .= "Version: $config->{version}\n";
634 $data .= "Type: lxc\n";
635 $data .= "OS: archlinux\n";
636 $data .= "Section: $config->{section}\n";
637 $data .= "Maintainer: $config->{maintainer}\n";
638 $data .= "Architecture: $config->{architecture}\n";
639 $data .= "Infopage: https://www.archlinux.org\n";
640 $data .= "Installed-Size: $size\n";
641
642 # optional
643 $data .= "Infopage: $config->{infopage}\n" if $config->{infopage};
644 $data .= "ManageUrl: $config->{manageurl}\n" if $config->{manageurl};
645 $data .= "Certified: $config->{certified}\n" if $config->{certified};
646
647 # description
648 $data .= "Description: $config->{headline}\n";
649 $data .= "$config->{description}\n" if $config->{description};
650
651 write_file ($data, $filename, 0644);
652 }
653
654 sub finalize {
655 my ($self, $compressor) = @_;
656
657 my $use_zstd = 1;
658 if (defined($compressor)) {
659 if ($compressor =~ /^\s*--zstd?\s*$/) {
660 $use_zstd = 1;
661 } elsif ($compressor =~ /^\s*--(?:gz|gzip)\s*$/) {
662 $use_zstd = 0; # just boolean for now..
663 } else {
664 die "unkown compressor '$compressor'!\n";
665 }
666 }
667
668 my $rootdir = $self->{rootfs};
669
670 print "Stopping container...\n";
671 $self->stop_container();
672
673 print "Rolling back mirrorlist changes...\n";
674 my $file = "$rootdir/etc/pacman.d/mirrorlist";
675 unlink $file;
676 rename_file($file.'.aab_orig', $file);
677
678 print "Removing weak temporary pacman keyring...\n";
679 rmtree("$rootdir/etc/pacman.d/gnupg");
680
681 my $sizestr = $self->run_command("du -sm $rootdir", undef, 1);
682 my $size;
683 if ($sizestr =~ m/^(\d+)\s+\Q$rootdir\E$/) {
684 $size = $1;
685 } else {
686 die "unable to detect size\n";
687 }
688 $self->logmsg ("uncompressed size: $size MB\n");
689
690 $self->write_config ("$rootdir/etc/appliance.info", $size);
691
692 $self->logmsg ("creating final appliance archive\n");
693
694 my $compressor_ext = $use_zstd ? 'zst' : 'gz';
695
696 my $target = "$self->{targetname}.tar";
697 unlink $target;
698 unlink "$target.$compressor_ext";
699
700 $self->run_command ("tar cpf $target --numeric-owner -C '$rootdir' ./etc/appliance.info");
701 $self->run_command ("tar rpf $target --numeric-owner -C '$rootdir' --exclude ./etc/appliance.info .");
702
703 $self->logmsg ("compressing archive ($compressor_ext)\n");
704 if ($use_zstd) {
705 $self->run_command ("zstd -19 --rm $target");
706 } else {
707 $self->run_command ("gzip -9 $target");
708 }
709
710 my $target_size = int(-s "$target.$compressor_ext") >> 20;
711 $self->logmsg ("created '$target.$compressor_ext' with size: $target_size MB\n");
712 }
713
714 sub enter {
715 my ($self) = @_;
716 my $veid = $self->{veid};
717 my $conffile = $self->{veconffile};
718
719 my $vestat = $self->ve_status();
720 if (!$vestat->{exist}) {
721 $self->logmsg ("Please create the appliance first (bootstrap)");
722 return;
723 }
724
725 if (!$vestat->{running}) {
726 $self->start_container();
727 }
728
729 system ("lxc-attach -n $veid --rcfile $conffile --clear-env");
730 }
731
732 sub clean {
733 my ($self, $all) = @_;
734
735 unlink $self->{logfile};
736 unlink $self->{'pacman.conf'}, 'pacman.caching.conf';
737 $self->ve_destroy();
738 unlink '.veid';
739 unlink $self->{veconffile};
740
741 rmtree $self->{pkgcache} if $all;
742 }
743
744 1;