tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest/\${MACHINE}";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
  40 $default{"CLEAR_LOG"}           = 0;
  41 $default{"BISECT_MANUAL"}       = 0;
  42 $default{"BISECT_SKIP"}         = 1;
  43 $default{"SUCCESS_LINE"}        = "login:";
  44 $default{"DETECT_TRIPLE_FAULT"} = 1;
  45 $default{"BOOTED_TIMEOUT"}      = 1;
  46 $default{"DIE_ON_FAILURE"}      = 1;
  47 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  48 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  49 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  50 $default{"STOP_AFTER_SUCCESS"}  = 10;
  51 $default{"STOP_AFTER_FAILURE"}  = 60;
  52 $default{"STOP_TEST_AFTER"}     = 600;
  53 $default{"LOCALVERSION"}        = "-test";
  54
  55 my $ktest_config;
  56 my $version;
  57 my $machine;
  58 my $ssh_user;
  59 my $tmpdir;
  60 my $builddir;
  61 my $outputdir;
  62 my $output_config;
  63 my $test_type;
  64 my $build_type;
  65 my $build_options;
  66 my $pre_build;
  67 my $post_build;
  68 my $pre_build_die;
  69 my $post_build_die;
  70 my $reboot_type;
  71 my $reboot_script;
  72 my $power_cycle;
  73 my $reboot;
  74 my $reboot_on_error;
  75 my $poweroff_on_error;
  76 my $die_on_failure;
  77 my $powercycle_after_reboot;
  78 my $poweroff_after_halt;
  79 my $ssh_exec;
  80 my $scp_to_target;
  81 my $power_off;
  82 my $grub_menu;
  83 my $grub_number;
  84 my $target;
  85 my $make;
  86 my $post_install;
  87 my $noclean;
  88 my $minconfig;
  89 my $start_minconfig;
  90 my $start_minconfig_defined;
  91 my $output_minconfig;
  92 my $ignore_config;
  93 my $addconfig;
  94 my $in_bisect = 0;
  95 my $bisect_bad = "";
  96 my $reverse_bisect;
  97 my $bisect_manual;
  98 my $bisect_skip;
  99 my $config_bisect_good;
 100 my $in_patchcheck = 0;
 101 my $run_test;
 102 my $redirect;
 103 my $buildlog;
 104 my $dmesg;
 105 my $monitor_fp;
 106 my $monitor_pid;
 107 my $monitor_cnt = 0;
 108 my $sleep_time;
 109 my $bisect_sleep_time;
 110 my $patchcheck_sleep_time;
 111 my $ignore_warnings;
 112 my $store_failures;
 113 my $test_name;
 114 my $timeout;
 115 my $booted_timeout;
 116 my $detect_triplefault;
 117 my $console;
 118 my $success_line;
 119 my $stop_after_success;
 120 my $stop_after_failure;
 121 my $stop_test_after;
 122 my $build_target;
 123 my $target_image;
 124 my $localversion;
 125 my $iteration = 0;
 126 my $successes = 0;
 127
 128 my %entered_configs;
 129 my %config_help;
 130 my %variable;
 131 my %force_config;
 132
 133 $config_help{"MACHINE"} = << "EOF"
 134  The machine hostname that you will test.
 135 EOF
 136     ;
 137 $config_help{"SSH_USER"} = << "EOF"
 138  The box is expected to have ssh on normal bootup, provide the user
 139   (most likely root, since you need privileged operations)
 140 EOF
 141     ;
 142 $config_help{"BUILD_DIR"} = << "EOF"
 143  The directory that contains the Linux source code (full path).
 144 EOF
 145     ;
 146 $config_help{"OUTPUT_DIR"} = << "EOF"
 147  The directory that the objects will be built (full path).
 148  (can not be same as BUILD_DIR)
 149 EOF
 150     ;
 151 $config_help{"BUILD_TARGET"} = << "EOF"
 152  The location of the compiled file to copy to the target.
 153  (relative to OUTPUT_DIR)
 154 EOF
 155     ;
 156 $config_help{"TARGET_IMAGE"} = << "EOF"
 157  The place to put your image on the test machine.
 158 EOF
 159     ;
 160 $config_help{"POWER_CYCLE"} = << "EOF"
 161  A script or command to reboot the box.
 162
 163  Here is a digital loggers power switch example
 164  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 165
 166  Here is an example to reboot a virtual box on the current host
 167  with the name "Guest".
 168  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 169 EOF
 170     ;
 171 $config_help{"CONSOLE"} = << "EOF"
 172  The script or command that reads the console
 173
 174   If you use ttywatch server, something like the following would work.
 175 CONSOLE = nc -d localhost 3001
 176
 177  For a virtual machine with guest name "Guest".
 178 CONSOLE =  virsh console Guest
 179 EOF
 180     ;
 181 $config_help{"LOCALVERSION"} = << "EOF"
 182  Required version ending to differentiate the test
 183  from other linux builds on the system.
 184 EOF
 185     ;
 186 $config_help{"REBOOT_TYPE"} = << "EOF"
 187  Way to reboot the box to the test kernel.
 188  Only valid options so far are "grub" and "script".
 189
 190  If you specify grub, it will assume grub version 1
 191  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 192  and select that target to reboot to the kernel. If this is not
 193  your setup, then specify "script" and have a command or script
 194  specified in REBOOT_SCRIPT to boot to the target.
 195
 196  The entry in /boot/grub/menu.lst must be entered in manually.
 197  The test will not modify that file.
 198 EOF
 199     ;
 200 $config_help{"GRUB_MENU"} = << "EOF"
 201  The grub title name for the test kernel to boot
 202  (Only mandatory if REBOOT_TYPE = grub)
 203
 204  Note, ktest.pl will not update the grub menu.lst, you need to
 205  manually add an option for the test. ktest.pl will search
 206  the grub menu.lst for this option to find what kernel to
 207  reboot into.
 208
 209  For example, if in the /boot/grub/menu.lst the test kernel title has:
 210  title Test Kernel
 211  kernel vmlinuz-test
 212  GRUB_MENU = Test Kernel
 213 EOF
 214     ;
 215 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 216  A script to reboot the target into the test kernel
 217  (Only mandatory if REBOOT_TYPE = script)
 218 EOF
 219     ;
 220
 221 sub read_yn {
 222     my ($prompt) = @_;
 223
 224     my $ans;
 225
 226     for (;;) {
 227         print "$prompt [Y/n] ";
 228         $ans = <STDIN>;
 229         chomp $ans;
 230         if ($ans =~ /^\s*$/) {
 231             $ans = "y";
 232         }
 233         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 234         print "Please answer either 'y' or 'n'.\n";
 235     }
 236     if ($ans !~ /^y$/i) {
 237         return 0;
 238     }
 239     return 1;
 240 }
 241
 242 sub get_ktest_config {
 243     my ($config) = @_;
 244
 245     return if (defined($opt{$config}));
 246
 247     if (defined($config_help{$config})) {
 248         print "\n";
 249         print $config_help{$config};
 250     }
 251
 252     for (;;) {
 253         print "$config = ";
 254         if (defined($default{$config})) {
 255             print "\[$default{$config}\] ";
 256         }
 257         $entered_configs{$config} = <STDIN>;
 258         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 259         if ($entered_configs{$config} =~ /^\s*$/) {
 260             if ($default{$config}) {
 261                 $entered_configs{$config} = $default{$config};
 262             } else {
 263                 print "Your answer can not be blank\n";
 264                 next;
 265             }
 266         }
 267         last;
 268     }
 269 }
 270
 271 sub get_ktest_configs {
 272     get_ktest_config("MACHINE");
 273     get_ktest_config("SSH_USER");
 274     get_ktest_config("BUILD_DIR");
 275     get_ktest_config("OUTPUT_DIR");
 276     get_ktest_config("BUILD_TARGET");
 277     get_ktest_config("TARGET_IMAGE");
 278     get_ktest_config("POWER_CYCLE");
 279     get_ktest_config("CONSOLE");
 280     get_ktest_config("LOCALVERSION");
 281
 282     my $rtype = $opt{"REBOOT_TYPE"};
 283
 284     if (!defined($rtype)) {
 285         if (!defined($opt{"GRUB_MENU"})) {
 286             get_ktest_config("REBOOT_TYPE");
 287             $rtype = $entered_configs{"REBOOT_TYPE"};
 288         } else {
 289             $rtype = "grub";
 290         }
 291     }
 292
 293     if ($rtype eq "grub") {
 294         get_ktest_config("GRUB_MENU");
 295     } else {
 296         get_ktest_config("REBOOT_SCRIPT");
 297     }
 298 }
 299
 300 sub process_variables {
 301     my ($value) = @_;
 302     my $retval = "";
 303
 304     # We want to check for '\', and it is just easier
 305     # to check the previous characet of '$' and not need
 306     # to worry if '$' is the first character. By adding
 307     # a space to $value, we can just check [^\\]\$ and
 308     # it will still work.
 309     $value = " $value";
 310
 311     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 312         my $begin = $1;
 313         my $var = $2;
 314         my $end = $3;
 315         # append beginning of value to retval
 316         $retval = "$retval$begin";
 317         if (defined($variable{$var})) {
 318             $retval = "$retval$variable{$var}";
 319         } else {
 320             # put back the origin piece.
 321             $retval = "$retval\$\{$var\}";
 322         }
 323         $value = $end;
 324     }
 325     $retval = "$retval$value";
 326
 327     # remove the space added in the beginning
 328     $retval =~ s/ //;
 329
 330     return "$retval"
 331 }
 332
 333 sub set_value {
 334     my ($lvalue, $rvalue) = @_;
 335
 336     if (defined($opt{$lvalue})) {
 337         die "Error: Option $lvalue defined more than once!\n";
 338     }
 339     if ($rvalue =~ /^\s*$/) {
 340         delete $opt{$lvalue};
 341     } else {
 342         $rvalue = process_variables($rvalue);
 343         $opt{$lvalue} = $rvalue;
 344     }
 345 }
 346
 347 sub set_variable {
 348     my ($lvalue, $rvalue) = @_;
 349
 350     if ($rvalue =~ /^\s*$/) {
 351         delete $variable{$lvalue};
 352     } else {
 353         $rvalue = process_variables($rvalue);
 354         $variable{$lvalue} = $rvalue;
 355     }
 356 }
 357
 358 sub read_config {
 359     my ($config) = @_;
 360
 361     open(IN, $config) || die "can't read file $config";
 362
 363     my $name = $config;
 364     $name =~ s,.*/(.*),$1,;
 365
 366     my $test_num = 0;
 367     my $default = 1;
 368     my $repeat = 1;
 369     my $num_tests_set = 0;
 370     my $skip = 0;
 371     my $rest;
 372     my $test_case = 0;
 373
 374     while (<IN>) {
 375
 376         # ignore blank lines and comments
 377         next if (/^\s*$/ || /\s*\#/);
 378
 379         if (/^\s*TEST_START(.*)/) {
 380
 381             $rest = $1;
 382
 383             if ($num_tests_set) {
 384                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 385             }
 386
 387             my $old_test_num = $test_num;
 388             my $old_repeat = $repeat;
 389
 390             $test_num += $repeat;
 391             $default = 0;
 392             $repeat = 1;
 393
 394             if ($rest =~ /\s+SKIP(.*)/) {
 395                 $rest = $1;
 396                 $skip = 1;
 397             } else {
 398                 $test_case = 1;
 399                 $skip = 0;
 400             }
 401
 402             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 403                 $repeat = $1;
 404                 $rest = $2;
 405                 $repeat_tests{"$test_num"} = $repeat;
 406             }
 407
 408             if ($rest =~ /\s+SKIP(.*)/) {
 409                 $rest = $1;
 410                 $skip = 1;
 411             }
 412
 413             if ($rest !~ /^\s*$/) {
 414                 die "$name: $.: Gargbage found after TEST_START\n$_";
 415             }
 416
 417             if ($skip) {
 418                 $test_num = $old_test_num;
 419                 $repeat = $old_repeat;
 420             }
 421
 422         } elsif (/^\s*DEFAULTS(.*)$/) {
 423             $default = 1;
 424
 425             $rest = $1;
 426
 427             if ($rest =~ /\s+SKIP(.*)/) {
 428                 $rest = $1;
 429                 $skip = 1;
 430             } else {
 431                 $skip = 0;
 432             }
 433
 434             if ($rest !~ /^\s*$/) {
 435                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 436             }
 437
 438         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 439
 440             next if ($skip);
 441
 442             my $lvalue = $1;
 443             my $rvalue = $2;
 444
 445             if (!$default &&
 446                 ($lvalue eq "NUM_TESTS" ||
 447                  $lvalue eq "LOG_FILE" ||
 448                  $lvalue eq "CLEAR_LOG")) {
 449                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 450             }
 451
 452             if ($lvalue eq "NUM_TESTS") {
 453                 if ($test_num) {
 454                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 455                 }
 456                 if (!$default) {
 457                     die "$name: $.: NUM_TESTS must be set in default section\n";
 458                 }
 459                 $num_tests_set = 1;
 460             }
 461
 462             if ($default || $lvalue =~ /\[\d+\]$/) {
 463                 set_value($lvalue, $rvalue);
 464             } else {
 465                 my $val = "$lvalue\[$test_num\]";
 466                 set_value($val, $rvalue);
 467
 468                 if ($repeat > 1) {
 469                     $repeats{$val} = $repeat;
 470                 }
 471             }
 472         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 473             next if ($skip);
 474
 475             my $lvalue = $1;
 476             my $rvalue = $2;
 477
 478             # process config variables.
 479             # Config variables are only active while reading the
 480             # config and can be defined anywhere. They also ignore
 481             # TEST_START and DEFAULTS, but are skipped if they are in
 482             # on of these sections that have SKIP defined.
 483             # The save variable can be
 484             # defined multiple times and the new one simply overrides
 485             # the prevous one.
 486             set_variable($lvalue, $rvalue);
 487
 488         } else {
 489             die "$name: $.: Garbage found in config\n$_";
 490         }
 491     }
 492
 493     close(IN);
 494
 495     if ($test_num) {
 496         $test_num += $repeat - 1;
 497         $opt{"NUM_TESTS"} = $test_num;
 498     }
 499
 500     # make sure we have all mandatory configs
 501     get_ktest_configs;
 502
 503     # was a test specified?
 504     if (!$test_case) {
 505         print "No test case specified.\n";
 506         print "What test case would you like to run?\n";
 507         my $ans = <STDIN>;
 508         chomp $ans;
 509         $default{"TEST_TYPE"} = $ans;
 510     }
 511
 512     # set any defaults
 513
 514     foreach my $default (keys %default) {
 515         if (!defined($opt{$default})) {
 516             $opt{$default} = $default{$default};
 517         }
 518     }
 519 }
 520
 521 sub __eval_option {
 522     my ($option, $i) = @_;
 523
 524     # Add space to evaluate the character before $
 525     $option = " $option";
 526     my $retval = "";
 527
 528     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 529         my $start = $1;
 530         my $var = $2;
 531         my $end = $3;
 532
 533         # Append beginning of line
 534         $retval = "$retval$start";
 535
 536         # If the iteration option OPT[$i] exists, then use that.
 537         # otherwise see if the default OPT (without [$i]) exists.
 538
 539         my $o = "$var\[$i\]";
 540
 541         if (defined($opt{$o})) {
 542             $o = $opt{$o};
 543             $retval = "$retval$o";
 544         } elsif (defined($opt{$var})) {
 545             $o = $opt{$var};
 546             $retval = "$retval$o";
 547         } else {
 548             $retval = "$retval\$\{$var\}";
 549         }
 550
 551         $option = $end;
 552     }
 553
 554     $retval = "$retval$option";
 555
 556     $retval =~ s/^ //;
 557
 558     return $retval;
 559 }
 560
 561 sub eval_option {
 562     my ($option, $i) = @_;
 563
 564     my $prev = "";
 565
 566     # Since an option can evaluate to another option,
 567     # keep iterating until we do not evaluate any more
 568     # options.
 569     my $r = 0;
 570     while ($prev ne $option) {
 571         # Check for recursive evaluations.
 572         # 100 deep should be more than enough.
 573         if ($r++ > 100) {
 574             die "Over 100 evaluations accurred with $option\n" .
 575                 "Check for recursive variables\n";
 576         }
 577         $prev = $option;
 578         $option = __eval_option($option, $i);
 579     }
 580
 581     return $option;
 582 }
 583
 584 sub _logit {
 585     if (defined($opt{"LOG_FILE"})) {
 586         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 587         print OUT @_;
 588         close(OUT);
 589     }
 590 }
 591
 592 sub logit {
 593     if (defined($opt{"LOG_FILE"})) {
 594         _logit @_;
 595     } else {
 596         print @_;
 597     }
 598 }
 599
 600 sub doprint {
 601     print @_;
 602     _logit @_;
 603 }
 604
 605 sub run_command;
 606
 607 sub reboot {
 608     # try to reboot normally
 609     if (run_command $reboot) {
 610         if (defined($powercycle_after_reboot)) {
 611             sleep $powercycle_after_reboot;
 612             run_command "$power_cycle";
 613         }
 614     } else {
 615         # nope? power cycle it.
 616         run_command "$power_cycle";
 617     }
 618 }
 619
 620 sub do_not_reboot {
 621     my $i = $iteration;
 622
 623     return $test_type eq "build" ||
 624         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 625         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 626 }
 627
 628 sub dodie {
 629     doprint "CRITICAL FAILURE... ", @_, "\n";
 630
 631     my $i = $iteration;
 632
 633     if ($reboot_on_error && !do_not_reboot) {
 634
 635         doprint "REBOOTING\n";
 636         reboot;
 637
 638     } elsif ($poweroff_on_error && defined($power_off)) {
 639         doprint "POWERING OFF\n";
 640         `$power_off`;
 641     }
 642
 643     if (defined($opt{"LOG_FILE"})) {
 644         print " See $opt{LOG_FILE} for more info.\n";
 645     }
 646
 647     die @_, "\n";
 648 }
 649
 650 sub open_console {
 651     my ($fp) = @_;
 652
 653     my $flags;
 654
 655     my $pid = open($fp, "$console|") or
 656         dodie "Can't open console $console";
 657
 658     $flags = fcntl($fp, F_GETFL, 0) or
 659         dodie "Can't get flags for the socket: $!";
 660     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 661         dodie "Can't set flags for the socket: $!";
 662
 663     return $pid;
 664 }
 665
 666 sub close_console {
 667     my ($fp, $pid) = @_;
 668
 669     doprint "kill child process $pid\n";
 670     kill 2, $pid;
 671
 672     print "closing!\n";
 673     close($fp);
 674 }
 675
 676 sub start_monitor {
 677     if ($monitor_cnt++) {
 678         return;
 679     }
 680     $monitor_fp = \*MONFD;
 681     $monitor_pid = open_console $monitor_fp;
 682
 683     return;
 684
 685     open(MONFD, "Stop perl from warning about single use of MONFD");
 686 }
 687
 688 sub end_monitor {
 689     if (--$monitor_cnt) {
 690         return;
 691     }
 692     close_console($monitor_fp, $monitor_pid);
 693 }
 694
 695 sub wait_for_monitor {
 696     my ($time) = @_;
 697     my $line;
 698
 699     doprint "** Wait for monitor to settle down **\n";
 700
 701     # read the monitor and wait for the system to calm down
 702     do {
 703         $line = wait_for_input($monitor_fp, $time);
 704         print "$line" if (defined($line));
 705     } while (defined($line));
 706     print "** Monitor flushed **\n";
 707 }
 708
 709 sub fail {
 710
 711         if ($die_on_failure) {
 712                 dodie @_;
 713         }
 714
 715         doprint "FAILED\n";
 716
 717         my $i = $iteration;
 718
 719         # no need to reboot for just building.
 720         if (!do_not_reboot) {
 721             doprint "REBOOTING\n";
 722             reboot;
 723             start_monitor;
 724             wait_for_monitor $sleep_time;
 725             end_monitor;
 726         }
 727
 728         my $name = "";
 729
 730         if (defined($test_name)) {
 731             $name = " ($test_name)";
 732         }
 733
 734         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 735         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 736         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
 737         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 738         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 739
 740         return 1 if (!defined($store_failures));
 741
 742         my @t = localtime;
 743         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 744                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 745
 746         my $type = $build_type;
 747         if ($type =~ /useconfig/) {
 748             $type = "useconfig";
 749         }
 750
 751         my $dir = "$machine-$test_type-$type-fail-$date";
 752         my $faildir = "$store_failures/$dir";
 753
 754         if (!-d $faildir) {
 755             mkpath($faildir) or
 756                 die "can't create $faildir";
 757         }
 758         if (-f "$output_config") {
 759             cp "$output_config", "$faildir/config" or
 760                 die "failed to copy .config";
 761         }
 762         if (-f $buildlog) {
 763             cp $buildlog, "$faildir/buildlog" or
 764                 die "failed to move $buildlog";
 765         }
 766         if (-f $dmesg) {
 767             cp $dmesg, "$faildir/dmesg" or
 768                 die "failed to move $dmesg";
 769         }
 770
 771         doprint "*** Saved info to $faildir ***\n";
 772
 773         return 1;
 774 }
 775
 776 sub run_command {
 777     my ($command) = @_;
 778     my $dolog = 0;
 779     my $dord = 0;
 780     my $pid;
 781
 782     $command =~ s/\$SSH_USER/$ssh_user/g;
 783     $command =~ s/\$MACHINE/$machine/g;
 784
 785     doprint("$command ... ");
 786
 787     $pid = open(CMD, "$command 2>&1 |") or
 788         (fail "unable to exec $command" and return 0);
 789
 790     if (defined($opt{"LOG_FILE"})) {
 791         open(LOG, ">>$opt{LOG_FILE}") or
 792             dodie "failed to write to log";
 793         $dolog = 1;
 794     }
 795
 796     if (defined($redirect)) {
 797         open (RD, ">$redirect") or
 798             dodie "failed to write to redirect $redirect";
 799         $dord = 1;
 800     }
 801
 802     while (<CMD>) {
 803         print LOG if ($dolog);
 804         print RD  if ($dord);
 805     }
 806
 807     waitpid($pid, 0);
 808     my $failed = $?;
 809
 810     close(CMD);
 811     close(LOG) if ($dolog);
 812     close(RD)  if ($dord);
 813
 814     if ($failed) {
 815         doprint "FAILED!\n";
 816     } else {
 817         doprint "SUCCESS\n";
 818     }
 819
 820     return !$failed;
 821 }
 822
 823 sub run_ssh {
 824     my ($cmd) = @_;
 825     my $cp_exec = $ssh_exec;
 826
 827     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 828     return run_command "$cp_exec";
 829 }
 830
 831 sub run_scp {
 832     my ($src, $dst) = @_;
 833     my $cp_scp = $scp_to_target;
 834
 835     $cp_scp =~ s/\$SRC_FILE/$src/g;
 836     $cp_scp =~ s/\$DST_FILE/$dst/g;
 837
 838     return run_command "$cp_scp";
 839 }
 840
 841 sub get_grub_index {
 842
 843     if ($reboot_type ne "grub") {
 844         return;
 845     }
 846     return if (defined($grub_number));
 847
 848     doprint "Find grub menu ... ";
 849     $grub_number = -1;
 850
 851     my $ssh_grub = $ssh_exec;
 852     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 853
 854     open(IN, "$ssh_grub |")
 855         or die "unable to get menu.lst";
 856
 857     while (<IN>) {
 858         if (/^\s*title\s+$grub_menu\s*$/) {
 859             $grub_number++;
 860             last;
 861         } elsif (/^\s*title\s/) {
 862             $grub_number++;
 863         }
 864     }
 865     close(IN);
 866
 867     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 868         if ($grub_number < 0);
 869     doprint "$grub_number\n";
 870 }
 871
 872 sub wait_for_input
 873 {
 874     my ($fp, $time) = @_;
 875     my $rin;
 876     my $ready;
 877     my $line;
 878     my $ch;
 879
 880     if (!defined($time)) {
 881         $time = $timeout;
 882     }
 883
 884     $rin = '';
 885     vec($rin, fileno($fp), 1) = 1;
 886     $ready = select($rin, undef, undef, $time);
 887
 888     $line = "";
 889
 890     # try to read one char at a time
 891     while (sysread $fp, $ch, 1) {
 892         $line .= $ch;
 893         last if ($ch eq "\n");
 894     }
 895
 896     if (!length($line)) {
 897         return undef;
 898     }
 899
 900     return $line;
 901 }
 902
 903 sub reboot_to {
 904     if ($reboot_type eq "grub") {
 905         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch && reboot)'";
 906         return;
 907     }
 908
 909     run_command "$reboot_script";
 910 }
 911
 912 sub get_sha1 {
 913     my ($commit) = @_;
 914
 915     doprint "git rev-list --max-count=1 $commit ... ";
 916     my $sha1 = `git rev-list --max-count=1 $commit`;
 917     my $ret = $?;
 918
 919     logit $sha1;
 920
 921     if ($ret) {
 922         doprint "FAILED\n";
 923         dodie "Failed to get git $commit";
 924     }
 925
 926     print "SUCCESS\n";
 927
 928     chomp $sha1;
 929
 930     return $sha1;
 931 }
 932
 933 sub monitor {
 934     my $booted = 0;
 935     my $bug = 0;
 936     my $skip_call_trace = 0;
 937     my $loops;
 938
 939     wait_for_monitor 5;
 940
 941     my $line;
 942     my $full_line = "";
 943
 944     open(DMESG, "> $dmesg") or
 945         die "unable to write to $dmesg";
 946
 947     reboot_to;
 948
 949     my $success_start;
 950     my $failure_start;
 951     my $monitor_start = time;
 952     my $done = 0;
 953     my $version_found = 0;
 954
 955     while (!$done) {
 956
 957         if ($bug && defined($stop_after_failure) &&
 958             $stop_after_failure >= 0) {
 959             my $time = $stop_after_failure - (time - $failure_start);
 960             $line = wait_for_input($monitor_fp, $time);
 961             if (!defined($line)) {
 962                 doprint "bug timed out after $booted_timeout seconds\n";
 963                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 964                 last;
 965             }
 966         } elsif ($booted) {
 967             $line = wait_for_input($monitor_fp, $booted_timeout);
 968             if (!defined($line)) {
 969                 my $s = $booted_timeout == 1 ? "" : "s";
 970                 doprint "Successful boot found: break after $booted_timeout second$s\n";
 971                 last;
 972             }
 973         } else {
 974             $line = wait_for_input($monitor_fp);
 975             if (!defined($line)) {
 976                 my $s = $timeout == 1 ? "" : "s";
 977                 doprint "Timed out after $timeout second$s\n";
 978                 last;
 979             }
 980         }
 981
 982         doprint $line;
 983         print DMESG $line;
 984
 985         # we are not guaranteed to get a full line
 986         $full_line .= $line;
 987
 988         if ($full_line =~ /$success_line/) {
 989             $booted = 1;
 990             $success_start = time;
 991         }
 992
 993         if ($booted && defined($stop_after_success) &&
 994             $stop_after_success >= 0) {
 995             my $now = time;
 996             if ($now - $success_start >= $stop_after_success) {
 997                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 998                 last;
 999             }
1000         }
1001
1002         if ($full_line =~ /\[ backtrace testing \]/) {
1003             $skip_call_trace = 1;
1004         }
1005
1006         if ($full_line =~ /call trace:/i) {
1007             if (!$bug && !$skip_call_trace) {
1008                 $bug = 1;
1009                 $failure_start = time;
1010             }
1011         }
1012
1013         if ($bug && defined($stop_after_failure) &&
1014             $stop_after_failure >= 0) {
1015             my $now = time;
1016             if ($now - $failure_start >= $stop_after_failure) {
1017                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1018                 last;
1019             }
1020         }
1021
1022         if ($full_line =~ /\[ end of backtrace testing \]/) {
1023             $skip_call_trace = 0;
1024         }
1025
1026         if ($full_line =~ /Kernel panic -/) {
1027             $failure_start = time;
1028             $bug = 1;
1029         }
1030
1031         # Detect triple faults by testing the banner
1032         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1033             if ($1 eq $version) {
1034                 $version_found = 1;
1035             } elsif ($version_found && $detect_triplefault) {
1036                 # We already booted into the kernel we are testing,
1037                 # but now we booted into another kernel?
1038                 # Consider this a triple fault.
1039                 doprint "Aleady booted in Linux kernel $version, but now\n";
1040                 doprint "we booted into Linux kernel $1.\n";
1041                 doprint "Assuming that this is a triple fault.\n";
1042                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1043                 last;
1044             }
1045         }
1046
1047         if ($line =~ /\n/) {
1048             $full_line = "";
1049         }
1050
1051         if ($stop_test_after > 0 && !$booted && !$bug) {
1052             if (time - $monitor_start > $stop_test_after) {
1053                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1054                 $done = 1;
1055             }
1056         }
1057     }
1058
1059     close(DMESG);
1060
1061     if ($bug) {
1062         return 0 if ($in_bisect);
1063         fail "failed - got a bug report" and return 0;
1064     }
1065
1066     if (!$booted) {
1067         return 0 if ($in_bisect);
1068         fail "failed - never got a boot prompt." and return 0;
1069     }
1070
1071     return 1;
1072 }
1073
1074 sub do_post_install {
1075
1076     return if (!defined($post_install));
1077
1078     my $cp_post_install = $post_install;
1079     $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
1080     run_command "$cp_post_install" or
1081         dodie "Failed to run post install";
1082 }
1083
1084 sub install {
1085
1086     run_scp "$outputdir/$build_target", "$target_image" or
1087         dodie "failed to copy image";
1088
1089     my $install_mods = 0;
1090
1091     # should we process modules?
1092     $install_mods = 0;
1093     open(IN, "$output_config") or dodie("Can't read config file");
1094     while (<IN>) {
1095         if (/CONFIG_MODULES(=y)?/) {
1096             $install_mods = 1 if (defined($1));
1097             last;
1098         }
1099     }
1100     close(IN);
1101
1102     if (!$install_mods) {
1103         do_post_install;
1104         doprint "No modules needed\n";
1105         return;
1106     }
1107
1108     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
1109         dodie "Failed to install modules";
1110
1111     my $modlib = "/lib/modules/$version";
1112     my $modtar = "ktest-mods.tar.bz2";
1113
1114     run_ssh "rm -rf $modlib" or
1115         dodie "failed to remove old mods: $modlib";
1116
1117     # would be nice if scp -r did not follow symbolic links
1118     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1119         dodie "making tarball";
1120
1121     run_scp "$tmpdir/$modtar", "/tmp" or
1122         dodie "failed to copy modules";
1123
1124     unlink "$tmpdir/$modtar";
1125
1126     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1127         dodie "failed to tar modules";
1128
1129     run_ssh "rm -f /tmp/$modtar";
1130
1131     do_post_install;
1132 }
1133
1134 sub get_version {
1135     # get the release name
1136     doprint "$make kernelrelease ... ";
1137     $version = `$make kernelrelease | tail -1`;
1138     chomp($version);
1139     doprint "$version\n";
1140 }
1141
1142 sub start_monitor_and_boot {
1143     get_grub_index;
1144     get_version;
1145     install;
1146
1147     start_monitor;
1148     return monitor;
1149 }
1150
1151 sub check_buildlog {
1152     my ($patch) = @_;
1153
1154     my @files = `git show $patch | diffstat -l`;
1155
1156     open(IN, "git show $patch |") or
1157         dodie "failed to show $patch";
1158     while (<IN>) {
1159         if (m,^--- a/(.*),) {
1160             chomp $1;
1161             $files[$#files] = $1;
1162         }
1163     }
1164     close(IN);
1165
1166     open(IN, $buildlog) or dodie "Can't open $buildlog";
1167     while (<IN>) {
1168         if (/^\s*(.*?):.*(warning|error)/) {
1169             my $err = $1;
1170             foreach my $file (@files) {
1171                 my $fullpath = "$builddir/$file";
1172                 if ($file eq $err || $fullpath eq $err) {
1173                     fail "$file built with warnings" and return 0;
1174                 }
1175             }
1176         }
1177     }
1178     close(IN);
1179
1180     return 1;
1181 }
1182
1183 sub apply_min_config {
1184     my $outconfig = "$output_config.new";
1185
1186     # Read the config file and remove anything that
1187     # is in the force_config hash (from minconfig and others)
1188     # then add the force config back.
1189
1190     doprint "Applying minimum configurations into $output_config.new\n";
1191
1192     open (OUT, ">$outconfig") or
1193         dodie "Can't create $outconfig";
1194
1195     if (-f $output_config) {
1196         open (IN, $output_config) or
1197             dodie "Failed to open $output_config";
1198         while (<IN>) {
1199             if (/^(# )?(CONFIG_[^\s=]*)/) {
1200                 next if (defined($force_config{$2}));
1201             }
1202             print OUT;
1203         }
1204         close IN;
1205     }
1206     foreach my $config (keys %force_config) {
1207         print OUT "$force_config{$config}\n";
1208     }
1209     close OUT;
1210
1211     run_command "mv $outconfig $output_config";
1212 }
1213
1214 sub make_oldconfig {
1215
1216     my @force_list = keys %force_config;
1217
1218     if ($#force_list >= 0) {
1219         apply_min_config;
1220     }
1221
1222     if (!run_command "$make oldnoconfig") {
1223         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1224         # try a yes '' | oldconfig
1225         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1226         run_command "yes '' | $make oldconfig" or
1227             dodie "failed make config oldconfig";
1228     }
1229 }
1230
1231 # read a config file and use this to force new configs.
1232 sub load_force_config {
1233     my ($config) = @_;
1234
1235     open(IN, $config) or
1236         dodie "failed to read $config";
1237     while (<IN>) {
1238         chomp;
1239         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1240             $force_config{$1} = $_;
1241         } elsif (/^# (CONFIG_\S*) is not set/) {
1242             $force_config{$1} = $_;
1243         }
1244     }
1245     close IN;
1246 }
1247
1248 sub build {
1249     my ($type) = @_;
1250
1251     unlink $buildlog;
1252
1253     if (defined($pre_build)) {
1254         my $ret = run_command $pre_build;
1255         if (!$ret && defined($pre_build_die) &&
1256             $pre_build_die) {
1257             dodie "failed to pre_build\n";
1258         }
1259     }
1260
1261     if ($type =~ /^useconfig:(.*)/) {
1262         run_command "cp $1 $output_config" or
1263             dodie "could not copy $1 to .config";
1264
1265         $type = "oldconfig";
1266     }
1267
1268     # old config can ask questions
1269     if ($type eq "oldconfig") {
1270         $type = "oldnoconfig";
1271
1272         # allow for empty configs
1273         run_command "touch $output_config";
1274
1275         run_command "mv $output_config $outputdir/config_temp" or
1276             dodie "moving .config";
1277
1278         if (!$noclean && !run_command "$make mrproper") {
1279             dodie "make mrproper";
1280         }
1281
1282         run_command "mv $outputdir/config_temp $output_config" or
1283             dodie "moving config_temp";
1284
1285     } elsif (!$noclean) {
1286         unlink "$output_config";
1287         run_command "$make mrproper" or
1288             dodie "make mrproper";
1289     }
1290
1291     # add something to distinguish this build
1292     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1293     print OUT "$localversion\n";
1294     close(OUT);
1295
1296     if (defined($minconfig)) {
1297         load_force_config($minconfig);
1298     }
1299
1300     if ($type ne "oldnoconfig") {
1301         run_command "$make $type" or
1302             dodie "failed make config";
1303     }
1304     # Run old config regardless, to enforce min configurations
1305     make_oldconfig;
1306
1307     $redirect = "$buildlog";
1308     my $build_ret = run_command "$make $build_options";
1309     undef $redirect;
1310
1311     if (defined($post_build)) {
1312         my $ret = run_command $post_build;
1313         if (!$ret && defined($post_build_die) &&
1314             $post_build_die) {
1315             dodie "failed to post_build\n";
1316         }
1317     }
1318
1319     if (!$build_ret) {
1320         # bisect may need this to pass
1321         return 0 if ($in_bisect);
1322         fail "failed build" and return 0;
1323     }
1324
1325     return 1;
1326 }
1327
1328 sub halt {
1329     if (!run_ssh "halt" or defined($power_off)) {
1330         if (defined($poweroff_after_halt)) {
1331             sleep $poweroff_after_halt;
1332             run_command "$power_off";
1333         }
1334     } else {
1335         # nope? the zap it!
1336         run_command "$power_off";
1337     }
1338 }
1339
1340 sub success {
1341     my ($i) = @_;
1342
1343     $successes++;
1344
1345     my $name = "";
1346
1347     if (defined($test_name)) {
1348         $name = " ($test_name)";
1349     }
1350
1351     doprint "\n\n*******************************************\n";
1352     doprint     "*******************************************\n";
1353     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
1354     doprint     "*******************************************\n";
1355     doprint     "*******************************************\n";
1356
1357     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1358         doprint "Reboot and wait $sleep_time seconds\n";
1359         reboot;
1360         start_monitor;
1361         wait_for_monitor $sleep_time;
1362         end_monitor;
1363     }
1364 }
1365
1366 sub answer_bisect {
1367     for (;;) {
1368         doprint "Pass or fail? [p/f]";
1369         my $ans = <STDIN>;
1370         chomp $ans;
1371         if ($ans eq "p" || $ans eq "P") {
1372             return 1;
1373         } elsif ($ans eq "f" || $ans eq "F") {
1374             return 0;
1375         } else {
1376             print "Please answer 'P' or 'F'\n";
1377         }
1378     }
1379 }
1380
1381 sub child_run_test {
1382     my $failed = 0;
1383
1384     # child should have no power
1385     $reboot_on_error = 0;
1386     $poweroff_on_error = 0;
1387     $die_on_failure = 1;
1388
1389     run_command $run_test or $failed = 1;
1390     exit $failed;
1391 }
1392
1393 my $child_done;
1394
1395 sub child_finished {
1396     $child_done = 1;
1397 }
1398
1399 sub do_run_test {
1400     my $child_pid;
1401     my $child_exit;
1402     my $line;
1403     my $full_line;
1404     my $bug = 0;
1405
1406     wait_for_monitor 1;
1407
1408     doprint "run test $run_test\n";
1409
1410     $child_done = 0;
1411
1412     $SIG{CHLD} = qw(child_finished);
1413
1414     $child_pid = fork;
1415
1416     child_run_test if (!$child_pid);
1417
1418     $full_line = "";
1419
1420     do {
1421         $line = wait_for_input($monitor_fp, 1);
1422         if (defined($line)) {
1423
1424             # we are not guaranteed to get a full line
1425             $full_line .= $line;
1426             doprint $line;
1427
1428             if ($full_line =~ /call trace:/i) {
1429                 $bug = 1;
1430             }
1431
1432             if ($full_line =~ /Kernel panic -/) {
1433                 $bug = 1;
1434             }
1435
1436             if ($line =~ /\n/) {
1437                 $full_line = "";
1438             }
1439         }
1440     } while (!$child_done && !$bug);
1441
1442     if ($bug) {
1443         my $failure_start = time;
1444         my $now;
1445         do {
1446             $line = wait_for_input($monitor_fp, 1);
1447             if (defined($line)) {
1448                 doprint $line;
1449             }
1450             $now = time;
1451             if ($now - $failure_start >= $stop_after_failure) {
1452                 last;
1453             }
1454         } while (defined($line));
1455
1456         doprint "Detected kernel crash!\n";
1457         # kill the child with extreme prejudice
1458         kill 9, $child_pid;
1459     }
1460
1461     waitpid $child_pid, 0;
1462     $child_exit = $?;
1463
1464     if ($bug || $child_exit) {
1465         return 0 if $in_bisect;
1466         fail "test failed" and return 0;
1467     }
1468     return 1;
1469 }
1470
1471 sub run_git_bisect {
1472     my ($command) = @_;
1473
1474     doprint "$command ... ";
1475
1476     my $output = `$command 2>&1`;
1477     my $ret = $?;
1478
1479     logit $output;
1480
1481     if ($ret) {
1482         doprint "FAILED\n";
1483         dodie "Failed to git bisect";
1484     }
1485
1486     doprint "SUCCESS\n";
1487     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1488         doprint "$1 [$2]\n";
1489     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1490         $bisect_bad = $1;
1491         doprint "Found bad commit... $1\n";
1492         return 0;
1493     } else {
1494         # we already logged it, just print it now.
1495         print $output;
1496     }
1497
1498     return 1;
1499 }
1500
1501 sub bisect_reboot {
1502     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1503     reboot;
1504     start_monitor;
1505     wait_for_monitor $bisect_sleep_time;
1506     end_monitor;
1507 }
1508
1509 # returns 1 on success, 0 on failure, -1 on skip
1510 sub run_bisect_test {
1511     my ($type, $buildtype) = @_;
1512
1513     my $failed = 0;
1514     my $result;
1515     my $output;
1516     my $ret;
1517
1518     $in_bisect = 1;
1519
1520     build $buildtype or $failed = 1;
1521
1522     if ($type ne "build") {
1523         if ($failed && $bisect_skip) {
1524             $in_bisect = 0;
1525             return -1;
1526         }
1527         dodie "Failed on build" if $failed;
1528
1529         # Now boot the box
1530         start_monitor_and_boot or $failed = 1;
1531
1532         if ($type ne "boot") {
1533             if ($failed && $bisect_skip) {
1534                 end_monitor;
1535                 bisect_reboot;
1536                 $in_bisect = 0;
1537                 return -1;
1538             }
1539             dodie "Failed on boot" if $failed;
1540
1541             do_run_test or $failed = 1;
1542         }
1543         end_monitor;
1544     }
1545
1546     if ($failed) {
1547         $result = 0;
1548     } else {
1549         $result = 1;
1550     }
1551
1552     # reboot the box to a kernel we can ssh to
1553     if ($type ne "build") {
1554         bisect_reboot;
1555     }
1556     $in_bisect = 0;
1557
1558     return $result;
1559 }
1560
1561 sub run_bisect {
1562     my ($type) = @_;
1563     my $buildtype = "oldconfig";
1564
1565     # We should have a minconfig to use?
1566     if (defined($minconfig)) {
1567         $buildtype = "useconfig:$minconfig";
1568     }
1569
1570     my $ret = run_bisect_test $type, $buildtype;
1571
1572     if ($bisect_manual) {
1573         $ret = answer_bisect;
1574     }
1575
1576     # Are we looking for where it worked, not failed?
1577     if ($reverse_bisect) {
1578         $ret = !$ret;
1579     }
1580
1581     if ($ret > 0) {
1582         return "good";
1583     } elsif ($ret == 0) {
1584         return  "bad";
1585     } elsif ($bisect_skip) {
1586         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1587         return "skip";
1588     }
1589 }
1590
1591 sub bisect {
1592     my ($i) = @_;
1593
1594     my $result;
1595
1596     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1597     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1598     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1599
1600     my $good = $opt{"BISECT_GOOD[$i]"};
1601     my $bad = $opt{"BISECT_BAD[$i]"};
1602     my $type = $opt{"BISECT_TYPE[$i]"};
1603     my $start = $opt{"BISECT_START[$i]"};
1604     my $replay = $opt{"BISECT_REPLAY[$i]"};
1605     my $start_files = $opt{"BISECT_FILES[$i]"};
1606
1607     if (defined($start_files)) {
1608         $start_files = " -- " . $start_files;
1609     } else {
1610         $start_files = "";
1611     }
1612
1613     # convert to true sha1's
1614     $good = get_sha1($good);
1615     $bad = get_sha1($bad);
1616
1617     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1618         $opt{"BISECT_REVERSE[$i]"} == 1) {
1619         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1620         $reverse_bisect = 1;
1621     } else {
1622         $reverse_bisect = 0;
1623     }
1624
1625     # Can't have a test without having a test to run
1626     if ($type eq "test" && !defined($run_test)) {
1627         $type = "boot";
1628     }
1629
1630     my $check = $opt{"BISECT_CHECK[$i]"};
1631     if (defined($check) && $check ne "0") {
1632
1633         # get current HEAD
1634         my $head = get_sha1("HEAD");
1635
1636         if ($check ne "good") {
1637             doprint "TESTING BISECT BAD [$bad]\n";
1638             run_command "git checkout $bad" or
1639                 die "Failed to checkout $bad";
1640
1641             $result = run_bisect $type;
1642
1643             if ($result ne "bad") {
1644                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1645             }
1646         }
1647
1648         if ($check ne "bad") {
1649             doprint "TESTING BISECT GOOD [$good]\n";
1650             run_command "git checkout $good" or
1651                 die "Failed to checkout $good";
1652
1653             $result = run_bisect $type;
1654
1655             if ($result ne "good") {
1656                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1657             }
1658         }
1659
1660         # checkout where we started
1661         run_command "git checkout $head" or
1662             die "Failed to checkout $head";
1663     }
1664
1665     run_command "git bisect start$start_files" or
1666         dodie "could not start bisect";
1667
1668     run_command "git bisect good $good" or
1669         dodie "could not set bisect good to $good";
1670
1671     run_git_bisect "git bisect bad $bad" or
1672         dodie "could not set bisect bad to $bad";
1673
1674     if (defined($replay)) {
1675         run_command "git bisect replay $replay" or
1676             dodie "failed to run replay";
1677     }
1678
1679     if (defined($start)) {
1680         run_command "git checkout $start" or
1681             dodie "failed to checkout $start";
1682     }
1683
1684     my $test;
1685     do {
1686         $result = run_bisect $type;
1687         $test = run_git_bisect "git bisect $result";
1688     } while ($test);
1689
1690     run_command "git bisect log" or
1691         dodie "could not capture git bisect log";
1692
1693     run_command "git bisect reset" or
1694         dodie "could not reset git bisect";
1695
1696     doprint "Bad commit was [$bisect_bad]\n";
1697
1698     success $i;
1699 }
1700
1701 my %config_ignore;
1702 my %config_set;
1703
1704 my %config_list;
1705 my %null_config;
1706
1707 my %dependency;
1708
1709 sub assign_configs {
1710     my ($hash, $config) = @_;
1711
1712     open (IN, $config)
1713         or dodie "Failed to read $config";
1714
1715     while (<IN>) {
1716         if (/^((CONFIG\S*)=.*)/) {
1717             ${$hash}{$2} = $1;
1718         }
1719     }
1720
1721     close(IN);
1722 }
1723
1724 sub process_config_ignore {
1725     my ($config) = @_;
1726
1727     assign_configs \%config_ignore, $config;
1728 }
1729
1730 sub read_current_config {
1731     my ($config_ref) = @_;
1732
1733     %{$config_ref} = ();
1734     undef %{$config_ref};
1735
1736     my @key = keys %{$config_ref};
1737     if ($#key >= 0) {
1738         print "did not delete!\n";
1739         exit;
1740     }
1741     open (IN, "$output_config");
1742
1743     while (<IN>) {
1744         if (/^(CONFIG\S+)=(.*)/) {
1745             ${$config_ref}{$1} = $2;
1746         }
1747     }
1748     close(IN);
1749 }
1750
1751 sub get_dependencies {
1752     my ($config) = @_;
1753
1754     my $arr = $dependency{$config};
1755     if (!defined($arr)) {
1756         return ();
1757     }
1758
1759     my @deps = @{$arr};
1760
1761     foreach my $dep (@{$arr}) {
1762         print "ADD DEP $dep\n";
1763         @deps = (@deps, get_dependencies $dep);
1764     }
1765
1766     return @deps;
1767 }
1768
1769 sub create_config {
1770     my @configs = @_;
1771
1772     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1773
1774     foreach my $config (@configs) {
1775         print OUT "$config_set{$config}\n";
1776         my @deps = get_dependencies $config;
1777         foreach my $dep (@deps) {
1778             print OUT "$config_set{$dep}\n";
1779         }
1780     }
1781
1782     foreach my $config (keys %config_ignore) {
1783         print OUT "$config_ignore{$config}\n";
1784     }
1785     close(OUT);
1786
1787 #    exit;
1788     make_oldconfig;
1789 }
1790
1791 sub compare_configs {
1792     my (%a, %b) = @_;
1793
1794     foreach my $item (keys %a) {
1795         if (!defined($b{$item})) {
1796             print "diff $item\n";
1797             return 1;
1798         }
1799         delete $b{$item};
1800     }
1801
1802     my @keys = keys %b;
1803     if ($#keys) {
1804         print "diff2 $keys[0]\n";
1805     }
1806     return -1 if ($#keys >= 0);
1807
1808     return 0;
1809 }
1810
1811 sub run_config_bisect_test {
1812     my ($type) = @_;
1813
1814     return run_bisect_test $type, "oldconfig";
1815 }
1816
1817 sub process_passed {
1818     my (%configs) = @_;
1819
1820     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1821     # Passed! All these configs are part of a good compile.
1822     # Add them to the min options.
1823     foreach my $config (keys %configs) {
1824         if (defined($config_list{$config})) {
1825             doprint " removing $config\n";
1826             $config_ignore{$config} = $config_list{$config};
1827             delete $config_list{$config};
1828         }
1829     }
1830     doprint "config copied to $outputdir/config_good\n";
1831     run_command "cp -f $output_config $outputdir/config_good";
1832 }
1833
1834 sub process_failed {
1835     my ($config) = @_;
1836
1837     doprint "\n\n***************************************\n";
1838     doprint "Found bad config: $config\n";
1839     doprint "***************************************\n\n";
1840 }
1841
1842 sub run_config_bisect {
1843
1844     my @start_list = keys %config_list;
1845
1846     if ($#start_list < 0) {
1847         doprint "No more configs to test!!!\n";
1848         return -1;
1849     }
1850
1851     doprint "***** RUN TEST ***\n";
1852     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1853     my $ret;
1854     my %current_config;
1855
1856     my $count = $#start_list + 1;
1857     doprint "  $count configs to test\n";
1858
1859     my $half = int($#start_list / 2);
1860
1861     do {
1862         my @tophalf = @start_list[0 .. $half];
1863
1864         create_config @tophalf;
1865         read_current_config \%current_config;
1866
1867         $count = $#tophalf + 1;
1868         doprint "Testing $count configs\n";
1869         my $found = 0;
1870         # make sure we test something
1871         foreach my $config (@tophalf) {
1872             if (defined($current_config{$config})) {
1873                 logit " $config\n";
1874                 $found = 1;
1875             }
1876         }
1877         if (!$found) {
1878             # try the other half
1879             doprint "Top half produced no set configs, trying bottom half\n";
1880             @tophalf = @start_list[$half + 1 .. $#start_list];
1881             create_config @tophalf;
1882             read_current_config \%current_config;
1883             foreach my $config (@tophalf) {
1884                 if (defined($current_config{$config})) {
1885                     logit " $config\n";
1886                     $found = 1;
1887                 }
1888             }
1889             if (!$found) {
1890                 doprint "Failed: Can't make new config with current configs\n";
1891                 foreach my $config (@start_list) {
1892                     doprint "  CONFIG: $config\n";
1893                 }
1894                 return -1;
1895             }
1896             $count = $#tophalf + 1;
1897             doprint "Testing $count configs\n";
1898         }
1899
1900         $ret = run_config_bisect_test $type;
1901         if ($bisect_manual) {
1902             $ret = answer_bisect;
1903         }
1904         if ($ret) {
1905             process_passed %current_config;
1906             return 0;
1907         }
1908
1909         doprint "This config had a failure.\n";
1910         doprint "Removing these configs that were not set in this config:\n";
1911         doprint "config copied to $outputdir/config_bad\n";
1912         run_command "cp -f $output_config $outputdir/config_bad";
1913
1914         # A config exists in this group that was bad.
1915         foreach my $config (keys %config_list) {
1916             if (!defined($current_config{$config})) {
1917                 doprint " removing $config\n";
1918                 delete $config_list{$config};
1919             }
1920         }
1921
1922         @start_list = @tophalf;
1923
1924         if ($#start_list == 0) {
1925             process_failed $start_list[0];
1926             return 1;
1927         }
1928
1929         # remove half the configs we are looking at and see if
1930         # they are good.
1931         $half = int($#start_list / 2);
1932     } while ($#start_list > 0);
1933
1934     # we found a single config, try it again unless we are running manually
1935
1936     if ($bisect_manual) {
1937         process_failed $start_list[0];
1938         return 1;
1939     }
1940
1941     my @tophalf = @start_list[0 .. 0];
1942
1943     $ret = run_config_bisect_test $type;
1944     if ($ret) {
1945         process_passed %current_config;
1946         return 0;
1947     }
1948
1949     process_failed $start_list[0];
1950     return 1;
1951 }
1952
1953 sub config_bisect {
1954     my ($i) = @_;
1955
1956     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1957
1958     my $tmpconfig = "$tmpdir/use_config";
1959
1960     if (defined($config_bisect_good)) {
1961         process_config_ignore $config_bisect_good;
1962     }
1963
1964     # Make the file with the bad config and the min config
1965     if (defined($minconfig)) {
1966         # read the min config for things to ignore
1967         run_command "cp $minconfig $tmpconfig" or
1968             dodie "failed to copy $minconfig to $tmpconfig";
1969     } else {
1970         unlink $tmpconfig;
1971     }
1972
1973     if (-f $tmpconfig) {
1974         load_force_config($tmpconfig);
1975         process_config_ignore $tmpconfig;
1976     }
1977
1978     # now process the start config
1979     run_command "cp $start_config $output_config" or
1980         dodie "failed to copy $start_config to $output_config";
1981
1982     # read directly what we want to check
1983     my %config_check;
1984     open (IN, $output_config)
1985         or dodie "faied to open $output_config";
1986
1987     while (<IN>) {
1988         if (/^((CONFIG\S*)=.*)/) {
1989             $config_check{$2} = $1;
1990         }
1991     }
1992     close(IN);
1993
1994     # Now run oldconfig with the minconfig
1995     make_oldconfig;
1996
1997     # check to see what we lost (or gained)
1998     open (IN, $output_config)
1999         or dodie "Failed to read $start_config";
2000
2001     my %removed_configs;
2002     my %added_configs;
2003
2004     while (<IN>) {
2005         if (/^((CONFIG\S*)=.*)/) {
2006             # save off all options
2007             $config_set{$2} = $1;
2008             if (defined($config_check{$2})) {
2009                 if (defined($config_ignore{$2})) {
2010                     $removed_configs{$2} = $1;
2011                 } else {
2012                     $config_list{$2} = $1;
2013                 }
2014             } elsif (!defined($config_ignore{$2})) {
2015                 $added_configs{$2} = $1;
2016                 $config_list{$2} = $1;
2017             }
2018         }
2019     }
2020     close(IN);
2021
2022     my @confs = keys %removed_configs;
2023     if ($#confs >= 0) {
2024         doprint "Configs overridden by default configs and removed from check:\n";
2025         foreach my $config (@confs) {
2026             doprint " $config\n";
2027         }
2028     }
2029     @confs = keys %added_configs;
2030     if ($#confs >= 0) {
2031         doprint "Configs appearing in make oldconfig and added:\n";
2032         foreach my $config (@confs) {
2033             doprint " $config\n";
2034         }
2035     }
2036
2037     my %config_test;
2038     my $once = 0;
2039
2040     # Sometimes kconfig does weird things. We must make sure
2041     # that the config we autocreate has everything we need
2042     # to test, otherwise we may miss testing configs, or
2043     # may not be able to create a new config.
2044     # Here we create a config with everything set.
2045     create_config (keys %config_list);
2046     read_current_config \%config_test;
2047     foreach my $config (keys %config_list) {
2048         if (!defined($config_test{$config})) {
2049             if (!$once) {
2050                 $once = 1;
2051                 doprint "Configs not produced by kconfig (will not be checked):\n";
2052             }
2053             doprint "  $config\n";
2054             delete $config_list{$config};
2055         }
2056     }
2057     my $ret;
2058     do {
2059         $ret = run_config_bisect;
2060     } while (!$ret);
2061
2062     return $ret if ($ret < 0);
2063
2064     success $i;
2065 }
2066
2067 sub patchcheck_reboot {
2068     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2069     reboot;
2070     start_monitor;
2071     wait_for_monitor $patchcheck_sleep_time;
2072     end_monitor;
2073 }
2074
2075 sub patchcheck {
2076     my ($i) = @_;
2077
2078     die "PATCHCHECK_START[$i] not defined\n"
2079         if (!defined($opt{"PATCHCHECK_START[$i]"}));
2080     die "PATCHCHECK_TYPE[$i] not defined\n"
2081         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
2082
2083     my $start = $opt{"PATCHCHECK_START[$i]"};
2084
2085     my $end = "HEAD";
2086     if (defined($opt{"PATCHCHECK_END[$i]"})) {
2087         $end = $opt{"PATCHCHECK_END[$i]"};
2088     }
2089
2090     # Get the true sha1's since we can use things like HEAD~3
2091     $start = get_sha1($start);
2092     $end = get_sha1($end);
2093
2094     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
2095
2096     # Can't have a test without having a test to run
2097     if ($type eq "test" && !defined($run_test)) {
2098         $type = "boot";
2099     }
2100
2101     open (IN, "git log --pretty=oneline $end|") or
2102         dodie "could not get git list";
2103
2104     my @list;
2105
2106     while (<IN>) {
2107         chomp;
2108         $list[$#list+1] = $_;
2109         last if (/^$start/);
2110     }
2111     close(IN);
2112
2113     if ($list[$#list] !~ /^$start/) {
2114         fail "SHA1 $start not found";
2115     }
2116
2117     # go backwards in the list
2118     @list = reverse @list;
2119
2120     my $save_clean = $noclean;
2121     my %ignored_warnings;
2122
2123     if (defined($ignore_warnings)) {
2124         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2125             $ignored_warnings{$sha1} = 1;
2126         }
2127     }
2128
2129     $in_patchcheck = 1;
2130     foreach my $item (@list) {
2131         my $sha1 = $item;
2132         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2133
2134         doprint "\nProcessing commit $item\n\n";
2135
2136         run_command "git checkout $sha1" or
2137             die "Failed to checkout $sha1";
2138
2139         # only clean on the first and last patch
2140         if ($item eq $list[0] ||
2141             $item eq $list[$#list]) {
2142             $noclean = $save_clean;
2143         } else {
2144             $noclean = 1;
2145         }
2146
2147         if (defined($minconfig)) {
2148             build "useconfig:$minconfig" or return 0;
2149         } else {
2150             # ?? no config to use?
2151             build "oldconfig" or return 0;
2152         }
2153
2154
2155         if (!defined($ignored_warnings{$sha1})) {
2156             check_buildlog $sha1 or return 0;
2157         }
2158
2159         next if ($type eq "build");
2160
2161         my $failed = 0;
2162
2163         start_monitor_and_boot or $failed = 1;
2164
2165         if (!$failed && $type ne "boot"){
2166             do_run_test or $failed = 1;
2167         }
2168         end_monitor;
2169         return 0 if ($failed);
2170
2171         patchcheck_reboot;
2172
2173     }
2174     $in_patchcheck = 0;
2175     success $i;
2176
2177     return 1;
2178 }
2179
2180 my %depends;
2181 my $iflevel = 0;
2182 my @ifdeps;
2183
2184 # prevent recursion
2185 my %read_kconfigs;
2186
2187 # taken from streamline_config.pl
2188 sub read_kconfig {
2189     my ($kconfig) = @_;
2190
2191     my $state = "NONE";
2192     my $config;
2193     my @kconfigs;
2194
2195     my $cont = 0;
2196     my $line;
2197
2198
2199     if (! -f $kconfig) {
2200         doprint "file $kconfig does not exist, skipping\n";
2201         return;
2202     }
2203
2204     open(KIN, "$kconfig")
2205         or die "Can't open $kconfig";
2206     while (<KIN>) {
2207         chomp;
2208
2209         # Make sure that lines ending with \ continue
2210         if ($cont) {
2211             $_ = $line . " " . $_;
2212         }
2213
2214         if (s/\\$//) {
2215             $cont = 1;
2216             $line = $_;
2217             next;
2218         }
2219
2220         $cont = 0;
2221
2222         # collect any Kconfig sources
2223         if (/^source\s*"(.*)"/) {
2224             $kconfigs[$#kconfigs+1] = $1;
2225         }
2226
2227         # configs found
2228         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
2229             $state = "NEW";
2230             $config = $2;
2231
2232             for (my $i = 0; $i < $iflevel; $i++) {
2233                 if ($i) {
2234                     $depends{$config} .= " " . $ifdeps[$i];
2235                 } else {
2236                     $depends{$config} = $ifdeps[$i];
2237                 }
2238                 $state = "DEP";
2239             }
2240
2241         # collect the depends for the config
2242         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
2243
2244             if (defined($depends{$1})) {
2245                 $depends{$config} .= " " . $1;
2246             } else {
2247                 $depends{$config} = $1;
2248             }
2249
2250         # Get the configs that select this config
2251         } elsif ($state ne "NONE" && /^\s*select\s+(\S+)/) {
2252             if (defined($depends{$1})) {
2253                 $depends{$1} .= " " . $config;
2254             } else {
2255                 $depends{$1} = $config;
2256             }
2257
2258         # Check for if statements
2259         } elsif (/^if\s+(.*\S)\s*$/) {
2260             my $deps = $1;
2261             # remove beginning and ending non text
2262             $deps =~ s/^[^a-zA-Z0-9_]*//;
2263             $deps =~ s/[^a-zA-Z0-9_]*$//;
2264
2265             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
2266
2267             $ifdeps[$iflevel++] = join ':', @deps;
2268
2269         } elsif (/^endif/) {
2270
2271             $iflevel-- if ($iflevel);
2272
2273         # stop on "help"
2274         } elsif (/^\s*help\s*$/) {
2275             $state = "NONE";
2276         }
2277     }
2278     close(KIN);
2279
2280     # read in any configs that were found.
2281     foreach $kconfig (@kconfigs) {
2282         if (!defined($read_kconfigs{$kconfig})) {
2283             $read_kconfigs{$kconfig} = 1;
2284             read_kconfig("$builddir/$kconfig");
2285         }
2286     }
2287 }
2288
2289 sub read_depends {
2290     # find out which arch this is by the kconfig file
2291     open (IN, $output_config)
2292         or dodie "Failed to read $output_config";
2293     my $arch;
2294     while (<IN>) {
2295         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
2296             $arch = $1;
2297             last;
2298         }
2299     }
2300     close IN;
2301
2302     if (!defined($arch)) {
2303         doprint "Could not find arch from config file\n";
2304         doprint "no dependencies used\n";
2305         return;
2306     }
2307
2308     # arch is really the subarch, we need to know
2309     # what directory to look at.
2310     if ($arch eq "i386" || $arch eq "x86_64") {
2311         $arch = "x86";
2312     } elsif ($arch =~ /^tile/) {
2313         $arch = "tile";
2314     }
2315
2316     my $kconfig = "$builddir/arch/$arch/Kconfig";
2317
2318     if (! -f $kconfig && $arch =~ /\d$/) {
2319         my $orig = $arch;
2320         # some subarchs have numbers, truncate them
2321         $arch =~ s/\d*$//;
2322         $kconfig = "$builddir/arch/$arch/Kconfig";
2323         if (! -f $kconfig) {
2324             doprint "No idea what arch dir $orig is for\n";
2325             doprint "no dependencies used\n";
2326             return;
2327         }
2328     }
2329
2330     read_kconfig($kconfig);
2331 }
2332
2333 sub read_config_list {
2334     my ($config) = @_;
2335
2336     open (IN, $config)
2337         or dodie "Failed to read $config";
2338
2339     while (<IN>) {
2340         if (/^((CONFIG\S*)=.*)/) {
2341             if (!defined($config_ignore{$2})) {
2342                 $config_list{$2} = $1;
2343             }
2344         }
2345     }
2346
2347     close(IN);
2348 }
2349
2350 sub read_output_config {
2351     my ($config) = @_;
2352
2353     assign_configs \%config_ignore, $config;
2354 }
2355
2356 sub make_new_config {
2357     my @configs = @_;
2358
2359     open (OUT, ">$output_config")
2360         or dodie "Failed to write $output_config";
2361
2362     foreach my $config (@configs) {
2363         print OUT "$config\n";
2364     }
2365     close OUT;
2366 }
2367
2368 sub get_depends {
2369     my ($dep) = @_;
2370
2371     my $kconfig = $dep;
2372     $kconfig =~ s/CONFIG_//;
2373
2374     $dep = $depends{"$kconfig"};
2375
2376     # the dep string we have saves the dependencies as they
2377     # were found, including expressions like ! && ||. We
2378     # want to split this out into just an array of configs.
2379
2380     my $valid = "A-Za-z_0-9";
2381
2382     my @configs;
2383
2384     while ($dep =~ /[$valid]/) {
2385
2386         if ($dep =~ /^[^$valid]*([$valid]+)/) {
2387             my $conf = "CONFIG_" . $1;
2388
2389             $configs[$#configs + 1] = $conf;
2390
2391             $dep =~ s/^[^$valid]*[$valid]+//;
2392         } else {
2393             die "this should never happen";
2394         }
2395     }
2396
2397     return @configs;
2398 }
2399
2400 my %min_configs;
2401 my %keep_configs;
2402 my %save_configs;
2403 my %processed_configs;
2404 my %nochange_config;
2405
2406 sub test_this_config {
2407     my ($config) = @_;
2408
2409     my $found;
2410
2411     # if we already processed this config, skip it
2412     if (defined($processed_configs{$config})) {
2413         return undef;
2414     }
2415     $processed_configs{$config} = 1;
2416
2417     # if this config failed during this round, skip it
2418     if (defined($nochange_config{$config})) {
2419         return undef;
2420     }
2421
2422     my $kconfig = $config;
2423     $kconfig =~ s/CONFIG_//;
2424
2425     # Test dependencies first
2426     if (defined($depends{"$kconfig"})) {
2427         my @parents = get_depends $config;
2428         foreach my $parent (@parents) {
2429             # if the parent is in the min config, check it first
2430             next if (!defined($min_configs{$parent}));
2431             $found = test_this_config($parent);
2432             if (defined($found)) {
2433                 return $found;
2434             }
2435         }
2436     }
2437
2438     # Remove this config from the list of configs
2439     # do a make oldnoconfig and then read the resulting
2440     # .config to make sure it is missing the config that
2441     # we had before
2442     my %configs = %min_configs;
2443     delete $configs{$config};
2444     make_new_config ((values %configs), (values %keep_configs));
2445     make_oldconfig;
2446     undef %configs;
2447     assign_configs \%configs, $output_config;
2448
2449     return $config if (!defined($configs{$config}));
2450
2451     doprint "disabling config $config did not change .config\n";
2452
2453     $nochange_config{$config} = 1;
2454
2455     return undef;
2456 }
2457
2458 sub make_min_config {
2459     my ($i) = @_;
2460
2461     if (!defined($output_minconfig)) {
2462         fail "OUTPUT_MIN_CONFIG not defined" and return;
2463     }
2464
2465     # If output_minconfig exists, and the start_minconfig
2466     # came from min_config, than ask if we should use
2467     # that instead.
2468     if (-f $output_minconfig && !$start_minconfig_defined) {
2469         print "$output_minconfig exists\n";
2470         if (read_yn " Use it as minconfig?") {
2471             $start_minconfig = $output_minconfig;
2472         }
2473     }
2474
2475     if (!defined($start_minconfig)) {
2476         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
2477     }
2478
2479     my $temp_config = "$tmpdir/temp_config";
2480
2481     # First things first. We build an allnoconfig to find
2482     # out what the defaults are that we can't touch.
2483     # Some are selections, but we really can't handle selections.
2484
2485     my $save_minconfig = $minconfig;
2486     undef $minconfig;
2487
2488     run_command "$make allnoconfig" or return 0;
2489
2490     read_depends;
2491
2492     process_config_ignore $output_config;
2493
2494     undef %save_configs;
2495     undef %min_configs;
2496
2497     if (defined($ignore_config)) {
2498         # make sure the file exists
2499         `touch $ignore_config`;
2500         assign_configs \%save_configs, $ignore_config;
2501     }
2502
2503     %keep_configs = %save_configs;
2504
2505     doprint "Load initial configs from $start_minconfig\n";
2506
2507     # Look at the current min configs, and save off all the
2508     # ones that were set via the allnoconfig
2509     assign_configs \%min_configs, $start_minconfig;
2510
2511     my @config_keys = keys %min_configs;
2512
2513     # Remove anything that was set by the make allnoconfig
2514     # we shouldn't need them as they get set for us anyway.
2515     foreach my $config (@config_keys) {
2516         # Remove anything in the ignore_config
2517         if (defined($keep_configs{$config})) {
2518             my $file = $ignore_config;
2519             $file =~ s,.*/(.*?)$,$1,;
2520             doprint "$config set by $file ... ignored\n";
2521             delete $min_configs{$config};
2522             next;
2523         }
2524         # But make sure the settings are the same. If a min config
2525         # sets a selection, we do not want to get rid of it if
2526         # it is not the same as what we have. Just move it into
2527         # the keep configs.
2528         if (defined($config_ignore{$config})) {
2529             if ($config_ignore{$config} ne $min_configs{$config}) {
2530                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
2531                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
2532                 $keep_configs{$config} = $min_configs{$config};
2533             } else {
2534                 doprint "$config set by allnoconfig ... ignored\n";
2535             }
2536             delete $min_configs{$config};
2537         }
2538     }
2539
2540     my $done = 0;
2541     my $take_two = 0;
2542
2543     while (!$done) {
2544
2545         my $config;
2546         my $found;
2547
2548         # Now disable each config one by one and do a make oldconfig
2549         # till we find a config that changes our list.
2550
2551         # Put configs that did not modify the config at the end.
2552         my @test_configs = keys %min_configs;
2553         my $reset = 1;
2554         for (my $i = 0; $i < $#test_configs; $i++) {
2555             if (!defined($nochange_config{$test_configs[0]})) {
2556                 $reset = 0;
2557                 last;
2558             }
2559             # This config didn't change the .config last time.
2560             # Place it at the end
2561             my $config = shift @test_configs;
2562             push @test_configs, $config;
2563         }
2564
2565         # if every test config has failed to modify the .config file
2566         # in the past, then reset and start over.
2567         if ($reset) {
2568             undef %nochange_config;
2569         }
2570
2571         undef %processed_configs;
2572
2573         foreach my $config (@test_configs) {
2574
2575             $found = test_this_config $config;
2576
2577             last if (defined($found));
2578
2579             # oh well, try another config
2580         }
2581
2582         if (!defined($found)) {
2583             # we could have failed due to the nochange_config hash
2584             # reset and try again
2585             if (!$take_two) {
2586                 undef %nochange_config;
2587                 $take_two = 1;
2588                 next;
2589             }
2590             doprint "No more configs found that we can disable\n";
2591             $done = 1;
2592             last;
2593         }
2594         $take_two = 0;
2595
2596         $config = $found;
2597
2598         doprint "Test with $config disabled\n";
2599
2600         # set in_bisect to keep build and monitor from dieing
2601         $in_bisect = 1;
2602
2603         my $failed = 0;
2604         build "oldconfig";
2605         start_monitor_and_boot or $failed = 1;
2606         end_monitor;
2607
2608         $in_bisect = 0;
2609
2610         if ($failed) {
2611             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
2612             # this config is needed, add it to the ignore list.
2613             $keep_configs{$config} = $min_configs{$config};
2614             $save_configs{$config} = $min_configs{$config};
2615             delete $min_configs{$config};
2616
2617             # update new ignore configs
2618             if (defined($ignore_config)) {
2619                 open (OUT, ">$temp_config")
2620                     or die "Can't write to $temp_config";
2621                 foreach my $config (keys %save_configs) {
2622                     print OUT "$save_configs{$config}\n";
2623                 }
2624                 close OUT;
2625                 run_command "mv $temp_config $ignore_config" or
2626                     dodie "failed to copy update to $ignore_config";
2627             }
2628
2629         } else {
2630             # We booted without this config, remove it from the minconfigs.
2631             doprint "$config is not needed, disabling\n";
2632
2633             delete $min_configs{$config};
2634
2635             # Also disable anything that is not enabled in this config
2636             my %configs;
2637             assign_configs \%configs, $output_config;
2638             my @config_keys = keys %min_configs;
2639             foreach my $config (@config_keys) {
2640                 if (!defined($configs{$config})) {
2641                     doprint "$config is not set, disabling\n";
2642                     delete $min_configs{$config};
2643                 }
2644             }
2645
2646             # Save off all the current mandidory configs
2647             open (OUT, ">$temp_config")
2648                 or die "Can't write to $temp_config";
2649             foreach my $config (keys %keep_configs) {
2650                 print OUT "$keep_configs{$config}\n";
2651             }
2652             foreach my $config (keys %min_configs) {
2653                 print OUT "$min_configs{$config}\n";
2654             }
2655             close OUT;
2656
2657             run_command "mv $temp_config $output_minconfig" or
2658                 dodie "failed to copy update to $output_minconfig";
2659         }
2660
2661         doprint "Reboot and wait $sleep_time seconds\n";
2662         reboot;
2663         start_monitor;
2664         wait_for_monitor $sleep_time;
2665         end_monitor;
2666     }
2667
2668     success $i;
2669     return 1;
2670 }
2671
2672 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
2673
2674 if ($#ARGV == 0) {
2675     $ktest_config = $ARGV[0];
2676     if (! -f $ktest_config) {
2677         print "$ktest_config does not exist.\n";
2678         if (!read_yn "Create it?") {
2679             exit 0;
2680         }
2681     }
2682 } else {
2683     $ktest_config = "ktest.conf";
2684 }
2685
2686 if (! -f $ktest_config) {
2687     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
2688     print OUT << "EOF"
2689 # Generated by ktest.pl
2690 #
2691 # Define each test with TEST_START
2692 # The config options below it will override the defaults
2693 TEST_START
2694
2695 DEFAULTS
2696 EOF
2697 ;
2698     close(OUT);
2699 }
2700 read_config $ktest_config;
2701
2702 if (defined($opt{"LOG_FILE"})) {
2703     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
2704 }
2705
2706 # Append any configs entered in manually to the config file.
2707 my @new_configs = keys %entered_configs;
2708 if ($#new_configs >= 0) {
2709     print "\nAppending entered in configs to $ktest_config\n";
2710     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
2711     foreach my $config (@new_configs) {
2712         print OUT "$config = $entered_configs{$config}\n";
2713         $opt{$config} = $entered_configs{$config};
2714     }
2715 }
2716
2717 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
2718     unlink $opt{"LOG_FILE"};
2719 }
2720
2721 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
2722
2723 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
2724
2725     if (!$i) {
2726         doprint "DEFAULT OPTIONS:\n";
2727     } else {
2728         doprint "\nTEST $i OPTIONS";
2729         if (defined($repeat_tests{$i})) {
2730             $repeat = $repeat_tests{$i};
2731             doprint " ITERATE $repeat";
2732         }
2733         doprint "\n";
2734     }
2735
2736     foreach my $option (sort keys %opt) {
2737
2738         if ($option =~ /\[(\d+)\]$/) {
2739             next if ($i != $1);
2740         } else {
2741             next if ($i);
2742         }
2743
2744         doprint "$option = $opt{$option}\n";
2745     }
2746 }
2747
2748 sub __set_test_option {
2749     my ($name, $i) = @_;
2750
2751     my $option = "$name\[$i\]";
2752
2753     if (defined($opt{$option})) {
2754         return $opt{$option};
2755     }
2756
2757     foreach my $test (keys %repeat_tests) {
2758         if ($i >= $test &&
2759             $i < $test + $repeat_tests{$test}) {
2760             $option = "$name\[$test\]";
2761             if (defined($opt{$option})) {
2762                 return $opt{$option};
2763             }
2764         }
2765     }
2766
2767     if (defined($opt{$name})) {
2768         return $opt{$name};
2769     }
2770
2771     return undef;
2772 }
2773
2774 sub set_test_option {
2775     my ($name, $i) = @_;
2776
2777     my $option = __set_test_option($name, $i);
2778     return $option if (!defined($option));
2779
2780     return eval_option($option, $i);
2781 }
2782
2783 # First we need to do is the builds
2784 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
2785
2786     $iteration = $i;
2787
2788     my $makecmd = set_test_option("MAKE_CMD", $i);
2789
2790     $machine = set_test_option("MACHINE", $i);
2791     $ssh_user = set_test_option("SSH_USER", $i);
2792     $tmpdir = set_test_option("TMP_DIR", $i);
2793     $outputdir = set_test_option("OUTPUT_DIR", $i);
2794     $builddir = set_test_option("BUILD_DIR", $i);
2795     $test_type = set_test_option("TEST_TYPE", $i);
2796     $build_type = set_test_option("BUILD_TYPE", $i);
2797     $build_options = set_test_option("BUILD_OPTIONS", $i);
2798     $pre_build = set_test_option("PRE_BUILD", $i);
2799     $post_build = set_test_option("POST_BUILD", $i);
2800     $pre_build_die = set_test_option("PRE_BUILD_DIE", $i);
2801     $post_build_die = set_test_option("POST_BUILD_DIE", $i);
2802     $power_cycle = set_test_option("POWER_CYCLE", $i);
2803     $reboot = set_test_option("REBOOT", $i);
2804     $noclean = set_test_option("BUILD_NOCLEAN", $i);
2805     $minconfig = set_test_option("MIN_CONFIG", $i);
2806     $output_minconfig = set_test_option("OUTPUT_MIN_CONFIG", $i);
2807     $start_minconfig = set_test_option("START_MIN_CONFIG", $i);
2808     $ignore_config = set_test_option("IGNORE_CONFIG", $i);
2809     $run_test = set_test_option("TEST", $i);
2810     $addconfig = set_test_option("ADD_CONFIG", $i);
2811     $reboot_type = set_test_option("REBOOT_TYPE", $i);
2812     $grub_menu = set_test_option("GRUB_MENU", $i);
2813     $post_install = set_test_option("POST_INSTALL", $i);
2814     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
2815     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
2816     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
2817     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
2818     $power_off = set_test_option("POWER_OFF", $i);
2819     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
2820     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
2821     $sleep_time = set_test_option("SLEEP_TIME", $i);
2822     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
2823     $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
2824     $ignore_warnings = set_test_option("IGNORE_WARNINGS", $i);
2825     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
2826     $bisect_skip = set_test_option("BISECT_SKIP", $i);
2827     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
2828     $store_failures = set_test_option("STORE_FAILURES", $i);
2829     $test_name = set_test_option("TEST_NAME", $i);
2830     $timeout = set_test_option("TIMEOUT", $i);
2831     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
2832     $console = set_test_option("CONSOLE", $i);
2833     $detect_triplefault = set_test_option("DETECT_TRIPLE_FAULT", $i);
2834     $success_line = set_test_option("SUCCESS_LINE", $i);
2835     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
2836     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
2837     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
2838     $build_target = set_test_option("BUILD_TARGET", $i);
2839     $ssh_exec = set_test_option("SSH_EXEC", $i);
2840     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
2841     $target_image = set_test_option("TARGET_IMAGE", $i);
2842     $localversion = set_test_option("LOCALVERSION", $i);
2843
2844     $start_minconfig_defined = 1;
2845
2846     if (!defined($start_minconfig)) {
2847         $start_minconfig_defined = 0;
2848         $start_minconfig = $minconfig;
2849     }
2850
2851     chdir $builddir || die "can't change directory to $builddir";
2852
2853     foreach my $dir ($tmpdir, $outputdir) {
2854         if (!-d $dir) {
2855             mkpath($dir) or
2856                 die "can't create $dir";
2857         }
2858     }
2859
2860     $ENV{"SSH_USER"} = $ssh_user;
2861     $ENV{"MACHINE"} = $machine;
2862
2863     $target = "$ssh_user\@$machine";
2864
2865     $buildlog = "$tmpdir/buildlog-$machine";
2866     $dmesg = "$tmpdir/dmesg-$machine";
2867     $make = "$makecmd O=$outputdir";
2868     $output_config = "$outputdir/.config";
2869
2870     if ($reboot_type eq "grub") {
2871         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
2872     } elsif (!defined($reboot_script)) {
2873         dodie "REBOOT_SCRIPT not defined"
2874     }
2875
2876     my $run_type = $build_type;
2877     if ($test_type eq "patchcheck") {
2878         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
2879     } elsif ($test_type eq "bisect") {
2880         $run_type = $opt{"BISECT_TYPE[$i]"};
2881     } elsif ($test_type eq "config_bisect") {
2882         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2883     }
2884
2885     if ($test_type eq "make_min_config") {
2886         $run_type = "";
2887     }
2888
2889     # mistake in config file?
2890     if (!defined($run_type)) {
2891         $run_type = "ERROR";
2892     }
2893
2894     doprint "\n\n";
2895     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2896
2897     unlink $dmesg;
2898     unlink $buildlog;
2899
2900     if (defined($addconfig)) {
2901         my $min = $minconfig;
2902         if (!defined($minconfig)) {
2903             $min = "";
2904         }
2905         run_command "cat $addconfig $min > $tmpdir/add_config" or
2906             dodie "Failed to create temp config";
2907         $minconfig = "$tmpdir/add_config";
2908     }
2909
2910     my $checkout = $opt{"CHECKOUT[$i]"};
2911     if (defined($checkout)) {
2912         run_command "git checkout $checkout" or
2913             die "failed to checkout $checkout";
2914     }
2915
2916     if ($test_type eq "bisect") {
2917         bisect $i;
2918         next;
2919     } elsif ($test_type eq "config_bisect") {
2920         config_bisect $i;
2921         next;
2922     } elsif ($test_type eq "patchcheck") {
2923         patchcheck $i;
2924         next;
2925     } elsif ($test_type eq "make_min_config") {
2926         make_min_config $i;
2927         next;
2928     }
2929
2930     if ($build_type ne "nobuild") {
2931         build $build_type or next;
2932     }
2933
2934     if ($test_type eq "install") {
2935         get_version;
2936         install;
2937         success $i;
2938         next;
2939     }
2940
2941     if ($test_type ne "build") {
2942         my $failed = 0;
2943         start_monitor_and_boot or $failed = 1;
2944
2945         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2946             do_run_test or $failed = 1;
2947         }
2948         end_monitor;
2949         next if ($failed);
2950     }
2951
2952     success $i;
2953 }
2954
2955 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2956     halt;
2957 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2958     reboot;
2959 }
2960
2961 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2962
2963 exit 0;