ceph/src/rocksdb/arcanist_util/config/RocksDBCommonHelper.php

   1 <?php
   2 // Copyright 2004-present Facebook. All Rights Reserved.
   3 // This source code is licensed under the BSD-style license found in the
   4 // LICENSE file in the root directory of this source tree. An additional grant
   5 // of patent rights can be found in the PATENTS file in the same directory.
   6
   7 // Name of the environment variables which need to be set by the entity which
   8 // triggers continuous runs so that code at the end of the file gets executed
   9 // and Sandcastle run starts.
  10 define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
  11 define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
  12 define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
  13
  14 define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
  15 define("CONT_RUN_ALIAS", "leveldb");
  16
  17 //////////////////////////////////////////////////////////////////////
  18 /*  Run tests in sandcastle */
  19 function postURL($diffID, $url) {
  20   assert(strlen($diffID) > 0);
  21   assert(is_numeric($diffID));
  22   assert(strlen($url) > 0);
  23
  24   $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
  25          . '"name":"click here for sandcastle tests for D' . $diffID . '", '
  26          . '"link":"' . $url . '"}\' | '
  27          . 'arc call-conduit '
  28          . 'differential.updateunitresults';
  29   shell_exec($cmd);
  30 }
  31
  32 function buildUpdateTestStatusCmd($diffID, $test, $status) {
  33   assert(strlen($diffID) > 0);
  34   assert(is_numeric($diffID));
  35   assert(strlen($test) > 0);
  36   assert(strlen($status) > 0);
  37
  38   $cmd = 'echo \'{"diff_id": ' . $diffID . ', '
  39          . '"name":"' . $test . '", '
  40          . '"result":"' . $status . '"}\' | '
  41          . 'arc call-conduit '
  42          . 'differential.updateunitresults';
  43   return $cmd;
  44 }
  45
  46 function updateTestStatus($diffID, $test) {
  47   assert(strlen($diffID) > 0);
  48   assert(is_numeric($diffID));
  49   assert(strlen($test) > 0);
  50
  51   shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
  52 }
  53
  54 function getSteps($applyDiff, $diffID, $username, $test) {
  55   assert(strlen($username) > 0);
  56   assert(strlen($test) > 0);
  57
  58   if ($applyDiff) {
  59     assert(strlen($diffID) > 0);
  60     assert(is_numeric($diffID));
  61
  62     $arcrc_content = (PHP_OS == "Darwin" ?
  63         exec("cat ~/.arcrc | gzip -f | base64") :
  64             exec("cat ~/.arcrc | gzip -f | base64 -w0"));
  65     assert(strlen($arcrc_content) > 0);
  66
  67     // Sandcastle machines don't have arc setup. We copy the user certificate
  68     // and authenticate using that in Sandcastle.
  69     $setup = array(
  70       "name" => "Setup arcrc",
  71       "shell" => "echo " . $arcrc_content . " | base64 --decode"
  72                  . " | gzip -d > ~/.arcrc",
  73       "user" => "root"
  74     );
  75
  76     // arc demands certain permission on its config.
  77     // also fix the sticky bit issue in sandcastle
  78     $fix_permission = array(
  79       "name" => "Fix environment",
  80       "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
  81       "user" => "root"
  82     );
  83
  84     // Construct the steps in the order of execution.
  85     $steps[] = $setup;
  86     $steps[] = $fix_permission;
  87   }
  88
  89   // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
  90   // Git thinks it is an uncommited change.
  91   $fix_git_ignore = array(
  92     "name" => "Fix git ignore",
  93     "shell" => "echo fbcode >> .git/info/exclude",
  94     "user" => "root"
  95   );
  96
  97   $steps[] = $fix_git_ignore;
  98
  99   // This will be the command used to execute particular type of tests.
 100   $cmd = "";
 101
 102   if ($applyDiff) {
 103     // Patch the code (keep your fingures crossed).
 104     $patch = array(
 105       "name" => "Patch " . $diffID,
 106       "shell" => "arc --arcrc-file ~/.arcrc "
 107                   . "patch --nocommit --diff " . $diffID,
 108       "user" => "root"
 109     );
 110
 111     $steps[] = $patch;
 112
 113     updateTestStatus($diffID, $test);
 114     $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
 115   }
 116
 117   // Run the actual command.
 118   $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " . $test
 119            . "; exit_code=$?; ";
 120
 121   if ($applyDiff) {
 122     $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
 123                 . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
 124                 . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
 125                 . "; ";
 126   }
 127
 128   // shell command to sort the tests based on exit code and print
 129   // the output of the log files.
 130   $cat_sorted_logs = "
 131     while read code log_file;
 132       do echo \"################ cat \$log_file [exit_code : \$code] ################\";
 133       cat \$log_file;
 134     done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
 135
 136   // Shell command to cat all log files
 137   $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
 138
 139   // If LOG file exist use it to cat log files sorted by exit code, otherwise
 140   // cat everything
 141   $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
 142
 143   $cmd = $cmd . " cat /tmp/precommit-check.log"
 144               . "; shopt -s extglob; {$logs_cmd}"
 145               . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
 146   assert(strlen($cmd) > 0);
 147
 148   $run_test = array(
 149     "name" => "Run " . $test,
 150     "shell" => $cmd,
 151     "user" => "root",
 152     "parser" => "python build_tools/error_filter.py " . $test,
 153   );
 154
 155   $steps[] = $run_test;
 156
 157   if ($applyDiff) {
 158     // Clean up the user arc config we are using.
 159     $cleanup = array(
 160       "name" => "Arc cleanup",
 161       "shell" => "rm -f ~/.arcrc",
 162       "user" => "root"
 163     );
 164
 165     $steps[] = $cleanup;
 166   }
 167
 168   assert(count($steps) > 0);
 169   return $steps;
 170 }
 171
 172 function getSandcastleConfig() {
 173   $sandcastle_config = array();
 174
 175   $cwd = getcwd();
 176   $cwd_token_file = "{$cwd}/.sandcastle";
 177   // This is a case when we're executed from a continuous run. Fetch the values
 178   // from the environment.
 179   if (getenv(ENV_POST_RECEIVE_HOOK)) {
 180     $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
 181     $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
 182   } else {
 183     // This is a typical `[p]arc diff` case. Fetch the values from the specific
 184     // configuration files.
 185     for ($i = 0; $i < 50; $i++) {
 186       if (file_exists(PRIMARY_TOKEN_FILE) ||
 187           file_exists($cwd_token_file)) {
 188         break;
 189       }
 190       // If we failed to fetch the tokens, sleep for 0.2 second and try again
 191       usleep(200000);
 192     }
 193     assert(file_exists(PRIMARY_TOKEN_FILE) ||
 194            file_exists($cwd_token_file));
 195
 196     // Try the primary location first, followed by a secondary.
 197     if (file_exists(PRIMARY_TOKEN_FILE)) {
 198       $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
 199     } else {
 200       $cmd = 'cat ' . $cwd_token_file;
 201     }
 202
 203     assert(strlen($cmd) > 0);
 204     $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
 205   }
 206
 207   // In this case be very explicit about the implications.
 208   if (count($sandcastle_config) != 2) {
 209     echo "Sandcastle configuration files don't contain valid information " .
 210          "or the necessary environment variables aren't defined. Unable " .
 211          "to validate the code changes.";
 212     exit(1);
 213   }
 214
 215   assert(strlen($sandcastle_config[0]) > 0);
 216   assert(strlen($sandcastle_config[1]) > 0);
 217   assert(count($sandcastle_config) > 0);
 218
 219   return $sandcastle_config;
 220 }
 221
 222 // This function can be called either from `[p]arc diff` command or during
 223 // the Git post-receive hook.
 224  function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
 225   // Default options don't terminate on failure, but that's what we want. In
 226   // the current case we use assertions intentionally as "terminate on failure
 227   // invariants".
 228   assert_options(ASSERT_BAIL, true);
 229
 230   // In case of a diff we'll send notificatios to the author. Else it'll go to
 231   // the entire team because failures indicate that build quality has regressed.
 232   $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
 233   assert(strlen($username) > 0);
 234
 235   if ($applyDiff) {
 236     assert($workflow);
 237     assert(strlen($diffID) > 0);
 238     assert(is_numeric($diffID));
 239   }
 240
 241   // List of tests we want to run in Sandcastle.
 242   $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
 243                  "asan", "lite_test", "valgrind", "release", "release_481",
 244                  "clang_release", "punit", "clang_analyze", "code_cov",
 245                  "java_build", "no_compression", "unity", "ubsan");
 246
 247   $send_email_template = array(
 248     'type' => 'email',
 249     'triggers' => array('fail'),
 250     'emails' => array($username . '@fb.com'),
 251   );
 252
 253   // Construct a job definition for each test and add it to the master plan.
 254   foreach ($tests as $test) {
 255     $stepName = "RocksDB diff " . $diffID . " test " . $test;
 256
 257     if (!$applyDiff) {
 258       $stepName = "RocksDB continuous integration test " . $test;
 259     }
 260
 261     $arg[] = array(
 262       "name" => $stepName,
 263       "report" => array($send_email_template),
 264       "steps" => getSteps($applyDiff, $diffID, $username, $test)
 265     );
 266   }
 267
 268   // We cannot submit the parallel execution master plan to Sandcastle and
 269   // need supply the job plan as a determinator. So we construct a small job
 270   // that will spit out the master job plan which Sandcastle will parse and
 271   // execute. Why compress the job definitions? Otherwise we run over the max
 272   // string size.
 273   $cmd = "echo " . base64_encode(json_encode($arg))
 274          . (PHP_OS == "Darwin" ?
 275              " | gzip -f | base64" :
 276                  " | gzip -f | base64 -w0");
 277   assert(strlen($cmd) > 0);
 278
 279   $arg_encoded = shell_exec($cmd);
 280   assert(strlen($arg_encoded) > 0);
 281
 282   $runName = "Run diff " . $diffID . "for user " . $username;
 283
 284   if (!$applyDiff) {
 285     $runName = "RocksDB continuous integration build and test run";
 286   }
 287
 288   $command = array(
 289     "name" => $runName,
 290     "steps" => array()
 291   );
 292
 293   $command["steps"][] = array(
 294     "name" => "Generate determinator",
 295     "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
 296                . " | base64 --decode",
 297     "determinator" => true,
 298     "user" => "root"
 299   );
 300
 301   // Submit to Sandcastle.
 302   $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
 303
 304   $job = array(
 305     'command' => 'SandcastleUniversalCommand',
 306     'args' => $command,
 307     'capabilities' => array(
 308       'vcs' => 'rocksdb-int-git',
 309       'type' => 'lego',
 310     ),
 311     'hash' => 'origin/master',
 312     'user' => $username,
 313     'alias' => 'rocksdb-precommit',
 314     'tags' => array('rocksdb'),
 315     'description' => 'Rocksdb precommit job',
 316   );
 317
 318   // Fetch the configuration necessary to submit a successful HTTPS request.
 319   $sandcastle_config = getSandcastleConfig();
 320
 321   $app = $sandcastle_config[0];
 322   $token = $sandcastle_config[1];
 323
 324   $cmd = 'curl -s -k -F app=' . $app . ' '
 325           . '-F token=' . $token . ' -F job=\'' . json_encode($job)
 326           .'\' "' . $url . '"';
 327
 328   $output = shell_exec($cmd);
 329   assert(strlen($output) > 0);
 330
 331   // Extract Sandcastle URL from the response.
 332   preg_match('/url": "(.+)"/', $output, $sandcastle_url);
 333
 334   assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
 335   assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
 336
 337   if ($applyDiff) {
 338     echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
 339     // Ask Phabricator to display it on the diff UI.
 340     postURL($diffID, $sandcastle_url[1]);
 341   } else {
 342     echo "Continuous integration started Sandcastle tests. You can look at ";
 343     echo "the progress at:\n" . $sandcastle_url[1] . "\n";
 344   }
 345 }
 346
 347 // Continuous run cript will set the environment variable and based on that
 348 // we'll trigger the execution of tests in Sandcastle. In that case we don't
 349 // need to apply any diffs and there's no associated workflow either.
 350 if (getenv(ENV_POST_RECEIVE_HOOK)) {
 351   startTestsInSandcastle(
 352     false /* $applyDiff */,
 353     NULL /* $workflow */,
 354     NULL /* $diffID */);
 355 }