ceph/src/rocksdb/buckifier/buckify_rocksdb.py

   1 from __future__ import absolute_import
   2 from __future__ import division
   3 from __future__ import print_function
   4 from __future__ import unicode_literals
   5 from targets_builder import TARGETSBuilder
   6 from optparse import OptionParser
   7 import os
   8 import fnmatch
   9 import sys
  10 import tempfile
  11
  12 from util import ColorString
  13 import util
  14
  15 # tests to export as libraries for inclusion in other projects
  16 _EXPORTED_TEST_LIBS = ["env_basic_test"]
  17
  18 # Parse src.mk files as a Dictionary of
  19 # VAR_NAME => list of files
  20 def parse_src_mk(repo_path):
  21     src_mk = repo_path + "/src.mk"
  22     src_files = {}
  23     for line in open(src_mk):
  24         line = line.strip()
  25         if len(line) == 0 or line[0] == '#':
  26             continue
  27         if '=' in line:
  28             current_src = line.split('=')[0].strip()
  29             src_files[current_src] = []
  30         elif '.cc' in line:
  31             src_path = line.split('.cc')[0].strip() + '.cc'
  32             src_files[current_src].append(src_path)
  33     return src_files
  34
  35
  36 # get all .cc / .c files
  37 def get_cc_files(repo_path):
  38     cc_files = []
  39     for root, dirnames, filenames in os.walk(repo_path):
  40         root = root[(len(repo_path) + 1):]
  41         if "java" in root:
  42             # Skip java
  43             continue
  44         for filename in fnmatch.filter(filenames, '*.cc'):
  45             cc_files.append(os.path.join(root, filename))
  46         for filename in fnmatch.filter(filenames, '*.c'):
  47             cc_files.append(os.path.join(root, filename))
  48     return cc_files
  49
  50
  51 # Get tests from Makefile
  52 def get_tests(repo_path):
  53     Makefile = repo_path + "/Makefile"
  54
  55     # Dictionary TEST_NAME => IS_PARALLEL
  56     tests = {}
  57
  58     found_tests = False
  59     for line in open(Makefile):
  60         line = line.strip()
  61         if line.startswith("TESTS ="):
  62             found_tests = True
  63         elif found_tests:
  64             if line.endswith("\\"):
  65                 # remove the trailing \
  66                 line = line[:-1]
  67                 line = line.strip()
  68                 tests[line] = False
  69             else:
  70                 # we consumed all the tests
  71                 break
  72
  73     found_parallel_tests = False
  74     for line in open(Makefile):
  75         line = line.strip()
  76         if line.startswith("PARALLEL_TEST ="):
  77             found_parallel_tests = True
  78         elif found_parallel_tests:
  79             if line.endswith("\\"):
  80                 # remove the trailing \
  81                 line = line[:-1]
  82                 line = line.strip()
  83                 tests[line] = True
  84             else:
  85                 # we consumed all the parallel tests
  86                 break
  87
  88     return tests
  89
  90
  91 # Prepare TARGETS file for buck
  92 def generate_targets(repo_path):
  93     print(ColorString.info("Generating TARGETS"))
  94     # parsed src.mk file
  95     src_mk = parse_src_mk(repo_path)
  96     # get all .cc files
  97     cc_files = get_cc_files(repo_path)
  98     # get tests from Makefile
  99     tests = get_tests(repo_path)
 100
 101     if src_mk is None or cc_files is None or tests is None:
 102         return False
 103
 104     TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
 105     # rocksdb_lib
 106     TARGETS.add_library(
 107         "rocksdb_lib",
 108         src_mk["LIB_SOURCES"] +
 109         src_mk["TOOL_LIB_SOURCES"])
 110     # rocksdb_test_lib
 111     TARGETS.add_library(
 112         "rocksdb_test_lib",
 113         src_mk.get("MOCK_LIB_SOURCES", []) +
 114         src_mk.get("TEST_LIB_SOURCES", []) +
 115         src_mk.get("EXP_LIB_SOURCES", []),
 116         [":rocksdb_lib"])
 117     # rocksdb_tools_lib
 118     TARGETS.add_library(
 119         "rocksdb_tools_lib",
 120         src_mk.get("BENCH_LIB_SOURCES", []) +
 121         ["util/testutil.cc"],
 122         [":rocksdb_lib"])
 123
 124     # test for every test we found in the Makefile
 125     for test in tests:
 126         match_src = [src for src in cc_files if ("/%s.c" % test) in src]
 127         if len(match_src) == 0:
 128             print(ColorString.warning("Cannot find .cc file for %s" % test))
 129             continue
 130         elif len(match_src) > 1:
 131             print(ColorString.warning("Found more than one .cc for %s" % test))
 132             print(match_src)
 133             continue
 134
 135         assert(len(match_src) == 1)
 136         is_parallel = tests[test]
 137         TARGETS.register_test(test, match_src[0], is_parallel)
 138
 139         if test in _EXPORTED_TEST_LIBS:
 140             test_library = "%s_lib" % test
 141             TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
 142     TARGETS.flush_tests()
 143
 144     print(ColorString.info("Generated TARGETS Summary:"))
 145     print(ColorString.info("- %d libs" % TARGETS.total_lib))
 146     print(ColorString.info("- %d binarys" % TARGETS.total_bin))
 147     print(ColorString.info("- %d tests" % TARGETS.total_test))
 148     return True
 149
 150
 151 def get_rocksdb_path():
 152     # rocksdb = {script_dir}/..
 153     script_dir = os.path.dirname(sys.argv[0])
 154     script_dir = os.path.abspath(script_dir)
 155     rocksdb_path = os.path.abspath(
 156         os.path.join(script_dir, "../"))
 157
 158     return rocksdb_path
 159
 160 def exit_with_error(msg):
 161     print(ColorString.error(msg))
 162     sys.exit(1)
 163
 164
 165 def main():
 166     # Generate TARGETS file for buck
 167     ok = generate_targets(get_rocksdb_path())
 168     if not ok:
 169         exit_with_error("Failed to generate TARGETS files")
 170
 171 if __name__ == "__main__":
 172     main()