ceph/cmake/modules/BuildArrow.cmake

   1 # apache arrow and its parquet library are used in radosgw for s3 select
   2
   3 function(build_arrow)
   4   # only enable the parquet component
   5   set(arrow_CMAKE_ARGS -DARROW_PARQUET=ON)
   6
   7   # only use preinstalled dependencies for arrow, don't fetch/build any
   8   list(APPEND arrow_CMAKE_ARGS -DARROW_DEPENDENCY_SOURCE=SYSTEM)
   9
  10   # only build static version of arrow and parquet
  11   list(APPEND arrow_CMAKE_ARGS -DARROW_BUILD_SHARED=OFF)
  12   list(APPEND arrow_CMAKE_ARGS -DARROW_BUILD_STATIC=ON)
  13
  14   # arrow only supports its own bundled version of jemalloc, so can't
  15   # share the version ceph is using
  16   list(APPEND arrow_CMAKE_ARGS -DARROW_JEMALLOC=OFF)
  17
  18   # transitive dependencies
  19   list(APPEND arrow_INTERFACE_LINK_LIBRARIES thrift)
  20
  21   if (NOT WITH_SYSTEM_UTF8PROC)
  22     # forward utf8proc_ROOT from build_utf8proc()
  23     list(APPEND arrow_CMAKE_ARGS -Dutf8proc_ROOT=${utf8proc_ROOT})
  24     # non-system utf8proc is bundled as a static library
  25     list(APPEND arrow_CMAKE_ARGS -DARROW_UTF8PROC_USE_SHARED=OFF)
  26     # make sure utf8proc submodule builds first, so arrow can find its byproducts
  27     list(APPEND arrow_DEPENDS utf8proc::utf8proc)
  28   endif()
  29
  30   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_BROTLI=${HAVE_BROTLI})
  31   if (HAVE_BROTLI) # optional, off by default
  32     list(APPEND arrow_INTERFACE_LINK_LIBRARIES ${brotli_libs})
  33   endif (HAVE_BROTLI)
  34
  35   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_BZ2=OFF)
  36
  37   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_LZ4=${HAVE_LZ4})
  38   if (HAVE_LZ4) # optional, on by default
  39     list(APPEND arrow_INTERFACE_LINK_LIBRARIES LZ4::LZ4)
  40   endif (HAVE_LZ4)
  41
  42   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_SNAPPY=ON) # required
  43   list(APPEND arrow_INTERFACE_LINK_LIBRARIES snappy::snappy)
  44
  45   if(WITH_RADOSGW_ARROW_FLIGHT)
  46     message("building arrow flight; make sure grpc-plugins is installed on the system")
  47     list(APPEND arrow_CMAKE_ARGS
  48       -DARROW_FLIGHT=ON -DARROW_WITH_RE2=OFF)
  49     find_package(gRPC REQUIRED)
  50     find_package(Protobuf REQUIRED)
  51     find_package(c-ares 1.13.0 QUIET REQUIRED)
  52   endif(WITH_RADOSGW_ARROW_FLIGHT)
  53
  54   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_ZLIB=ON) # required
  55   list(APPEND arrow_INTERFACE_LINK_LIBRARIES ZLIB::ZLIB)
  56
  57   list(APPEND arrow_CMAKE_ARGS -DARROW_WITH_ZSTD=${WITH_SYSTEM_ZSTD})
  58   if (WITH_SYSTEM_ZSTD)
  59     find_package(Zstd 1.4.4 REQUIRED)
  60     list(APPEND arrow_INTERFACE_LINK_LIBRARIES Zstd::Zstd)
  61   endif (WITH_SYSTEM_ZSTD)
  62
  63   list(APPEND arrow_CMAKE_ARGS -DBOOST_ROOT=${BOOST_ROOT})
  64   list(APPEND arrow_CMAKE_ARGS -DBOOST_INCLUDEDIR=${Boost_INCLUDE_DIR})
  65   list(APPEND arrow_CMAKE_ARGS -DBOOST_LIBRARYDIR=${BOOST_LIBRARYDIR})
  66
  67   if (NOT WITH_SYSTEM_BOOST)
  68     # make sure boost submodule builds first, so arrow can find its byproducts
  69     list(APPEND arrow_DEPENDS Boost)
  70   endif()
  71
  72   # cmake doesn't properly handle arguments containing ";", such as
  73   # CMAKE_PREFIX_PATH, for which reason we'll have to use some other separator.
  74   string(REPLACE ";" "!" CMAKE_PREFIX_PATH_ALT_SEP "${CMAKE_PREFIX_PATH}")
  75   list(APPEND arrow_CMAKE_ARGS -DCMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH_ALT_SEP})
  76   if(CMAKE_TOOLCHAIN_FILE)
  77     list(APPEND arrow_CMAKE_ARGS
  78          -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
  79   endif()
  80
  81   list(APPEND arrow_CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER})
  82   list(APPEND arrow_CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER})
  83   list(APPEND arrow_CMAKE_ARGS -DCMAKE_AR=${CMAKE_AR})
  84   if(CMAKE_BUILD_TYPE AND NOT CMAKE_BUILD_TYPE STREQUAL "None")
  85     list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE})
  86   else()
  87     list(APPEND arrow_CMAKE_ARGS -DCMAKE_BUILD_TYPE=Release)
  88   endif()
  89
  90   # we use an external project and copy the sources to bin directory to ensure
  91   # that object files are built outside of the source tree.
  92   include(ExternalProject)
  93   set(arrow_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/arrow/cpp")
  94   set(arrow_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/arrow/cpp")
  95
  96   set(arrow_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow")
  97   list(APPEND arrow_CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${arrow_INSTALL_PREFIX})
  98
  99   set(arrow_INSTALL_LIBDIR "lib") # force lib so we don't have to guess between lib/lib64
 100   list(APPEND arrow_CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${arrow_INSTALL_LIBDIR})
 101   set(arrow_LIBRARY_DIR "${arrow_INSTALL_PREFIX}/${arrow_INSTALL_LIBDIR}")
 102
 103   set(arrow_LIBRARY "${arrow_LIBRARY_DIR}/libarrow.a")
 104   set(parquet_LIBRARY "${arrow_LIBRARY_DIR}/libparquet.a")
 105
 106   set(arrow_INCLUDE_DIR "${arrow_INSTALL_PREFIX}/include")
 107   # this include directory won't exist until the install step, but the
 108   # imported targets need it early for INTERFACE_INCLUDE_DIRECTORIES
 109   file(MAKE_DIRECTORY "${arrow_INCLUDE_DIR}")
 110
 111   set(arrow_BYPRODUCTS ${arrow_LIBRARY})
 112   list(APPEND arrow_BYPRODUCTS ${parquet_LIBRARY})
 113
 114   if(WITH_RADOSGW_ARROW_FLIGHT)
 115     set(arrow_flight_LIBRARY "${arrow_LIBRARY_DIR}/libarrow_flight.a")
 116     list(APPEND arrow_BYPRODUCTS ${arrow_flight_LIBRARY})
 117   endif(WITH_RADOSGW_ARROW_FLIGHT)
 118
 119   if(CMAKE_MAKE_PROGRAM MATCHES "make")
 120     # try to inherit command line arguments passed by parent "make" job
 121     set(make_cmd $(MAKE))
 122     set(install_cmd $(MAKE) install)
 123   else()
 124     set(make_cmd ${CMAKE_COMMAND} --build <BINARY_DIR>)
 125     set(install_cmd ${CMAKE_COMMAND} --build <BINARY_DIR> --target install)
 126   endif()
 127
 128   # clear the DESTDIR environment variable from debian/rules,
 129   # because it messes with the internal install paths of arrow's bundled deps
 130   set(NO_DESTDIR_COMMAND ${CMAKE_COMMAND} -E env --unset=DESTDIR)
 131
 132   ExternalProject_Add(arrow_ext
 133     SOURCE_DIR "${arrow_SOURCE_DIR}"
 134     CMAKE_ARGS ${arrow_CMAKE_ARGS}
 135     BINARY_DIR "${arrow_BINARY_DIR}"
 136     BUILD_COMMAND ${NO_DESTDIR_COMMAND} ${make_cmd}
 137     BUILD_BYPRODUCTS "${arrow_BYPRODUCTS}"
 138     INSTALL_COMMAND ${NO_DESTDIR_COMMAND} ${install_cmd}
 139     INSTALL_DIR "${arrow_INSTALL_PREFIX}"
 140     DEPENDS "${arrow_DEPENDS}"
 141     LIST_SEPARATOR !)
 142
 143   add_library(Arrow::Arrow STATIC IMPORTED)
 144   add_dependencies(Arrow::Arrow arrow_ext)
 145   set_target_properties(Arrow::Arrow PROPERTIES
 146     INTERFACE_INCLUDE_DIRECTORIES "${arrow_INCLUDE_DIR}"
 147     INTERFACE_LINK_LIBRARIES "${arrow_INTERFACE_LINK_LIBRARIES}"
 148     IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
 149     IMPORTED_LOCATION "${arrow_LIBRARY}")
 150
 151   add_library(Arrow::Parquet STATIC IMPORTED)
 152   add_dependencies(Arrow::Parquet arrow_ext)
 153   target_link_libraries(Arrow::Parquet INTERFACE Arrow::Arrow)
 154   set_target_properties(Arrow::Parquet PROPERTIES
 155     IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
 156     IMPORTED_LOCATION "${parquet_LIBRARY}")
 157
 158   if(WITH_RADOSGW_ARROW_FLIGHT)
 159     add_library(Arrow::Flight STATIC IMPORTED)
 160     add_dependencies(Arrow::Flight arrow_ext)
 161     target_link_libraries(Arrow::Flight INTERFACE Arrow::Arrow gRPC::grpc++)
 162     set_target_properties(Arrow::Flight PROPERTIES
 163       INTERFACE_INCLUDE_DIRECTORIES "${arrow_INCLUDE_DIR}" # flight is accessed via "arrow/flight"
 164       IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
 165       IMPORTED_LOCATION "${arrow_flight_LIBRARY}")
 166   endif(WITH_RADOSGW_ARROW_FLIGHT)
 167 endfunction()