]> git.proxmox.com Git - ceph.git/blob - ceph/src/rocksdb/.circleci/config.yml
import quincy beta 17.1.0
[ceph.git] / ceph / src / rocksdb / .circleci / config.yml
1 version: 2.1
2
3 orbs:
4 win: circleci/windows@2.4.0
5 slack: circleci/slack@3.4.2
6
7 aliases:
8 - &notify-on-master-failure
9 fail_only: true
10 only_for_branches: master
11
12 commands:
13 install-pyenv-on-macos:
14 steps:
15 - run:
16 name: Install pyenv on macos
17 command: |
18 HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv
19
20 install-cmake-on-macos:
21 steps:
22 - run:
23 name: Install cmake on macos
24 command: |
25 HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
26
27 increase-max-open-files-on-macos:
28 steps:
29 - run:
30 name: Increase max open files
31 command: |
32 sudo sysctl -w kern.maxfiles=1048576
33 sudo sysctl -w kern.maxfilesperproc=1048576
34 sudo launchctl limit maxfiles 1048576
35
36 pre-steps:
37 parameters:
38 python-version:
39 default: "3.5.9"
40 type: string
41 steps:
42 - checkout
43 - run: pyenv install --skip-existing <<parameters.python-version>>
44 - run: pyenv global <<parameters.python-version>>
45 - run:
46 name: Setup Environment Variables
47 command: |
48 echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
49 echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
50 echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
51 echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
52
53 pre-steps-macos:
54 steps:
55 - pre-steps:
56 python-version: "3.6.0"
57
58 post-steps:
59 steps:
60 - slack/status: *notify-on-master-failure
61 - store_test_results: # store test result if there's any
62 path: /tmp/test-results
63 - store_artifacts: # store LOG for debugging if there's any
64 path: LOG
65
66 install-clang-10:
67 steps:
68 - run:
69 name: Install Clang 10
70 command: |
71 echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
72 echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
73 echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
74 sudo apt-get update -y && sudo apt-get install -y clang-10
75
76 install-gflags:
77 steps:
78 - run:
79 name: Install gflags
80 command: |
81 sudo apt-get update -y && sudo apt-get install -y libgflags-dev
82
83 install-gflags-on-macos:
84 steps:
85 - run:
86 name: Install gflags on macos
87 command: |
88 HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
89
90 install-gtest-parallel:
91 steps:
92 - run:
93 name: Install gtest-parallel
94 command: |
95 git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
96 echo "export PATH=$HOME/gtest-parallel:$PATH" >> $BASH_ENV
97
98 executors:
99 windows-2xlarge:
100 machine:
101 image: 'windows-server-2019-vs2019:stable'
102 resource_class: windows.2xlarge
103 shell: bash.exe
104
105 jobs:
106 build-macos:
107 macos:
108 xcode: 9.4.1
109 steps:
110 - increase-max-open-files-on-macos
111 - install-pyenv-on-macos
112 - install-gflags-on-macos
113 - pre-steps-macos
114 - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
115 - post-steps
116
117 build-macos-cmake:
118 macos:
119 xcode: 9.4.1
120 steps:
121 - increase-max-open-files-on-macos
122 - install-pyenv-on-macos
123 - install-cmake-on-macos
124 - install-gflags-on-macos
125 - pre-steps-macos
126 - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
127 - post-steps
128
129 build-linux:
130 machine:
131 image: ubuntu-1604:202007-01
132 resource_class: 2xlarge
133 steps:
134 - pre-steps
135 - install-gflags
136 - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
137 - post-steps
138
139 build-linux-mem-env:
140 machine:
141 image: ubuntu-1604:202007-01
142 resource_class: 2xlarge
143 steps:
144 - pre-steps
145 - install-gflags
146 - run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
147 - post-steps
148
149 build-linux-encrypted-env:
150 machine:
151 image: ubuntu-1604:202007-01
152 resource_class: 2xlarge
153 steps:
154 - pre-steps
155 - install-gflags
156 - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
157 - post-steps
158
159 build-linux-shared_lib-alt_namespace-status_checked:
160 machine:
161 image: ubuntu-1604:202007-01
162 resource_class: 2xlarge
163 steps:
164 - pre-steps
165 - install-gflags
166 - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 all check_some | .circleci/cat_ignore_eagain
167 - post-steps
168
169 build-linux-release:
170 machine:
171 image: ubuntu-1604:202007-01
172 resource_class: 2xlarge
173 steps:
174 - checkout # check out the code in the project directory
175 - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
176 - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
177 - install-gflags
178 - run: make V=1 -j32 release | .circleci/cat_ignore_eagain
179 - run: ./db_stress --version # ensure with gflags
180 - post-steps
181
182 build-linux-release-rtti:
183 machine:
184 image: ubuntu-1604:201903-01
185 resource_class: large
186 steps:
187 - checkout # check out the code in the project directory
188 - run: make clean
189 - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
190 - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
191 - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
192 - run: make clean
193 - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench | .circleci/cat_ignore_eagain
194 - run: ./db_stress --version # ensure with gflags
195
196 build-linux-lite:
197 machine:
198 image: ubuntu-1604:202007-01
199 resource_class: 2xlarge
200 steps:
201 - pre-steps
202 - install-gflags
203 - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
204 - post-steps
205
206 build-linux-lite-release:
207 machine:
208 image: ubuntu-1604:202007-01
209 resource_class: large
210 steps:
211 - checkout # check out the code in the project directory
212 - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
213 - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
214 - install-gflags
215 - run: LITE=1 make V=1 -j32 release | .circleci/cat_ignore_eagain
216 - run: ./db_stress --version # ensure with gflags
217 - post-steps
218
219 build-linux-clang-no-test:
220 machine:
221 image: ubuntu-1604:202007-01
222 resource_class: 2xlarge
223 steps:
224 - checkout # check out the code in the project directory
225 - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
226 - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j32 all | .circleci/cat_ignore_eagain
227 - post-steps
228
229 build-linux-clang10-asan:
230 machine:
231 image: ubuntu-1604:202007-01
232 resource_class: 2xlarge
233 steps:
234 - pre-steps
235 - install-gflags
236 - install-clang-10
237 - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
238 - post-steps
239
240 build-linux-clang10-mini-tsan:
241 machine:
242 image: ubuntu-1604:202007-01
243 resource_class: 2xlarge
244 steps:
245 - pre-steps
246 - install-gflags
247 - install-clang-10
248 - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
249 - post-steps
250
251 build-linux-clang10-ubsan:
252 machine:
253 image: ubuntu-1604:202007-01
254 resource_class: 2xlarge
255 steps:
256 - pre-steps
257 - install-gflags
258 - install-clang-10
259 - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
260 - post-steps
261
262 build-linux-clang10-clang-analyze:
263 machine:
264 image: ubuntu-1604:202007-01
265 resource_class: 2xlarge
266 steps:
267 - pre-steps
268 - install-gflags
269 - install-clang-10
270 - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
271 - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
272 - post-steps
273
274 build-linux-cmake:
275 machine:
276 image: ubuntu-1604:202007-01
277 resource_class: 2xlarge
278 steps:
279 - checkout # check out the code in the project directory
280 - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
281 - post-steps
282
283 build-linux-unity:
284 docker: # executor type
285 - image: gcc:latest
286 resource_class: xlarge
287 steps:
288 - checkout # check out the code in the project directory
289 - run: apt-get update -y && apt-get install -y libgflags-dev
290 - run: TEST_TMPDIR=/dev/shm && make V=1 -j16 unity_test | .circleci/cat_ignore_eagain
291 - post-steps
292
293 build-linux-gcc-4-8:
294 machine:
295 image: ubuntu-1604:202007-01
296 resource_class: large
297 steps:
298 - pre-steps
299 - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
300 - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j4 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
301 - post-steps
302
303 build-windows:
304 executor: windows-2xlarge
305 parameters:
306 extra_cmake_opt:
307 default: ""
308 type: string
309 vs_year:
310 default: "2019"
311 type: string
312 cmake_generator:
313 default: "Visual Studio 16 2019"
314 type: string
315 environment:
316 THIRDPARTY_HOME: C:/Users/circleci/thirdparty
317 CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
318 CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
319 SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
320 SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
321 SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
322 VS_YEAR: <<parameters.vs_year>>
323 CMAKE_GENERATOR: <<parameters.cmake_generator>>
324 steps:
325 - checkout
326 - run:
327 name: "Setup VS"
328 command: |
329 if [[ "${VS_YEAR}" == "2017" ]]; then
330 powershell .circleci/vs2017_install.ps1
331 elif [[ "${VS_YEAR}" == "2015" ]]; then
332 powershell .circleci/vs2015_install.ps1
333 fi
334 - run:
335 name: "Install thirdparty dependencies"
336 command: |
337 mkdir ${THIRDPARTY_HOME}
338 cd ${THIRDPARTY_HOME}
339 echo "Installing CMake..."
340 curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
341 unzip -q cmake-3.16.4-win64-x64.zip
342 echo "Building Snappy dependency..."
343 curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
344 unzip -q snappy-1.1.7.zip
345 cd snappy-1.1.7
346 mkdir build
347 cd build
348 ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
349 msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
350 - run:
351 name: "Build RocksDB"
352 command: |
353 mkdir build
354 cd build
355 ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
356 cd ..
357 msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
358 - run:
359 name: "Test RocksDB"
360 shell: powershell.exe
361 command: |
362 build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
363
364 build-linux-java:
365 machine:
366 image: ubuntu-1604:202007-01
367 resource_class: large
368 environment:
369 JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
370 steps:
371 - pre-steps
372 - install-gflags
373 - run:
374 name: "Set Java Environment"
375 command: |
376 echo "JAVA_HOME=${JAVA_HOME}"
377 echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
378 which java && java -version
379 which javac && javac -version
380 - run:
381 name: "Build RocksDBJava Shared Library"
382 command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
383 - run:
384 name: "Test RocksDBJava"
385 command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
386 - post-steps
387
388 build-linux-java-static:
389 machine:
390 image: ubuntu-1604:202007-01
391 resource_class: large
392 environment:
393 JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
394 steps:
395 - pre-steps
396 - install-gflags
397 - run:
398 name: "Set Java Environment"
399 command: |
400 echo "JAVA_HOME=${JAVA_HOME}"
401 echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
402 which java && java -version
403 which javac && javac -version
404 - run:
405 name: "Build RocksDBJava Static Library"
406 command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
407 - post-steps
408
409 build-macos-java:
410 macos:
411 xcode: 9.4.1
412 resource_class: medium
413 environment:
414 JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
415 steps:
416 - increase-max-open-files-on-macos
417 - install-pyenv-on-macos
418 - install-gflags-on-macos
419 - pre-steps-macos
420 - run:
421 name: "Set Java Environment"
422 command: |
423 echo "JAVA_HOME=${JAVA_HOME}"
424 echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
425 which java && java -version
426 which javac && javac -version
427 - run:
428 name: "Build RocksDBJava Shared Library"
429 command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
430 - run:
431 name: "Test RocksDBJava"
432 command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
433 - post-steps
434
435 build-macos-java-static:
436 macos:
437 xcode: 9.4.1
438 resource_class: medium
439 environment:
440 JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
441 steps:
442 - increase-max-open-files-on-macos
443 - install-pyenv-on-macos
444 - install-gflags-on-macos
445 - install-cmake-on-macos
446 - pre-steps-macos
447 - run:
448 name: "Set Java Environment"
449 command: |
450 echo "JAVA_HOME=${JAVA_HOME}"
451 echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
452 which java && java -version
453 which javac && javac -version
454 - run:
455 name: "Build RocksDBJava Static Library"
456 command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
457 - post-steps
458
459 build-examples:
460 machine:
461 image: ubuntu-1604:202007-01
462 resource_class: medium
463 steps:
464 - pre-steps
465 - install-gflags
466 - run:
467 name: "Build examples"
468 command: |
469 OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
470 - post-steps
471
472 build-linux-non-shm:
473 machine:
474 image: ubuntu-1604:202007-01
475 resource_class: 2xlarge
476 parameters:
477 start_test:
478 default: ""
479 type: string
480 end_test:
481 default: ""
482 type: string
483 steps:
484 - pre-steps
485 - install-gflags
486 - install-gtest-parallel
487 - run:
488 name: "Build unit tests"
489 command: |
490 echo "env: $(env)"
491 echo "** done env"
492 ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
493 - run:
494 name: "Run unit tests in parallel"
495 command: |
496 sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
497 cat /tmp/test_list
498 export TEST_TMPDIR=/tmp/rocksdb_test_tmp
499 /usr/bin/python ../gtest-parallel/gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
500 - post-steps
501
502 workflows:
503 build-linux:
504 jobs:
505 - build-linux
506 build-linux-mem-env:
507 jobs:
508 - build-linux-mem-env
509 build-linux-encrypted-env:
510 jobs:
511 - build-linux-encrypted-env
512 build-linux-shared_lib-alt_namespace-status_checked:
513 jobs:
514 - build-linux-shared_lib-alt_namespace-status_checked
515 build-linux-lite:
516 jobs:
517 - build-linux-lite
518 build-linux-release:
519 jobs:
520 - build-linux-release
521 build-linux-release-rtti:
522 jobs:
523 - build-linux-release-rtti
524 build-linux-lite-release:
525 jobs:
526 - build-linux-lite-release
527 build-linux-clang-no-test:
528 jobs:
529 - build-linux-clang-no-test
530 build-linux-clang10-asan:
531 jobs:
532 - build-linux-clang10-asan
533 build-linux-clang10-mini-tsan:
534 jobs:
535 - build-linux-clang10-mini-tsan
536 build-linux-clang10-ubsan:
537 jobs:
538 - build-linux-clang10-ubsan
539 build-linux-clang10-clang-analyze:
540 jobs:
541 - build-linux-clang10-clang-analyze
542 build-linux-cmake:
543 jobs:
544 - build-linux-cmake
545 build-linux-unity:
546 jobs:
547 - build-linux-unity
548 build-windows:
549 jobs:
550 - build-windows
551 build-windows-vs2017:
552 jobs:
553 - build-windows:
554 vs_year: "2017"
555 cmake_generator: "Visual Studio 15 Win64"
556 build-windows-vs2015:
557 jobs:
558 - build-windows:
559 vs_year: "2015"
560 cmake_generator: "Visual Studio 14 Win64"
561 build-windows-cxx20:
562 jobs:
563 - build-windows:
564 extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
565 build-java:
566 jobs:
567 - build-linux-java
568 - build-linux-java-static
569 - build-macos-java
570 - build-macos-java-static
571 build-examples:
572 jobs:
573 - build-examples
574 build-linux-non-shm:
575 jobs:
576 - build-linux-non-shm:
577 start_test: ""
578 end_test: "db_options_test" # make sure unique in src.mk
579 - build-linux-non-shm:
580 start_test: "db_options_test" # make sure unique in src.mk
581 end_test: "filename_test" # make sure unique in src.mk
582 - build-linux-non-shm:
583 start_test: "filename_test" # make sure unique in src.mk
584 end_test: "statistics_test" # make sure unique in src.mk
585 - build-linux-non-shm:
586 start_test: "statistics_test" # make sure unique in src.mk
587 end_test: ""
588 build-linux-gcc-4-8:
589 jobs:
590 - build-linux-gcc-4-8
591 build-macos:
592 jobs:
593 - build-macos