]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | #!/usr/bin/env bash |
7c673cae FG |
2 | # This script is executed by Sandcastle |
3 | # to determine next steps to run | |
4 | ||
5 | # Usage: | |
6 | # EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name> | |
7 | # | |
8 | # Input Value | |
9 | # ------------------------------------------------------------------------- | |
10 | # EMAIL Email address to report on trigger conditions | |
11 | # ONCALL Email address to raise a task on failure | |
12 | # TRIGGER Trigger conditions for email. Valid values are fail, warn, all | |
13 | # SUBSCRIBER Email addresss to add as subscriber for task | |
14 | # | |
15 | ||
16 | # | |
17 | # Report configuration | |
18 | # | |
19 | REPORT_EMAIL= | |
20 | if [ ! -z $EMAIL ]; then | |
21 | if [ -z $TRIGGER ]; then | |
22 | TRIGGER="fail" | |
23 | fi | |
24 | ||
25 | REPORT_EMAIL=" | |
26 | { | |
27 | 'type':'email', | |
28 | 'triggers': [ '$TRIGGER' ], | |
29 | 'emails':['$EMAIL'] | |
30 | }," | |
31 | fi | |
32 | ||
33 | CREATE_TASK= | |
34 | if [ ! -z $ONCALL ]; then | |
35 | CREATE_TASK=" | |
36 | { | |
37 | 'type':'task', | |
38 | 'triggers':[ 'fail' ], | |
39 | 'priority':0, | |
40 | 'subscribers':[ '$SUBSCRIBER' ], | |
41 | 'tags':[ 'rocksdb', 'ci' ], | |
42 | }," | |
43 | fi | |
44 | ||
45 | # For now, create the tasks using only the dedicated task creation tool. | |
46 | CREATE_TASK= | |
47 | ||
48 | REPORT= | |
49 | if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then | |
50 | REPORT="'report': [ | |
51 | $REPORT_EMAIL | |
52 | $CREATE_TASK | |
53 | ]" | |
54 | fi | |
55 | ||
56 | # | |
57 | # Helper variables | |
58 | # | |
59 | CLEANUP_ENV=" | |
60 | { | |
61 | 'name':'Cleanup environment', | |
62 | 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean', | |
63 | 'user':'root' | |
64 | }" | |
65 | ||
66 | # We will eventually set the RATIO to 1, but we want do this | |
67 | # in steps. RATIO=$(nproc) will make it work as J=1 | |
68 | if [ -z $RATIO ]; then | |
69 | RATIO=$(nproc) | |
70 | fi | |
71 | ||
72 | if [ -z $PARALLEL_J ]; then | |
73 | PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" | |
74 | fi | |
75 | ||
76 | if [ -z $PARALLEL_j ]; then | |
77 | PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" | |
78 | fi | |
79 | ||
80 | PARALLELISM="$PARALLEL_J $PARALLEL_j" | |
81 | ||
82 | DEBUG="OPT=-g" | |
83 | SHM="TEST_TMPDIR=/dev/shm/rocksdb" | |
84 | NON_SHM="TMPD=/tmp/rocksdb_test_tmp" | |
85 | GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" | |
86 | ASAN="COMPILE_WITH_ASAN=1" | |
87 | CLANG="USE_CLANG=1" | |
494da23a TL |
88 | # in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090. |
89 | # using platform007 gives us gcc-8 or higher which has that bug fixed. | |
90 | TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1" | |
7c673cae | 91 | UBSAN="COMPILE_WITH_UBSAN=1" |
11fdf7f2 TL |
92 | TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"' |
93 | NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" | |
7c673cae FG |
94 | DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" |
95 | HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" | |
96 | SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" | |
97 | PARSER="'parser':'python build_tools/error_filter.py $1'" | |
98 | ||
99 | CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" | |
100 | ||
101 | # This code is getting called under various scenarios. What we care about is to | |
102 | # understand when it's called from nightly contruns because in that case we'll | |
103 | # create tasks for any failures. To follow the existing pattern, we'll check | |
104 | # the value of $ONCALL. If it's a diff then just call `false` to make sure | |
105 | # that errors will be properly propagated to the caller. | |
106 | if [ ! -z $ONCALL ]; then | |
107 | TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" | |
108 | else | |
109 | TASK_CREATION_TOOL="false" | |
110 | fi | |
111 | ||
112 | ARTIFACTS=" 'artifacts': [ | |
113 | { | |
114 | 'name':'database', | |
115 | 'paths':[ '/dev/shm/rocksdb' ], | |
116 | } | |
117 | ]" | |
118 | ||
119 | # | |
120 | # A mechanism to disable tests temporarily | |
121 | # | |
122 | DISABLE_COMMANDS="[ | |
123 | { | |
124 | 'name':'Disable test', | |
125 | 'oncall':'$ONCALL', | |
126 | 'steps': [ | |
127 | { | |
128 | 'name':'Job disabled. Please contact test owner', | |
129 | 'shell':'exit 1', | |
130 | 'user':'root' | |
131 | }, | |
132 | ], | |
133 | } | |
134 | ]" | |
135 | ||
7c673cae FG |
136 | # |
137 | # RocksDB unit test | |
138 | # | |
139 | UNIT_TEST_COMMANDS="[ | |
140 | { | |
141 | 'name':'Rocksdb Unit Test', | |
142 | 'oncall':'$ONCALL', | |
143 | 'steps': [ | |
144 | $CLEANUP_ENV, | |
145 | { | |
146 | 'name':'Build and test RocksDB debug version', | |
147 | 'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL', | |
148 | 'user':'root', | |
149 | $PARSER | |
150 | }, | |
151 | ], | |
152 | $REPORT | |
153 | } | |
154 | ]" | |
155 | ||
156 | # | |
157 | # RocksDB unit test not under /dev/shm | |
158 | # | |
159 | UNIT_TEST_NON_SHM_COMMANDS="[ | |
160 | { | |
161 | 'name':'Rocksdb Unit Test', | |
162 | 'oncall':'$ONCALL', | |
163 | 'timeout': 86400, | |
164 | 'steps': [ | |
165 | $CLEANUP_ENV, | |
166 | { | |
167 | 'name':'Build and test RocksDB debug version', | |
168 | 'timeout': 86400, | |
169 | 'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL', | |
170 | 'user':'root', | |
171 | $PARSER | |
172 | }, | |
173 | ], | |
174 | $REPORT | |
175 | } | |
176 | ]" | |
177 | ||
178 | # | |
179 | # RocksDB release build and unit tests | |
180 | # | |
181 | RELEASE_BUILD_COMMANDS="[ | |
182 | { | |
183 | 'name':'Rocksdb Release Build', | |
184 | 'oncall':'$ONCALL', | |
185 | 'steps': [ | |
186 | $CLEANUP_ENV, | |
187 | { | |
188 | 'name':'Build RocksDB release', | |
189 | 'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL', | |
190 | 'user':'root', | |
191 | $PARSER | |
192 | }, | |
193 | ], | |
194 | $REPORT | |
195 | } | |
196 | ]" | |
197 | ||
198 | # | |
199 | # RocksDB unit test on gcc-4.8.1 | |
200 | # | |
201 | UNIT_TEST_COMMANDS_481="[ | |
202 | { | |
203 | 'name':'Rocksdb Unit Test on GCC 4.8.1', | |
204 | 'oncall':'$ONCALL', | |
205 | 'steps': [ | |
206 | $CLEANUP_ENV, | |
207 | { | |
208 | 'name':'Build and test RocksDB debug version', | |
209 | 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL', | |
210 | 'user':'root', | |
211 | $PARSER | |
212 | }, | |
213 | ], | |
214 | $REPORT | |
215 | } | |
216 | ]" | |
217 | ||
218 | # | |
219 | # RocksDB release build and unit tests | |
220 | # | |
221 | RELEASE_BUILD_COMMANDS_481="[ | |
222 | { | |
223 | 'name':'Rocksdb Release on GCC 4.8.1', | |
224 | 'oncall':'$ONCALL', | |
225 | 'steps': [ | |
226 | $CLEANUP_ENV, | |
227 | { | |
228 | 'name':'Build RocksDB release on GCC 4.8.1', | |
229 | 'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL', | |
230 | 'user':'root', | |
231 | $PARSER | |
232 | }, | |
233 | ], | |
234 | $REPORT | |
235 | } | |
236 | ]" | |
237 | ||
238 | # | |
239 | # RocksDB unit test with CLANG | |
240 | # | |
241 | CLANG_UNIT_TEST_COMMANDS="[ | |
242 | { | |
243 | 'name':'Rocksdb Unit Test', | |
244 | 'oncall':'$ONCALL', | |
245 | 'steps': [ | |
246 | $CLEANUP_ENV, | |
247 | { | |
248 | 'name':'Build and test RocksDB debug', | |
249 | 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL', | |
250 | 'user':'root', | |
251 | $PARSER | |
252 | }, | |
253 | ], | |
254 | $REPORT | |
255 | } | |
256 | ]" | |
257 | ||
258 | # | |
259 | # RocksDB release build with CLANG | |
260 | # | |
261 | CLANG_RELEASE_BUILD_COMMANDS="[ | |
262 | { | |
263 | 'name':'Rocksdb CLANG Release Build', | |
264 | 'oncall':'$ONCALL', | |
265 | 'steps': [ | |
266 | $CLEANUP_ENV, | |
267 | { | |
268 | 'name':'Build RocksDB release', | |
269 | 'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL', | |
270 | 'user':'root', | |
271 | $PARSER | |
272 | }, | |
273 | ], | |
274 | $REPORT | |
275 | } | |
276 | ]" | |
277 | ||
278 | # | |
279 | # RocksDB analyze | |
280 | # | |
281 | CLANG_ANALYZE_COMMANDS="[ | |
282 | { | |
283 | 'name':'Rocksdb analyze', | |
284 | 'oncall':'$ONCALL', | |
285 | 'steps': [ | |
286 | $CLEANUP_ENV, | |
287 | { | |
288 | 'name':'RocksDB build and analyze', | |
289 | 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL', | |
290 | 'user':'root', | |
291 | $PARSER | |
292 | }, | |
293 | ], | |
294 | $REPORT | |
295 | } | |
296 | ]" | |
297 | ||
298 | # | |
299 | # RocksDB code coverage | |
300 | # | |
301 | CODE_COV_COMMANDS="[ | |
302 | { | |
303 | 'name':'Rocksdb Unit Test Code Coverage', | |
304 | 'oncall':'$ONCALL', | |
305 | 'steps': [ | |
306 | $CLEANUP_ENV, | |
307 | { | |
308 | 'name':'Build, test and collect code coverage info', | |
309 | 'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL', | |
310 | 'user':'root', | |
311 | $PARSER | |
312 | }, | |
313 | ], | |
314 | $REPORT | |
315 | } | |
316 | ]" | |
317 | ||
318 | # | |
319 | # RocksDB unity | |
320 | # | |
321 | UNITY_COMMANDS="[ | |
322 | { | |
323 | 'name':'Rocksdb Unity', | |
324 | 'oncall':'$ONCALL', | |
325 | 'steps': [ | |
326 | $CLEANUP_ENV, | |
327 | { | |
328 | 'name':'Build, test unity test', | |
329 | 'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL', | |
330 | 'user':'root', | |
331 | $PARSER | |
332 | }, | |
333 | ], | |
334 | $REPORT | |
335 | } | |
336 | ]" | |
337 | ||
338 | # | |
339 | # Build RocksDB lite | |
340 | # | |
341 | LITE_BUILD_COMMANDS="[ | |
342 | { | |
343 | 'name':'Rocksdb Lite build', | |
344 | 'oncall':'$ONCALL', | |
345 | 'steps': [ | |
346 | $CLEANUP_ENV, | |
347 | { | |
348 | 'name':'Build RocksDB debug version', | |
494da23a | 349 | 'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL', |
7c673cae FG |
350 | 'user':'root', |
351 | $PARSER | |
352 | }, | |
353 | ], | |
354 | $REPORT | |
355 | } | |
356 | ]" | |
357 | ||
358 | # | |
11fdf7f2 TL |
359 | # Report RocksDB lite binary size to scuba |
360 | REPORT_LITE_BINARY_SIZE_COMMANDS="[ | |
7c673cae | 361 | { |
11fdf7f2 | 362 | 'name':'Rocksdb Lite Binary Size', |
7c673cae FG |
363 | 'oncall':'$ONCALL', |
364 | 'steps': [ | |
365 | $CLEANUP_ENV, | |
366 | { | |
11fdf7f2 TL |
367 | 'name':'Report RocksDB Lite binary size to scuba', |
368 | 'shell':'tools/report_lite_binary_size.sh', | |
7c673cae | 369 | 'user':'root', |
7c673cae FG |
370 | }, |
371 | ], | |
7c673cae FG |
372 | ]" |
373 | ||
374 | # | |
375 | # RocksDB stress/crash test | |
376 | # | |
377 | STRESS_CRASH_TEST_COMMANDS="[ | |
378 | { | |
379 | 'name':'Rocksdb Stress/Crash Test', | |
380 | 'oncall':'$ONCALL', | |
381 | 'timeout': 86400, | |
382 | 'steps': [ | |
383 | $CLEANUP_ENV, | |
384 | { | |
385 | 'name':'Build and run RocksDB debug stress tests', | |
11fdf7f2 | 386 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', |
7c673cae FG |
387 | 'user':'root', |
388 | $PARSER | |
389 | }, | |
390 | { | |
391 | 'name':'Build and run RocksDB debug crash tests', | |
392 | 'timeout': 86400, | |
11fdf7f2 | 393 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
394 | 'user':'root', |
395 | $PARSER | |
396 | } | |
397 | ], | |
398 | $ARTIFACTS, | |
399 | $REPORT | |
400 | } | |
401 | ]" | |
402 | ||
494da23a TL |
403 | # |
404 | # RocksDB stress/crash test with atomic flush | |
405 | # | |
406 | STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
407 | { | |
408 | 'name':'Rocksdb Stress/Crash Test (atomic flush)', | |
409 | 'oncall':'$ONCALL', | |
410 | 'timeout': 86400, | |
411 | 'steps': [ | |
412 | $CLEANUP_ENV, | |
413 | { | |
414 | 'name':'Build and run RocksDB debug stress tests', | |
415 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', | |
416 | 'user':'root', | |
417 | $PARSER | |
418 | }, | |
419 | { | |
420 | 'name':'Build and run RocksDB debug crash tests with atomic flush', | |
421 | 'timeout': 86400, | |
422 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
423 | 'user':'root', | |
424 | $PARSER | |
425 | } | |
426 | ], | |
427 | $ARTIFACTS, | |
428 | $REPORT | |
429 | } | |
430 | ]" | |
431 | ||
7c673cae FG |
432 | # RocksDB write stress test. |
433 | # We run on disk device on purpose (i.e. no $SHM) | |
434 | # because we want to add some randomness to fsync commands | |
435 | WRITE_STRESS_COMMANDS="[ | |
436 | { | |
437 | 'name':'Rocksdb Write Stress Test', | |
438 | 'oncall':'$ONCALL', | |
439 | 'steps': [ | |
440 | $CLEANUP_ENV, | |
441 | { | |
442 | 'name':'Build and run RocksDB write stress tests', | |
443 | 'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL', | |
444 | 'user':'root', | |
445 | $PARSER | |
446 | } | |
447 | ], | |
448 | 'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}], | |
449 | $REPORT | |
450 | } | |
451 | ]" | |
452 | ||
453 | ||
454 | # | |
455 | # RocksDB test under address sanitizer | |
456 | # | |
457 | ASAN_TEST_COMMANDS="[ | |
458 | { | |
459 | 'name':'Rocksdb Unit Test under ASAN', | |
460 | 'oncall':'$ONCALL', | |
461 | 'steps': [ | |
462 | $CLEANUP_ENV, | |
463 | { | |
464 | 'name':'Test RocksDB debug under ASAN', | |
465 | 'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d', | |
466 | 'user':'root', | |
467 | $PARSER | |
468 | } | |
469 | ], | |
470 | $REPORT | |
471 | } | |
472 | ]" | |
473 | ||
474 | # | |
475 | # RocksDB crash testing under address sanitizer | |
476 | # | |
477 | ASAN_CRASH_TEST_COMMANDS="[ | |
478 | { | |
479 | 'name':'Rocksdb crash test under ASAN', | |
480 | 'oncall':'$ONCALL', | |
481 | 'timeout': 86400, | |
482 | 'steps': [ | |
483 | $CLEANUP_ENV, | |
484 | { | |
485 | 'name':'Build and run RocksDB debug asan_crash_test', | |
486 | 'timeout': 86400, | |
11fdf7f2 | 487 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
488 | 'user':'root', |
489 | $PARSER | |
490 | }, | |
491 | ], | |
492 | $REPORT | |
493 | } | |
494 | ]" | |
495 | ||
494da23a TL |
496 | # |
497 | # RocksDB crash testing with atomic flush under address sanitizer | |
498 | # | |
499 | ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
500 | { | |
501 | 'name':'Rocksdb crash test (atomic flush) under ASAN', | |
502 | 'oncall':'$ONCALL', | |
503 | 'timeout': 86400, | |
504 | 'steps': [ | |
505 | $CLEANUP_ENV, | |
506 | { | |
507 | 'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush', | |
508 | 'timeout': 86400, | |
509 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
510 | 'user':'root', | |
511 | $PARSER | |
512 | }, | |
513 | ], | |
514 | $REPORT | |
515 | } | |
516 | ]" | |
517 | ||
7c673cae FG |
518 | # |
519 | # RocksDB test under undefined behavior sanitizer | |
520 | # | |
521 | UBSAN_TEST_COMMANDS="[ | |
522 | { | |
523 | 'name':'Rocksdb Unit Test under UBSAN', | |
524 | 'oncall':'$ONCALL', | |
525 | 'steps': [ | |
526 | $CLEANUP_ENV, | |
527 | { | |
528 | 'name':'Test RocksDB debug under UBSAN', | |
529 | 'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL', | |
530 | 'user':'root', | |
531 | $PARSER | |
532 | } | |
533 | ], | |
534 | $REPORT | |
535 | } | |
536 | ]" | |
537 | ||
538 | # | |
539 | # RocksDB crash testing under udnefined behavior sanitizer | |
540 | # | |
541 | UBSAN_CRASH_TEST_COMMANDS="[ | |
542 | { | |
543 | 'name':'Rocksdb crash test under UBSAN', | |
544 | 'oncall':'$ONCALL', | |
545 | 'timeout': 86400, | |
546 | 'steps': [ | |
547 | $CLEANUP_ENV, | |
548 | { | |
549 | 'name':'Build and run RocksDB debug ubsan_crash_test', | |
550 | 'timeout': 86400, | |
11fdf7f2 | 551 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
552 | 'user':'root', |
553 | $PARSER | |
554 | }, | |
555 | ], | |
556 | $REPORT | |
557 | } | |
558 | ]" | |
559 | ||
494da23a TL |
560 | # |
561 | # RocksDB crash testing with atomic flush under undefined behavior sanitizer | |
562 | # | |
563 | UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
564 | { | |
565 | 'name':'Rocksdb crash test (atomic flush) under UBSAN', | |
566 | 'oncall':'$ONCALL', | |
567 | 'timeout': 86400, | |
568 | 'steps': [ | |
569 | $CLEANUP_ENV, | |
570 | { | |
571 | 'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush', | |
572 | 'timeout': 86400, | |
573 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
574 | 'user':'root', | |
575 | $PARSER | |
576 | }, | |
577 | ], | |
578 | $REPORT | |
579 | } | |
580 | ]" | |
581 | ||
7c673cae FG |
582 | # |
583 | # RocksDB unit test under valgrind | |
584 | # | |
585 | VALGRIND_TEST_COMMANDS="[ | |
586 | { | |
587 | 'name':'Rocksdb Unit Test under valgrind', | |
588 | 'oncall':'$ONCALL', | |
589 | 'timeout': 86400, | |
590 | 'steps': [ | |
591 | $CLEANUP_ENV, | |
592 | { | |
593 | 'name':'Run RocksDB debug unit tests', | |
594 | 'timeout': 86400, | |
595 | 'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL', | |
596 | 'user':'root', | |
597 | $PARSER | |
598 | }, | |
599 | ], | |
600 | $REPORT | |
601 | } | |
602 | ]" | |
603 | ||
604 | # | |
605 | # RocksDB test under TSAN | |
606 | # | |
607 | TSAN_UNIT_TEST_COMMANDS="[ | |
608 | { | |
609 | 'name':'Rocksdb Unit Test under TSAN', | |
610 | 'oncall':'$ONCALL', | |
611 | 'timeout': 86400, | |
612 | 'steps': [ | |
613 | $CLEANUP_ENV, | |
614 | { | |
615 | 'name':'Run RocksDB debug unit test', | |
616 | 'timeout': 86400, | |
617 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL', | |
618 | 'user':'root', | |
619 | $PARSER | |
620 | }, | |
621 | ], | |
622 | $REPORT | |
623 | } | |
624 | ]" | |
625 | ||
626 | # | |
627 | # RocksDB crash test under TSAN | |
628 | # | |
629 | TSAN_CRASH_TEST_COMMANDS="[ | |
630 | { | |
631 | 'name':'Rocksdb Crash Test under TSAN', | |
632 | 'oncall':'$ONCALL', | |
633 | 'timeout': 86400, | |
634 | 'steps': [ | |
635 | $CLEANUP_ENV, | |
636 | { | |
637 | 'name':'Compile and run', | |
638 | 'timeout': 86400, | |
11fdf7f2 | 639 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
640 | 'user':'root', |
641 | $PARSER | |
642 | }, | |
643 | ], | |
644 | $REPORT | |
645 | } | |
646 | ]" | |
647 | ||
494da23a TL |
648 | # |
649 | # RocksDB crash test with atomic flush under TSAN | |
650 | # | |
651 | TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
652 | { | |
653 | 'name':'Rocksdb Crash Test with atomic flush under TSAN', | |
654 | 'oncall':'$ONCALL', | |
655 | 'timeout': 86400, | |
656 | 'steps': [ | |
657 | $CLEANUP_ENV, | |
658 | { | |
659 | 'name':'Compile and run', | |
660 | 'timeout': 86400, | |
661 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
662 | 'user':'root', | |
663 | $PARSER | |
664 | }, | |
665 | ], | |
666 | $REPORT | |
667 | } | |
668 | ]" | |
669 | ||
7c673cae FG |
670 | # |
671 | # RocksDB format compatible | |
672 | # | |
673 | ||
674 | run_format_compatible() | |
675 | { | |
676 | export TEST_TMPDIR=/dev/shm/rocksdb | |
677 | rm -rf /dev/shm/rocksdb | |
678 | mkdir /dev/shm/rocksdb | |
679 | ||
11fdf7f2 | 680 | tools/check_format_compatible.sh |
7c673cae FG |
681 | } |
682 | ||
683 | FORMAT_COMPATIBLE_COMMANDS="[ | |
684 | { | |
685 | 'name':'Rocksdb Format Compatible tests', | |
686 | 'oncall':'$ONCALL', | |
687 | 'steps': [ | |
688 | $CLEANUP_ENV, | |
689 | { | |
690 | 'name':'Run RocksDB debug unit test', | |
691 | 'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL', | |
692 | 'user':'root', | |
693 | $PARSER | |
694 | }, | |
695 | ], | |
696 | $REPORT | |
697 | } | |
698 | ]" | |
699 | ||
700 | # | |
701 | # RocksDB no compression | |
702 | # | |
703 | run_no_compression() | |
704 | { | |
705 | export TEST_TMPDIR=/dev/shm/rocksdb | |
706 | rm -rf /dev/shm/rocksdb | |
707 | mkdir /dev/shm/rocksdb | |
708 | make clean | |
494da23a | 709 | cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh |
7c673cae FG |
710 | mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh |
711 | cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile | |
712 | mv .tmp.Makefile Makefile | |
713 | make $DEBUG J=1 check | |
714 | } | |
715 | ||
716 | NO_COMPRESSION_COMMANDS="[ | |
717 | { | |
718 | 'name':'Rocksdb No Compression tests', | |
719 | 'oncall':'$ONCALL', | |
720 | 'steps': [ | |
721 | $CLEANUP_ENV, | |
722 | { | |
723 | 'name':'Run RocksDB debug unit test', | |
724 | 'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL', | |
725 | 'user':'root', | |
726 | $PARSER | |
727 | }, | |
728 | ], | |
729 | $REPORT | |
730 | } | |
731 | ]" | |
732 | ||
733 | # | |
734 | # RocksDB regression | |
735 | # | |
736 | run_regression() | |
737 | { | |
738 | time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX) | |
739 | ||
740 | # ======= report size to ODS ======== | |
741 | ||
742 | # parameters: $1 -- key, $2 -- value | |
743 | function send_size_to_ods { | |
744 | curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ | |
745 | --connect-timeout 60 | |
746 | } | |
747 | ||
748 | # === normal build === | |
749 | make clean | |
750 | make -j$(nproc) static_lib | |
751 | send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) | |
752 | strip librocksdb.a | |
753 | send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) | |
754 | ||
755 | make -j$(nproc) shared_lib | |
756 | send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) | |
757 | strip `readlink -f librocksdb.so` | |
758 | send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) | |
759 | ||
760 | # === lite build === | |
761 | make clean | |
494da23a | 762 | make LITE=1 -j$(nproc) static_lib |
7c673cae FG |
763 | send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) |
764 | strip librocksdb.a | |
765 | send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) | |
766 | ||
494da23a | 767 | make LITE=1 -j$(nproc) shared_lib |
7c673cae FG |
768 | send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) |
769 | strip `readlink -f librocksdb.so` | |
770 | send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) | |
771 | } | |
772 | ||
773 | REGRESSION_COMMANDS="[ | |
774 | { | |
775 | 'name':'Rocksdb regression commands', | |
776 | 'oncall':'$ONCALL', | |
777 | 'steps': [ | |
778 | $CLEANUP_ENV, | |
779 | { | |
780 | 'name':'Make and run script', | |
781 | 'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL', | |
782 | 'user':'root', | |
783 | $PARSER | |
784 | }, | |
785 | ], | |
786 | $REPORT | |
787 | } | |
788 | ]" | |
789 | ||
790 | # | |
791 | # RocksDB Java build | |
792 | # | |
793 | JAVA_BUILD_TEST_COMMANDS="[ | |
794 | { | |
795 | 'name':'Rocksdb Java Build', | |
796 | 'oncall':'$ONCALL', | |
797 | 'steps': [ | |
798 | $CLEANUP_ENV, | |
799 | { | |
800 | 'name':'Build RocksDB for Java', | |
801 | 'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL', | |
802 | 'user':'root', | |
803 | $PARSER | |
804 | }, | |
805 | ], | |
806 | $REPORT | |
807 | } | |
808 | ]" | |
809 | ||
810 | ||
811 | case $1 in | |
7c673cae FG |
812 | unit) |
813 | echo $UNIT_TEST_COMMANDS | |
814 | ;; | |
815 | unit_non_shm) | |
816 | echo $UNIT_TEST_NON_SHM_COMMANDS | |
817 | ;; | |
818 | release) | |
819 | echo $RELEASE_BUILD_COMMANDS | |
820 | ;; | |
821 | unit_481) | |
822 | echo $UNIT_TEST_COMMANDS_481 | |
823 | ;; | |
824 | release_481) | |
825 | echo $RELEASE_BUILD_COMMANDS_481 | |
826 | ;; | |
827 | clang_unit) | |
828 | echo $CLANG_UNIT_TEST_COMMANDS | |
829 | ;; | |
830 | clang_release) | |
831 | echo $CLANG_RELEASE_BUILD_COMMANDS | |
832 | ;; | |
833 | clang_analyze) | |
834 | echo $CLANG_ANALYZE_COMMANDS | |
835 | ;; | |
836 | code_cov) | |
837 | echo $CODE_COV_COMMANDS | |
838 | ;; | |
839 | unity) | |
840 | echo $UNITY_COMMANDS | |
841 | ;; | |
842 | lite) | |
843 | echo $LITE_BUILD_COMMANDS | |
844 | ;; | |
11fdf7f2 TL |
845 | report_lite_binary_size) |
846 | echo $REPORT_LITE_BINARY_SIZE_COMMANDS | |
7c673cae FG |
847 | ;; |
848 | stress_crash) | |
849 | echo $STRESS_CRASH_TEST_COMMANDS | |
850 | ;; | |
494da23a TL |
851 | stress_crash_with_atomic_flush) |
852 | echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
853 | ;; | |
7c673cae FG |
854 | write_stress) |
855 | echo $WRITE_STRESS_COMMANDS | |
856 | ;; | |
857 | asan) | |
858 | echo $ASAN_TEST_COMMANDS | |
859 | ;; | |
860 | asan_crash) | |
861 | echo $ASAN_CRASH_TEST_COMMANDS | |
862 | ;; | |
494da23a TL |
863 | asan_crash_with_atomic_flush) |
864 | echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
865 | ;; | |
7c673cae FG |
866 | ubsan) |
867 | echo $UBSAN_TEST_COMMANDS | |
868 | ;; | |
869 | ubsan_crash) | |
870 | echo $UBSAN_CRASH_TEST_COMMANDS | |
871 | ;; | |
494da23a TL |
872 | ubsan_crash_with_atomic_flush) |
873 | echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
874 | ;; | |
7c673cae FG |
875 | valgrind) |
876 | echo $VALGRIND_TEST_COMMANDS | |
877 | ;; | |
878 | tsan) | |
879 | echo $TSAN_UNIT_TEST_COMMANDS | |
880 | ;; | |
881 | tsan_crash) | |
882 | echo $TSAN_CRASH_TEST_COMMANDS | |
883 | ;; | |
494da23a TL |
884 | tsan_crash_with_atomic_flush) |
885 | echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
886 | ;; | |
7c673cae FG |
887 | format_compatible) |
888 | echo $FORMAT_COMPATIBLE_COMMANDS | |
889 | ;; | |
890 | run_format_compatible) | |
891 | run_format_compatible | |
892 | ;; | |
893 | no_compression) | |
894 | echo $NO_COMPRESSION_COMMANDS | |
895 | ;; | |
896 | run_no_compression) | |
897 | run_no_compression | |
898 | ;; | |
899 | regression) | |
900 | echo $REGRESSION_COMMANDS | |
901 | ;; | |
902 | run_regression) | |
903 | run_regression | |
904 | ;; | |
905 | java_build) | |
906 | echo $JAVA_BUILD_TEST_COMMANDS | |
907 | ;; | |
908 | *) | |
909 | echo "Invalid determinator command" | |
910 | ;; | |
911 | esac |