]>
Commit | Line | Data |
---|---|---|
11fdf7f2 | 1 | #!/usr/bin/env bash |
7c673cae FG |
2 | # This script is executed by Sandcastle |
3 | # to determine next steps to run | |
4 | ||
5 | # Usage: | |
6 | # EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name> | |
7 | # | |
8 | # Input Value | |
9 | # ------------------------------------------------------------------------- | |
10 | # EMAIL Email address to report on trigger conditions | |
11 | # ONCALL Email address to raise a task on failure | |
12 | # TRIGGER Trigger conditions for email. Valid values are fail, warn, all | |
13 | # SUBSCRIBER Email addresss to add as subscriber for task | |
14 | # | |
15 | ||
16 | # | |
17 | # Report configuration | |
18 | # | |
19 | REPORT_EMAIL= | |
20 | if [ ! -z $EMAIL ]; then | |
21 | if [ -z $TRIGGER ]; then | |
22 | TRIGGER="fail" | |
23 | fi | |
24 | ||
25 | REPORT_EMAIL=" | |
26 | { | |
27 | 'type':'email', | |
28 | 'triggers': [ '$TRIGGER' ], | |
29 | 'emails':['$EMAIL'] | |
30 | }," | |
31 | fi | |
32 | ||
33 | CREATE_TASK= | |
34 | if [ ! -z $ONCALL ]; then | |
35 | CREATE_TASK=" | |
36 | { | |
37 | 'type':'task', | |
38 | 'triggers':[ 'fail' ], | |
39 | 'priority':0, | |
40 | 'subscribers':[ '$SUBSCRIBER' ], | |
41 | 'tags':[ 'rocksdb', 'ci' ], | |
42 | }," | |
43 | fi | |
44 | ||
45 | # For now, create the tasks using only the dedicated task creation tool. | |
46 | CREATE_TASK= | |
47 | ||
48 | REPORT= | |
49 | if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then | |
50 | REPORT="'report': [ | |
51 | $REPORT_EMAIL | |
52 | $CREATE_TASK | |
53 | ]" | |
54 | fi | |
55 | ||
56 | # | |
57 | # Helper variables | |
58 | # | |
59 | CLEANUP_ENV=" | |
60 | { | |
61 | 'name':'Cleanup environment', | |
62 | 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean', | |
63 | 'user':'root' | |
64 | }" | |
65 | ||
f67539c2 TL |
66 | UPLOAD_DB_DIR=" |
67 | { | |
68 | 'name':'Upload database directory', | |
69 | 'shell':'tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/', | |
70 | 'user':'root', | |
71 | 'cleanup':true, | |
72 | 'provide_artifacts': [ | |
73 | { | |
74 | 'name':'rocksdb_db_dir', | |
75 | 'paths': ['rocksdb_db.tar.gz'], | |
76 | 'bundle': false, | |
77 | }, | |
78 | ], | |
79 | }" | |
80 | ||
7c673cae FG |
81 | # We will eventually set the RATIO to 1, but we want do this |
82 | # in steps. RATIO=$(nproc) will make it work as J=1 | |
83 | if [ -z $RATIO ]; then | |
84 | RATIO=$(nproc) | |
85 | fi | |
86 | ||
87 | if [ -z $PARALLEL_J ]; then | |
88 | PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" | |
89 | fi | |
90 | ||
91 | if [ -z $PARALLEL_j ]; then | |
92 | PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" | |
93 | fi | |
94 | ||
95 | PARALLELISM="$PARALLEL_J $PARALLEL_j" | |
96 | ||
97 | DEBUG="OPT=-g" | |
98 | SHM="TEST_TMPDIR=/dev/shm/rocksdb" | |
99 | NON_SHM="TMPD=/tmp/rocksdb_test_tmp" | |
100 | GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" | |
101 | ASAN="COMPILE_WITH_ASAN=1" | |
102 | CLANG="USE_CLANG=1" | |
494da23a TL |
103 | # in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090. |
104 | # using platform007 gives us gcc-8 or higher which has that bug fixed. | |
105 | TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1" | |
7c673cae | 106 | UBSAN="COMPILE_WITH_UBSAN=1" |
11fdf7f2 TL |
107 | TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"' |
108 | NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" | |
7c673cae FG |
109 | DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" |
110 | HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" | |
111 | SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" | |
112 | PARSER="'parser':'python build_tools/error_filter.py $1'" | |
113 | ||
114 | CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" | |
115 | ||
116 | # This code is getting called under various scenarios. What we care about is to | |
117 | # understand when it's called from nightly contruns because in that case we'll | |
118 | # create tasks for any failures. To follow the existing pattern, we'll check | |
119 | # the value of $ONCALL. If it's a diff then just call `false` to make sure | |
120 | # that errors will be properly propagated to the caller. | |
121 | if [ ! -z $ONCALL ]; then | |
122 | TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" | |
123 | else | |
124 | TASK_CREATION_TOOL="false" | |
125 | fi | |
126 | ||
7c673cae FG |
127 | # |
128 | # A mechanism to disable tests temporarily | |
129 | # | |
130 | DISABLE_COMMANDS="[ | |
131 | { | |
132 | 'name':'Disable test', | |
133 | 'oncall':'$ONCALL', | |
134 | 'steps': [ | |
135 | { | |
136 | 'name':'Job disabled. Please contact test owner', | |
137 | 'shell':'exit 1', | |
138 | 'user':'root' | |
139 | }, | |
140 | ], | |
141 | } | |
142 | ]" | |
143 | ||
7c673cae FG |
144 | # |
145 | # RocksDB unit test | |
146 | # | |
147 | UNIT_TEST_COMMANDS="[ | |
148 | { | |
149 | 'name':'Rocksdb Unit Test', | |
150 | 'oncall':'$ONCALL', | |
f67539c2 | 151 | 'executeLocal': 'true', |
7c673cae FG |
152 | 'steps': [ |
153 | $CLEANUP_ENV, | |
154 | { | |
155 | 'name':'Build and test RocksDB debug version', | |
156 | 'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL', | |
157 | 'user':'root', | |
158 | $PARSER | |
159 | }, | |
160 | ], | |
161 | $REPORT | |
162 | } | |
163 | ]" | |
164 | ||
165 | # | |
166 | # RocksDB unit test not under /dev/shm | |
167 | # | |
168 | UNIT_TEST_NON_SHM_COMMANDS="[ | |
169 | { | |
170 | 'name':'Rocksdb Unit Test', | |
171 | 'oncall':'$ONCALL', | |
f67539c2 | 172 | 'executeLocal': 'true', |
7c673cae FG |
173 | 'timeout': 86400, |
174 | 'steps': [ | |
175 | $CLEANUP_ENV, | |
176 | { | |
177 | 'name':'Build and test RocksDB debug version', | |
178 | 'timeout': 86400, | |
179 | 'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL', | |
180 | 'user':'root', | |
181 | $PARSER | |
182 | }, | |
183 | ], | |
184 | $REPORT | |
185 | } | |
186 | ]" | |
187 | ||
188 | # | |
189 | # RocksDB release build and unit tests | |
190 | # | |
191 | RELEASE_BUILD_COMMANDS="[ | |
192 | { | |
193 | 'name':'Rocksdb Release Build', | |
194 | 'oncall':'$ONCALL', | |
f67539c2 | 195 | 'executeLocal': 'true', |
7c673cae FG |
196 | 'steps': [ |
197 | $CLEANUP_ENV, | |
198 | { | |
199 | 'name':'Build RocksDB release', | |
200 | 'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL', | |
201 | 'user':'root', | |
202 | $PARSER | |
203 | }, | |
204 | ], | |
205 | $REPORT | |
206 | } | |
207 | ]" | |
208 | ||
209 | # | |
210 | # RocksDB unit test on gcc-4.8.1 | |
211 | # | |
212 | UNIT_TEST_COMMANDS_481="[ | |
213 | { | |
214 | 'name':'Rocksdb Unit Test on GCC 4.8.1', | |
215 | 'oncall':'$ONCALL', | |
f67539c2 | 216 | 'executeLocal': 'true', |
7c673cae FG |
217 | 'steps': [ |
218 | $CLEANUP_ENV, | |
219 | { | |
220 | 'name':'Build and test RocksDB debug version', | |
221 | 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL', | |
222 | 'user':'root', | |
223 | $PARSER | |
224 | }, | |
225 | ], | |
226 | $REPORT | |
227 | } | |
228 | ]" | |
229 | ||
230 | # | |
231 | # RocksDB release build and unit tests | |
232 | # | |
233 | RELEASE_BUILD_COMMANDS_481="[ | |
234 | { | |
235 | 'name':'Rocksdb Release on GCC 4.8.1', | |
236 | 'oncall':'$ONCALL', | |
f67539c2 | 237 | 'executeLocal': 'true', |
7c673cae FG |
238 | 'steps': [ |
239 | $CLEANUP_ENV, | |
240 | { | |
241 | 'name':'Build RocksDB release on GCC 4.8.1', | |
242 | 'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL', | |
243 | 'user':'root', | |
244 | $PARSER | |
245 | }, | |
246 | ], | |
247 | $REPORT | |
248 | } | |
249 | ]" | |
250 | ||
251 | # | |
252 | # RocksDB unit test with CLANG | |
253 | # | |
254 | CLANG_UNIT_TEST_COMMANDS="[ | |
255 | { | |
256 | 'name':'Rocksdb Unit Test', | |
257 | 'oncall':'$ONCALL', | |
f67539c2 | 258 | 'executeLocal': 'true', |
7c673cae FG |
259 | 'steps': [ |
260 | $CLEANUP_ENV, | |
261 | { | |
262 | 'name':'Build and test RocksDB debug', | |
263 | 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL', | |
264 | 'user':'root', | |
265 | $PARSER | |
266 | }, | |
267 | ], | |
268 | $REPORT | |
269 | } | |
270 | ]" | |
271 | ||
272 | # | |
273 | # RocksDB release build with CLANG | |
274 | # | |
275 | CLANG_RELEASE_BUILD_COMMANDS="[ | |
276 | { | |
277 | 'name':'Rocksdb CLANG Release Build', | |
278 | 'oncall':'$ONCALL', | |
f67539c2 | 279 | 'executeLocal': 'true', |
7c673cae FG |
280 | 'steps': [ |
281 | $CLEANUP_ENV, | |
282 | { | |
283 | 'name':'Build RocksDB release', | |
284 | 'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL', | |
285 | 'user':'root', | |
286 | $PARSER | |
287 | }, | |
288 | ], | |
289 | $REPORT | |
290 | } | |
291 | ]" | |
292 | ||
293 | # | |
294 | # RocksDB analyze | |
295 | # | |
296 | CLANG_ANALYZE_COMMANDS="[ | |
297 | { | |
298 | 'name':'Rocksdb analyze', | |
299 | 'oncall':'$ONCALL', | |
f67539c2 | 300 | 'executeLocal': 'true', |
7c673cae FG |
301 | 'steps': [ |
302 | $CLEANUP_ENV, | |
303 | { | |
304 | 'name':'RocksDB build and analyze', | |
305 | 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL', | |
306 | 'user':'root', | |
307 | $PARSER | |
308 | }, | |
309 | ], | |
310 | $REPORT | |
311 | } | |
312 | ]" | |
313 | ||
314 | # | |
315 | # RocksDB code coverage | |
316 | # | |
317 | CODE_COV_COMMANDS="[ | |
318 | { | |
319 | 'name':'Rocksdb Unit Test Code Coverage', | |
320 | 'oncall':'$ONCALL', | |
f67539c2 | 321 | 'executeLocal': 'true', |
7c673cae FG |
322 | 'steps': [ |
323 | $CLEANUP_ENV, | |
324 | { | |
325 | 'name':'Build, test and collect code coverage info', | |
326 | 'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL', | |
327 | 'user':'root', | |
328 | $PARSER | |
329 | }, | |
330 | ], | |
331 | $REPORT | |
332 | } | |
333 | ]" | |
334 | ||
335 | # | |
336 | # RocksDB unity | |
337 | # | |
338 | UNITY_COMMANDS="[ | |
339 | { | |
340 | 'name':'Rocksdb Unity', | |
341 | 'oncall':'$ONCALL', | |
f67539c2 | 342 | 'executeLocal': 'true', |
7c673cae FG |
343 | 'steps': [ |
344 | $CLEANUP_ENV, | |
345 | { | |
346 | 'name':'Build, test unity test', | |
347 | 'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL', | |
348 | 'user':'root', | |
349 | $PARSER | |
350 | }, | |
351 | ], | |
352 | $REPORT | |
353 | } | |
354 | ]" | |
355 | ||
356 | # | |
357 | # Build RocksDB lite | |
358 | # | |
359 | LITE_BUILD_COMMANDS="[ | |
360 | { | |
361 | 'name':'Rocksdb Lite build', | |
362 | 'oncall':'$ONCALL', | |
f67539c2 | 363 | 'executeLocal': 'true', |
7c673cae FG |
364 | 'steps': [ |
365 | $CLEANUP_ENV, | |
366 | { | |
367 | 'name':'Build RocksDB debug version', | |
494da23a | 368 | 'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL', |
7c673cae FG |
369 | 'user':'root', |
370 | $PARSER | |
371 | }, | |
372 | ], | |
373 | $REPORT | |
374 | } | |
375 | ]" | |
376 | ||
377 | # | |
11fdf7f2 TL |
378 | # Report RocksDB lite binary size to scuba |
379 | REPORT_LITE_BINARY_SIZE_COMMANDS="[ | |
7c673cae | 380 | { |
11fdf7f2 | 381 | 'name':'Rocksdb Lite Binary Size', |
7c673cae | 382 | 'oncall':'$ONCALL', |
f67539c2 | 383 | 'executeLocal': 'true', |
7c673cae FG |
384 | 'steps': [ |
385 | $CLEANUP_ENV, | |
386 | { | |
11fdf7f2 TL |
387 | 'name':'Report RocksDB Lite binary size to scuba', |
388 | 'shell':'tools/report_lite_binary_size.sh', | |
7c673cae | 389 | 'user':'root', |
7c673cae FG |
390 | }, |
391 | ], | |
7c673cae FG |
392 | ]" |
393 | ||
394 | # | |
395 | # RocksDB stress/crash test | |
396 | # | |
397 | STRESS_CRASH_TEST_COMMANDS="[ | |
398 | { | |
f67539c2 | 399 | 'name':'Rocksdb Stress and Crash Test', |
7c673cae | 400 | 'oncall':'$ONCALL', |
f67539c2 | 401 | 'executeLocal': 'true', |
7c673cae FG |
402 | 'timeout': 86400, |
403 | 'steps': [ | |
404 | $CLEANUP_ENV, | |
405 | { | |
406 | 'name':'Build and run RocksDB debug stress tests', | |
11fdf7f2 | 407 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', |
7c673cae FG |
408 | 'user':'root', |
409 | $PARSER | |
410 | }, | |
411 | { | |
412 | 'name':'Build and run RocksDB debug crash tests', | |
413 | 'timeout': 86400, | |
11fdf7f2 | 414 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
415 | 'user':'root', |
416 | $PARSER | |
f67539c2 TL |
417 | }, |
418 | $UPLOAD_DB_DIR, | |
7c673cae | 419 | ], |
7c673cae FG |
420 | $REPORT |
421 | } | |
422 | ]" | |
423 | ||
20effc67 TL |
424 | # |
425 | # RocksDB blackbox stress/crash test | |
426 | # | |
427 | BLACKBOX_STRESS_CRASH_TEST_COMMANDS="[ | |
428 | { | |
429 | 'name':'Rocksdb Blackbox Stress and Crash Test', | |
430 | 'oncall':'$ONCALL', | |
431 | 'executeLocal': 'true', | |
432 | 'timeout': 86400, | |
433 | 'steps': [ | |
434 | $CLEANUP_ENV, | |
435 | { | |
436 | 'name':'Build and run RocksDB debug stress tests', | |
437 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', | |
438 | 'user':'root', | |
439 | $PARSER | |
440 | }, | |
441 | { | |
442 | 'name':'Build and run RocksDB debug blackbox crash tests', | |
443 | 'timeout': 86400, | |
444 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 blackbox_crash_test || $CONTRUN_NAME=blackbox_crash_test $TASK_CREATION_TOOL', | |
445 | 'user':'root', | |
446 | $PARSER | |
447 | }, | |
448 | $UPLOAD_DB_DIR, | |
449 | ], | |
450 | $REPORT | |
451 | } | |
452 | ]" | |
453 | ||
454 | # | |
455 | # RocksDB whitebox stress/crash test | |
456 | # | |
457 | WHITEBOX_STRESS_CRASH_TEST_COMMANDS="[ | |
458 | { | |
459 | 'name':'Rocksdb Whitebox Stress and Crash Test', | |
460 | 'oncall':'$ONCALL', | |
461 | 'executeLocal': 'true', | |
462 | 'timeout': 86400, | |
463 | 'steps': [ | |
464 | $CLEANUP_ENV, | |
465 | { | |
466 | 'name':'Build and run RocksDB debug stress tests', | |
467 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', | |
468 | 'user':'root', | |
469 | $PARSER | |
470 | }, | |
471 | { | |
472 | 'name':'Build and run RocksDB debug whitebox crash tests', | |
473 | 'timeout': 86400, | |
474 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 whitebox_crash_test || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL', | |
475 | 'user':'root', | |
476 | $PARSER | |
477 | }, | |
478 | $UPLOAD_DB_DIR, | |
479 | ], | |
480 | $REPORT | |
481 | } | |
482 | ]" | |
483 | ||
494da23a TL |
484 | # |
485 | # RocksDB stress/crash test with atomic flush | |
486 | # | |
487 | STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
488 | { | |
f67539c2 | 489 | 'name':'Rocksdb Stress and Crash Test with atomic flush', |
494da23a | 490 | 'oncall':'$ONCALL', |
f67539c2 | 491 | 'executeLocal': 'true', |
494da23a TL |
492 | 'timeout': 86400, |
493 | 'steps': [ | |
494 | $CLEANUP_ENV, | |
495 | { | |
496 | 'name':'Build and run RocksDB debug stress tests', | |
497 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', | |
498 | 'user':'root', | |
499 | $PARSER | |
500 | }, | |
501 | { | |
502 | 'name':'Build and run RocksDB debug crash tests with atomic flush', | |
503 | 'timeout': 86400, | |
504 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
505 | 'user':'root', | |
506 | $PARSER | |
f67539c2 TL |
507 | }, |
508 | $UPLOAD_DB_DIR, | |
509 | ], | |
510 | $REPORT | |
511 | } | |
512 | ]" | |
513 | ||
514 | # | |
515 | # RocksDB stress/crash test with txn | |
516 | # | |
517 | STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[ | |
518 | { | |
519 | 'name':'Rocksdb Stress and Crash Test with txn', | |
520 | 'oncall':'$ONCALL', | |
521 | 'executeLocal': 'true', | |
522 | 'timeout': 86400, | |
523 | 'steps': [ | |
524 | $CLEANUP_ENV, | |
525 | { | |
526 | 'name':'Build and run RocksDB debug stress tests', | |
527 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', | |
528 | 'user':'root', | |
529 | $PARSER | |
530 | }, | |
531 | { | |
532 | 'name':'Build and run RocksDB debug crash tests with txn', | |
533 | 'timeout': 86400, | |
534 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL', | |
535 | 'user':'root', | |
536 | $PARSER | |
537 | }, | |
538 | $UPLOAD_DB_DIR, | |
494da23a | 539 | ], |
494da23a TL |
540 | $REPORT |
541 | } | |
542 | ]" | |
543 | ||
7c673cae FG |
544 | # RocksDB write stress test. |
545 | # We run on disk device on purpose (i.e. no $SHM) | |
546 | # because we want to add some randomness to fsync commands | |
547 | WRITE_STRESS_COMMANDS="[ | |
548 | { | |
549 | 'name':'Rocksdb Write Stress Test', | |
550 | 'oncall':'$ONCALL', | |
f67539c2 | 551 | 'executeLocal': 'true', |
7c673cae FG |
552 | 'steps': [ |
553 | $CLEANUP_ENV, | |
554 | { | |
555 | 'name':'Build and run RocksDB write stress tests', | |
556 | 'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL', | |
557 | 'user':'root', | |
558 | $PARSER | |
559 | } | |
560 | ], | |
561 | 'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}], | |
562 | $REPORT | |
563 | } | |
564 | ]" | |
565 | ||
566 | ||
567 | # | |
568 | # RocksDB test under address sanitizer | |
569 | # | |
570 | ASAN_TEST_COMMANDS="[ | |
571 | { | |
572 | 'name':'Rocksdb Unit Test under ASAN', | |
573 | 'oncall':'$ONCALL', | |
f67539c2 | 574 | 'executeLocal': 'true', |
7c673cae FG |
575 | 'steps': [ |
576 | $CLEANUP_ENV, | |
577 | { | |
578 | 'name':'Test RocksDB debug under ASAN', | |
579 | 'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d', | |
580 | 'user':'root', | |
581 | $PARSER | |
582 | } | |
583 | ], | |
584 | $REPORT | |
585 | } | |
586 | ]" | |
587 | ||
588 | # | |
589 | # RocksDB crash testing under address sanitizer | |
590 | # | |
591 | ASAN_CRASH_TEST_COMMANDS="[ | |
592 | { | |
593 | 'name':'Rocksdb crash test under ASAN', | |
594 | 'oncall':'$ONCALL', | |
f67539c2 | 595 | 'executeLocal': 'true', |
7c673cae FG |
596 | 'timeout': 86400, |
597 | 'steps': [ | |
598 | $CLEANUP_ENV, | |
599 | { | |
600 | 'name':'Build and run RocksDB debug asan_crash_test', | |
601 | 'timeout': 86400, | |
11fdf7f2 | 602 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
603 | 'user':'root', |
604 | $PARSER | |
605 | }, | |
f67539c2 | 606 | $UPLOAD_DB_DIR, |
7c673cae FG |
607 | ], |
608 | $REPORT | |
609 | } | |
610 | ]" | |
611 | ||
20effc67 TL |
612 | # |
613 | # RocksDB blackbox crash testing under address sanitizer | |
614 | # | |
615 | ASAN_BLACKBOX_CRASH_TEST_COMMANDS="[ | |
616 | { | |
617 | 'name':'Rocksdb blackbox crash test under ASAN', | |
618 | 'oncall':'$ONCALL', | |
619 | 'executeLocal': 'true', | |
620 | 'timeout': 86400, | |
621 | 'steps': [ | |
622 | $CLEANUP_ENV, | |
623 | { | |
624 | 'name':'Build and run RocksDB debug blackbox asan_crash_test', | |
625 | 'timeout': 86400, | |
626 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 blackbox_asan_crash_test || $CONTRUN_NAME=blackbox_asan_crash_test $TASK_CREATION_TOOL', | |
627 | 'user':'root', | |
628 | $PARSER | |
629 | }, | |
630 | $UPLOAD_DB_DIR, | |
631 | ], | |
632 | $REPORT | |
633 | } | |
634 | ]" | |
635 | ||
636 | # | |
637 | # RocksDB whitebox crash testing under address sanitizer | |
638 | # | |
639 | ASAN_WHITEBOX_CRASH_TEST_COMMANDS="[ | |
640 | { | |
641 | 'name':'Rocksdb whitebox crash test under ASAN', | |
642 | 'oncall':'$ONCALL', | |
643 | 'executeLocal': 'true', | |
644 | 'timeout': 86400, | |
645 | 'steps': [ | |
646 | $CLEANUP_ENV, | |
647 | { | |
648 | 'name':'Build and run RocksDB debug whitebox asan_crash_test', | |
649 | 'timeout': 86400, | |
650 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 whitebox_asan_crash_test || $CONTRUN_NAME=whitebox_asan_crash_test $TASK_CREATION_TOOL', | |
651 | 'user':'root', | |
652 | $PARSER | |
653 | }, | |
654 | $UPLOAD_DB_DIR, | |
655 | ], | |
656 | $REPORT | |
657 | } | |
658 | ]" | |
659 | ||
494da23a TL |
660 | # |
661 | # RocksDB crash testing with atomic flush under address sanitizer | |
662 | # | |
663 | ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
664 | { | |
f67539c2 | 665 | 'name':'Rocksdb crash test with atomic flush under ASAN', |
494da23a | 666 | 'oncall':'$ONCALL', |
f67539c2 | 667 | 'executeLocal': 'true', |
494da23a TL |
668 | 'timeout': 86400, |
669 | 'steps': [ | |
670 | $CLEANUP_ENV, | |
671 | { | |
672 | 'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush', | |
673 | 'timeout': 86400, | |
674 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
675 | 'user':'root', | |
676 | $PARSER | |
677 | }, | |
f67539c2 TL |
678 | $UPLOAD_DB_DIR, |
679 | ], | |
680 | $REPORT | |
681 | } | |
682 | ]" | |
683 | ||
684 | # | |
685 | # RocksDB crash testing with txn under address sanitizer | |
686 | # | |
687 | ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[ | |
688 | { | |
689 | 'name':'Rocksdb crash test with txn under ASAN', | |
690 | 'oncall':'$ONCALL', | |
691 | 'executeLocal': 'true', | |
692 | 'timeout': 86400, | |
693 | 'steps': [ | |
694 | $CLEANUP_ENV, | |
695 | { | |
696 | 'name':'Build and run RocksDB debug asan_crash_test_with_txn', | |
697 | 'timeout': 86400, | |
698 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL', | |
699 | 'user':'root', | |
700 | $PARSER | |
701 | }, | |
702 | $UPLOAD_DB_DIR, | |
494da23a TL |
703 | ], |
704 | $REPORT | |
705 | } | |
706 | ]" | |
707 | ||
7c673cae FG |
708 | # |
709 | # RocksDB test under undefined behavior sanitizer | |
710 | # | |
711 | UBSAN_TEST_COMMANDS="[ | |
712 | { | |
713 | 'name':'Rocksdb Unit Test under UBSAN', | |
714 | 'oncall':'$ONCALL', | |
f67539c2 | 715 | 'executeLocal': 'true', |
7c673cae FG |
716 | 'steps': [ |
717 | $CLEANUP_ENV, | |
718 | { | |
719 | 'name':'Test RocksDB debug under UBSAN', | |
f67539c2 | 720 | 'shell':'set -o pipefail && $SHM $UBSAN $CLANG $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL', |
7c673cae FG |
721 | 'user':'root', |
722 | $PARSER | |
723 | } | |
724 | ], | |
725 | $REPORT | |
726 | } | |
727 | ]" | |
728 | ||
729 | # | |
20effc67 | 730 | # RocksDB crash testing under undefined behavior sanitizer |
7c673cae FG |
731 | # |
732 | UBSAN_CRASH_TEST_COMMANDS="[ | |
733 | { | |
734 | 'name':'Rocksdb crash test under UBSAN', | |
735 | 'oncall':'$ONCALL', | |
f67539c2 | 736 | 'executeLocal': 'true', |
7c673cae FG |
737 | 'timeout': 86400, |
738 | 'steps': [ | |
739 | $CLEANUP_ENV, | |
740 | { | |
741 | 'name':'Build and run RocksDB debug ubsan_crash_test', | |
742 | 'timeout': 86400, | |
f67539c2 | 743 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
744 | 'user':'root', |
745 | $PARSER | |
746 | }, | |
f67539c2 | 747 | $UPLOAD_DB_DIR, |
7c673cae FG |
748 | ], |
749 | $REPORT | |
750 | } | |
751 | ]" | |
752 | ||
20effc67 TL |
753 | # |
754 | # RocksDB crash testing under undefined behavior sanitizer | |
755 | # | |
756 | UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ | |
757 | { | |
758 | 'name':'Rocksdb blackbox crash test under UBSAN', | |
759 | 'oncall':'$ONCALL', | |
760 | 'executeLocal': 'true', | |
761 | 'timeout': 86400, | |
762 | 'steps': [ | |
763 | $CLEANUP_ENV, | |
764 | { | |
765 | 'name':'Build and run RocksDB debug blackbox ubsan_crash_test', | |
766 | 'timeout': 86400, | |
767 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL', | |
768 | 'user':'root', | |
769 | $PARSER | |
770 | }, | |
771 | $UPLOAD_DB_DIR, | |
772 | ], | |
773 | $REPORT | |
774 | } | |
775 | ]" | |
776 | ||
777 | # | |
778 | # RocksDB crash testing under undefined behavior sanitizer | |
779 | # | |
780 | UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ | |
781 | { | |
782 | 'name':'Rocksdb whitebox crash test under UBSAN', | |
783 | 'oncall':'$ONCALL', | |
784 | 'executeLocal': 'true', | |
785 | 'timeout': 86400, | |
786 | 'steps': [ | |
787 | $CLEANUP_ENV, | |
788 | { | |
789 | 'name':'Build and run RocksDB debug whitebox ubsan_crash_test', | |
790 | 'timeout': 86400, | |
791 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL', | |
792 | 'user':'root', | |
793 | $PARSER | |
794 | }, | |
795 | $UPLOAD_DB_DIR, | |
796 | ], | |
797 | $REPORT | |
798 | } | |
799 | ]" | |
800 | ||
494da23a TL |
801 | # |
802 | # RocksDB crash testing with atomic flush under undefined behavior sanitizer | |
803 | # | |
804 | UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
805 | { | |
f67539c2 | 806 | 'name':'Rocksdb crash test with atomic flush under UBSAN', |
494da23a | 807 | 'oncall':'$ONCALL', |
f67539c2 | 808 | 'executeLocal': 'true', |
494da23a TL |
809 | 'timeout': 86400, |
810 | 'steps': [ | |
811 | $CLEANUP_ENV, | |
812 | { | |
813 | 'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush', | |
814 | 'timeout': 86400, | |
f67539c2 TL |
815 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', |
816 | 'user':'root', | |
817 | $PARSER | |
818 | }, | |
819 | $UPLOAD_DB_DIR, | |
820 | ], | |
821 | $REPORT | |
822 | } | |
823 | ]" | |
824 | ||
825 | # | |
826 | # RocksDB crash testing with txn under undefined behavior sanitizer | |
827 | # | |
828 | UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ | |
829 | { | |
830 | 'name':'Rocksdb crash test with txn under UBSAN', | |
831 | 'oncall':'$ONCALL', | |
832 | 'executeLocal': 'true', | |
833 | 'timeout': 86400, | |
834 | 'steps': [ | |
835 | $CLEANUP_ENV, | |
836 | { | |
837 | 'name':'Build and run RocksDB debug ubsan_crash_test_with_txn', | |
838 | 'timeout': 86400, | |
839 | 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL', | |
494da23a TL |
840 | 'user':'root', |
841 | $PARSER | |
842 | }, | |
f67539c2 | 843 | $UPLOAD_DB_DIR, |
494da23a TL |
844 | ], |
845 | $REPORT | |
846 | } | |
847 | ]" | |
848 | ||
7c673cae FG |
849 | # |
850 | # RocksDB unit test under valgrind | |
851 | # | |
852 | VALGRIND_TEST_COMMANDS="[ | |
853 | { | |
854 | 'name':'Rocksdb Unit Test under valgrind', | |
855 | 'oncall':'$ONCALL', | |
f67539c2 | 856 | 'executeLocal': 'true', |
7c673cae FG |
857 | 'timeout': 86400, |
858 | 'steps': [ | |
859 | $CLEANUP_ENV, | |
860 | { | |
861 | 'name':'Run RocksDB debug unit tests', | |
862 | 'timeout': 86400, | |
863 | 'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL', | |
864 | 'user':'root', | |
865 | $PARSER | |
866 | }, | |
867 | ], | |
868 | $REPORT | |
869 | } | |
870 | ]" | |
871 | ||
872 | # | |
873 | # RocksDB test under TSAN | |
874 | # | |
875 | TSAN_UNIT_TEST_COMMANDS="[ | |
876 | { | |
877 | 'name':'Rocksdb Unit Test under TSAN', | |
878 | 'oncall':'$ONCALL', | |
f67539c2 | 879 | 'executeLocal': 'true', |
7c673cae FG |
880 | 'timeout': 86400, |
881 | 'steps': [ | |
882 | $CLEANUP_ENV, | |
883 | { | |
884 | 'name':'Run RocksDB debug unit test', | |
885 | 'timeout': 86400, | |
886 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL', | |
887 | 'user':'root', | |
888 | $PARSER | |
889 | }, | |
890 | ], | |
891 | $REPORT | |
892 | } | |
893 | ]" | |
894 | ||
895 | # | |
896 | # RocksDB crash test under TSAN | |
897 | # | |
898 | TSAN_CRASH_TEST_COMMANDS="[ | |
899 | { | |
900 | 'name':'Rocksdb Crash Test under TSAN', | |
901 | 'oncall':'$ONCALL', | |
f67539c2 | 902 | 'executeLocal': 'true', |
7c673cae FG |
903 | 'timeout': 86400, |
904 | 'steps': [ | |
905 | $CLEANUP_ENV, | |
906 | { | |
907 | 'name':'Compile and run', | |
908 | 'timeout': 86400, | |
11fdf7f2 | 909 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', |
7c673cae FG |
910 | 'user':'root', |
911 | $PARSER | |
912 | }, | |
f67539c2 | 913 | $UPLOAD_DB_DIR, |
7c673cae FG |
914 | ], |
915 | $REPORT | |
916 | } | |
917 | ]" | |
918 | ||
20effc67 TL |
919 | # |
920 | # RocksDB blackbox crash test under TSAN | |
921 | # | |
922 | TSAN_BLACKBOX_CRASH_TEST_COMMANDS="[ | |
923 | { | |
924 | 'name':'Rocksdb Blackbox Crash Test under TSAN', | |
925 | 'oncall':'$ONCALL', | |
926 | 'executeLocal': 'true', | |
927 | 'timeout': 86400, | |
928 | 'steps': [ | |
929 | $CLEANUP_ENV, | |
930 | { | |
931 | 'name':'Compile and run', | |
932 | 'timeout': 86400, | |
933 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 blackbox_crash_test || $CONTRUN_NAME=tsan_blackbox_crash_test $TASK_CREATION_TOOL', | |
934 | 'user':'root', | |
935 | $PARSER | |
936 | }, | |
937 | $UPLOAD_DB_DIR, | |
938 | ], | |
939 | $REPORT | |
940 | } | |
941 | ]" | |
942 | ||
943 | # | |
944 | # RocksDB whitebox crash test under TSAN | |
945 | # | |
946 | TSAN_WHITEBOX_CRASH_TEST_COMMANDS="[ | |
947 | { | |
948 | 'name':'Rocksdb Whitebox Crash Test under TSAN', | |
949 | 'oncall':'$ONCALL', | |
950 | 'executeLocal': 'true', | |
951 | 'timeout': 86400, | |
952 | 'steps': [ | |
953 | $CLEANUP_ENV, | |
954 | { | |
955 | 'name':'Compile and run', | |
956 | 'timeout': 86400, | |
957 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 whitebox_crash_test || $CONTRUN_NAME=tsan_whitebox_crash_test $TASK_CREATION_TOOL', | |
958 | 'user':'root', | |
959 | $PARSER | |
960 | }, | |
961 | $UPLOAD_DB_DIR, | |
962 | ], | |
963 | $REPORT | |
964 | } | |
965 | ]" | |
966 | ||
494da23a TL |
967 | # |
968 | # RocksDB crash test with atomic flush under TSAN | |
969 | # | |
970 | TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ | |
971 | { | |
972 | 'name':'Rocksdb Crash Test with atomic flush under TSAN', | |
973 | 'oncall':'$ONCALL', | |
f67539c2 | 974 | 'executeLocal': 'true', |
494da23a TL |
975 | 'timeout': 86400, |
976 | 'steps': [ | |
977 | $CLEANUP_ENV, | |
978 | { | |
979 | 'name':'Compile and run', | |
980 | 'timeout': 86400, | |
981 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', | |
982 | 'user':'root', | |
983 | $PARSER | |
984 | }, | |
f67539c2 TL |
985 | $UPLOAD_DB_DIR, |
986 | ], | |
987 | $REPORT | |
988 | } | |
989 | ]" | |
990 | ||
991 | # | |
992 | # RocksDB crash test with txn under TSAN | |
993 | # | |
994 | TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ | |
995 | { | |
996 | 'name':'Rocksdb Crash Test with txn under TSAN', | |
997 | 'oncall':'$ONCALL', | |
998 | 'executeLocal': 'true', | |
999 | 'timeout': 86400, | |
1000 | 'steps': [ | |
1001 | $CLEANUP_ENV, | |
1002 | { | |
1003 | 'name':'Compile and run', | |
1004 | 'timeout': 86400, | |
1005 | 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL', | |
1006 | 'user':'root', | |
1007 | $PARSER | |
1008 | }, | |
1009 | $UPLOAD_DB_DIR, | |
494da23a TL |
1010 | ], |
1011 | $REPORT | |
1012 | } | |
1013 | ]" | |
1014 | ||
7c673cae FG |
1015 | # |
1016 | # RocksDB format compatible | |
1017 | # | |
1018 | ||
1019 | run_format_compatible() | |
1020 | { | |
1021 | export TEST_TMPDIR=/dev/shm/rocksdb | |
1022 | rm -rf /dev/shm/rocksdb | |
1023 | mkdir /dev/shm/rocksdb | |
1024 | ||
20effc67 TL |
1025 | export https_proxy="fwdproxy:8080" |
1026 | ||
11fdf7f2 | 1027 | tools/check_format_compatible.sh |
7c673cae FG |
1028 | } |
1029 | ||
1030 | FORMAT_COMPATIBLE_COMMANDS="[ | |
1031 | { | |
1032 | 'name':'Rocksdb Format Compatible tests', | |
1033 | 'oncall':'$ONCALL', | |
f67539c2 | 1034 | 'executeLocal': 'true', |
7c673cae FG |
1035 | 'steps': [ |
1036 | $CLEANUP_ENV, | |
1037 | { | |
1038 | 'name':'Run RocksDB debug unit test', | |
1039 | 'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL', | |
1040 | 'user':'root', | |
1041 | $PARSER | |
1042 | }, | |
1043 | ], | |
1044 | $REPORT | |
1045 | } | |
1046 | ]" | |
1047 | ||
1048 | # | |
1049 | # RocksDB no compression | |
1050 | # | |
1051 | run_no_compression() | |
1052 | { | |
1053 | export TEST_TMPDIR=/dev/shm/rocksdb | |
1054 | rm -rf /dev/shm/rocksdb | |
1055 | mkdir /dev/shm/rocksdb | |
1056 | make clean | |
494da23a | 1057 | cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh |
7c673cae FG |
1058 | mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh |
1059 | cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile | |
1060 | mv .tmp.Makefile Makefile | |
1061 | make $DEBUG J=1 check | |
1062 | } | |
1063 | ||
1064 | NO_COMPRESSION_COMMANDS="[ | |
1065 | { | |
1066 | 'name':'Rocksdb No Compression tests', | |
1067 | 'oncall':'$ONCALL', | |
f67539c2 | 1068 | 'executeLocal': 'true', |
7c673cae FG |
1069 | 'steps': [ |
1070 | $CLEANUP_ENV, | |
1071 | { | |
1072 | 'name':'Run RocksDB debug unit test', | |
1073 | 'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL', | |
1074 | 'user':'root', | |
1075 | $PARSER | |
1076 | }, | |
1077 | ], | |
1078 | $REPORT | |
1079 | } | |
1080 | ]" | |
1081 | ||
1082 | # | |
1083 | # RocksDB regression | |
1084 | # | |
1085 | run_regression() | |
1086 | { | |
1087 | time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX) | |
1088 | ||
1089 | # ======= report size to ODS ======== | |
1090 | ||
1091 | # parameters: $1 -- key, $2 -- value | |
1092 | function send_size_to_ods { | |
f67539c2 | 1093 | curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ |
7c673cae FG |
1094 | --connect-timeout 60 |
1095 | } | |
1096 | ||
1097 | # === normal build === | |
1098 | make clean | |
1099 | make -j$(nproc) static_lib | |
1100 | send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) | |
1101 | strip librocksdb.a | |
1102 | send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) | |
1103 | ||
20effc67 | 1104 | make clean |
7c673cae FG |
1105 | make -j$(nproc) shared_lib |
1106 | send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) | |
1107 | strip `readlink -f librocksdb.so` | |
1108 | send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) | |
1109 | ||
1110 | # === lite build === | |
1111 | make clean | |
494da23a | 1112 | make LITE=1 -j$(nproc) static_lib |
7c673cae FG |
1113 | send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) |
1114 | strip librocksdb.a | |
1115 | send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) | |
1116 | ||
20effc67 | 1117 | make clean |
494da23a | 1118 | make LITE=1 -j$(nproc) shared_lib |
7c673cae FG |
1119 | send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) |
1120 | strip `readlink -f librocksdb.so` | |
1121 | send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) | |
1122 | } | |
1123 | ||
1124 | REGRESSION_COMMANDS="[ | |
1125 | { | |
1126 | 'name':'Rocksdb regression commands', | |
1127 | 'oncall':'$ONCALL', | |
1128 | 'steps': [ | |
1129 | $CLEANUP_ENV, | |
1130 | { | |
1131 | 'name':'Make and run script', | |
1132 | 'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL', | |
1133 | 'user':'root', | |
1134 | $PARSER | |
1135 | }, | |
1136 | ], | |
1137 | $REPORT | |
1138 | } | |
1139 | ]" | |
1140 | ||
1141 | # | |
1142 | # RocksDB Java build | |
1143 | # | |
1144 | JAVA_BUILD_TEST_COMMANDS="[ | |
1145 | { | |
1146 | 'name':'Rocksdb Java Build', | |
1147 | 'oncall':'$ONCALL', | |
f67539c2 | 1148 | 'executeLocal': 'true', |
7c673cae FG |
1149 | 'steps': [ |
1150 | $CLEANUP_ENV, | |
1151 | { | |
1152 | 'name':'Build RocksDB for Java', | |
1153 | 'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL', | |
1154 | 'user':'root', | |
1155 | $PARSER | |
1156 | }, | |
1157 | ], | |
1158 | $REPORT | |
1159 | } | |
1160 | ]" | |
1161 | ||
1162 | ||
1163 | case $1 in | |
7c673cae FG |
1164 | unit) |
1165 | echo $UNIT_TEST_COMMANDS | |
1166 | ;; | |
1167 | unit_non_shm) | |
1168 | echo $UNIT_TEST_NON_SHM_COMMANDS | |
1169 | ;; | |
1170 | release) | |
1171 | echo $RELEASE_BUILD_COMMANDS | |
1172 | ;; | |
1173 | unit_481) | |
1174 | echo $UNIT_TEST_COMMANDS_481 | |
1175 | ;; | |
1176 | release_481) | |
1177 | echo $RELEASE_BUILD_COMMANDS_481 | |
1178 | ;; | |
1179 | clang_unit) | |
1180 | echo $CLANG_UNIT_TEST_COMMANDS | |
1181 | ;; | |
1182 | clang_release) | |
1183 | echo $CLANG_RELEASE_BUILD_COMMANDS | |
1184 | ;; | |
1185 | clang_analyze) | |
1186 | echo $CLANG_ANALYZE_COMMANDS | |
1187 | ;; | |
1188 | code_cov) | |
1189 | echo $CODE_COV_COMMANDS | |
1190 | ;; | |
1191 | unity) | |
1192 | echo $UNITY_COMMANDS | |
1193 | ;; | |
1194 | lite) | |
1195 | echo $LITE_BUILD_COMMANDS | |
1196 | ;; | |
11fdf7f2 TL |
1197 | report_lite_binary_size) |
1198 | echo $REPORT_LITE_BINARY_SIZE_COMMANDS | |
7c673cae FG |
1199 | ;; |
1200 | stress_crash) | |
1201 | echo $STRESS_CRASH_TEST_COMMANDS | |
1202 | ;; | |
20effc67 TL |
1203 | blackbox_stress_crash) |
1204 | echo $BLACKBOX_STRESS_CRASH_TEST_COMMANDS | |
1205 | ;; | |
1206 | whitebox_stress_crash) | |
1207 | echo $WHITEBOX_STRESS_CRASH_TEST_COMMANDS | |
1208 | ;; | |
494da23a TL |
1209 | stress_crash_with_atomic_flush) |
1210 | echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
1211 | ;; | |
f67539c2 TL |
1212 | stress_crash_with_txn) |
1213 | echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS | |
1214 | ;; | |
7c673cae FG |
1215 | write_stress) |
1216 | echo $WRITE_STRESS_COMMANDS | |
1217 | ;; | |
1218 | asan) | |
1219 | echo $ASAN_TEST_COMMANDS | |
1220 | ;; | |
1221 | asan_crash) | |
1222 | echo $ASAN_CRASH_TEST_COMMANDS | |
1223 | ;; | |
20effc67 TL |
1224 | blackbox_asan_crash) |
1225 | echo $ASAN_BLACKBOX_CRASH_TEST_COMMANDS | |
1226 | ;; | |
1227 | whitebox_asan_crash) | |
1228 | echo $ASAN_WHITEBOX_CRASH_TEST_COMMANDS | |
1229 | ;; | |
494da23a TL |
1230 | asan_crash_with_atomic_flush) |
1231 | echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
1232 | ;; | |
f67539c2 TL |
1233 | asan_crash_with_txn) |
1234 | echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS | |
1235 | ;; | |
7c673cae FG |
1236 | ubsan) |
1237 | echo $UBSAN_TEST_COMMANDS | |
1238 | ;; | |
1239 | ubsan_crash) | |
1240 | echo $UBSAN_CRASH_TEST_COMMANDS | |
1241 | ;; | |
20effc67 TL |
1242 | blackbox_ubsan_crash) |
1243 | echo $UBSAN_BLACKBOX_CRASH_TEST_COMMANDS | |
1244 | ;; | |
1245 | whitebox_ubsan_crash) | |
1246 | echo $UBSAN_WHITEBOX_CRASH_TEST_COMMANDS | |
1247 | ;; | |
494da23a TL |
1248 | ubsan_crash_with_atomic_flush) |
1249 | echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
1250 | ;; | |
f67539c2 TL |
1251 | ubsan_crash_with_txn) |
1252 | echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS | |
1253 | ;; | |
7c673cae FG |
1254 | valgrind) |
1255 | echo $VALGRIND_TEST_COMMANDS | |
1256 | ;; | |
1257 | tsan) | |
1258 | echo $TSAN_UNIT_TEST_COMMANDS | |
1259 | ;; | |
1260 | tsan_crash) | |
1261 | echo $TSAN_CRASH_TEST_COMMANDS | |
1262 | ;; | |
20effc67 TL |
1263 | blackbox_tsan_crash) |
1264 | echo $TSAN_BLACKBOX_CRASH_TEST_COMMANDS | |
1265 | ;; | |
1266 | whitebox_tsan_crash) | |
1267 | echo $TSAN_WHITEBOX_CRASH_TEST_COMMANDS | |
1268 | ;; | |
494da23a TL |
1269 | tsan_crash_with_atomic_flush) |
1270 | echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS | |
1271 | ;; | |
f67539c2 TL |
1272 | tsan_crash_with_txn) |
1273 | echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS | |
1274 | ;; | |
7c673cae FG |
1275 | format_compatible) |
1276 | echo $FORMAT_COMPATIBLE_COMMANDS | |
1277 | ;; | |
1278 | run_format_compatible) | |
1279 | run_format_compatible | |
1280 | ;; | |
1281 | no_compression) | |
1282 | echo $NO_COMPRESSION_COMMANDS | |
1283 | ;; | |
1284 | run_no_compression) | |
1285 | run_no_compression | |
1286 | ;; | |
1287 | regression) | |
1288 | echo $REGRESSION_COMMANDS | |
1289 | ;; | |
1290 | run_regression) | |
1291 | run_regression | |
1292 | ;; | |
1293 | java_build) | |
1294 | echo $JAVA_BUILD_TEST_COMMANDS | |
1295 | ;; | |
1296 | *) | |
1297 | echo "Invalid determinator command" | |
f67539c2 | 1298 | exit 1 |
7c673cae FG |
1299 | ;; |
1300 | esac |