]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/bash -x |
2 | # | |
3 | # Copyright (C) 2014 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: Loic Dachary <loic@dachary.org> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
c07f9fc5 | 17 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh |
7c673cae | 18 | |
224ce89b | 19 | if [ `uname` = FreeBSD ]; then |
c07f9fc5 | 20 | # erasure coding overwrites are only tested on Bluestore |
224ce89b WB |
21 | # erasure coding on filestore is unsafe |
22 | # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites | |
23 | use_ec_overwrite=false | |
24 | else | |
25 | use_ec_overwrite=true | |
26 | fi | |
27 | ||
7c673cae FG |
28 | # Test development and debugging |
29 | # Set to "yes" in order to ignore diff errors and save results to update test | |
30 | getjson="no" | |
31 | ||
94b18763 FG |
32 | # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info. |
33 | jqfilter='def walk(f): | |
34 | . as $in | |
35 | | if type == "object" then | |
36 | reduce keys[] as $key | |
37 | ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f | |
38 | elif type == "array" then map( walk(f) ) | f | |
39 | else f | |
40 | end; | |
41 | walk(if type == "object" then del(.mtime) else . end) | |
42 | | walk(if type == "object" then del(.local_mtime) else . end) | |
43 | | walk(if type == "object" then del(.last_reqid) else . end) | |
44 | | walk(if type == "object" then del(.version) else . end) | |
45 | | walk(if type == "object" then del(.prior_version) else . end) | |
46 | | walk(if type == "object" then del(.redirect_target) else . end) | |
47 | | walk(if type == "object" then del(.legacy_snaps) else . end)' | |
7c673cae | 48 | |
94b18763 | 49 | sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' |
7c673cae FG |
50 | |
51 | function run() { | |
52 | local dir=$1 | |
53 | shift | |
54 | ||
55 | export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one | |
56 | export CEPH_ARGS | |
57 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
58 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
28e407b8 | 59 | CEPH_ARGS+="--osd-skip-data-digest=false " |
7c673cae FG |
60 | |
61 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
62 | for func in $funcs ; do | |
63 | $func $dir || return 1 | |
64 | done | |
65 | } | |
66 | ||
67 | function add_something() { | |
68 | local dir=$1 | |
69 | local poolname=$2 | |
70 | local obj=${3:-SOMETHING} | |
71 | local scrub=${4:-noscrub} | |
72 | ||
73 | if [ "$scrub" = "noscrub" ]; | |
74 | then | |
75 | ceph osd set noscrub || return 1 | |
76 | ceph osd set nodeep-scrub || return 1 | |
77 | else | |
78 | ceph osd unset noscrub || return 1 | |
79 | ceph osd unset nodeep-scrub || return 1 | |
80 | fi | |
81 | ||
82 | local payload=ABCDEF | |
83 | echo $payload > $dir/ORIGINAL | |
84 | rados --pool $poolname put $obj $dir/ORIGINAL || return 1 | |
85 | } | |
86 | ||
87 | # | |
88 | # Corrupt one copy of a replicated pool | |
89 | # | |
90 | function TEST_corrupt_and_repair_replicated() { | |
91 | local dir=$1 | |
92 | local poolname=rbd | |
93 | ||
94 | setup $dir || return 1 | |
95 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
96 | run_mgr $dir x || return 1 | |
97 | run_osd $dir 0 || return 1 | |
98 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 99 | create_rbd_pool || return 1 |
7c673cae FG |
100 | wait_for_clean || return 1 |
101 | ||
102 | add_something $dir $poolname || return 1 | |
103 | corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 | |
104 | # Reproduces http://tracker.ceph.com/issues/8914 | |
105 | corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 | |
106 | ||
107 | teardown $dir || return 1 | |
108 | } | |
109 | ||
110 | function corrupt_and_repair_two() { | |
111 | local dir=$1 | |
112 | local poolname=$2 | |
113 | local first=$3 | |
114 | local second=$4 | |
115 | ||
116 | # | |
117 | # 1) remove the corresponding file from the OSDs | |
118 | # | |
119 | pids="" | |
120 | run_in_background pids objectstore_tool $dir $first SOMETHING remove | |
121 | run_in_background pids objectstore_tool $dir $second SOMETHING remove | |
122 | wait_background pids | |
123 | return_code=$? | |
124 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
125 | ||
126 | # | |
127 | # 2) repair the PG | |
128 | # | |
129 | local pg=$(get_pg $poolname SOMETHING) | |
130 | repair $pg | |
131 | # | |
132 | # 3) The files must be back | |
133 | # | |
134 | pids="" | |
135 | run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs | |
136 | run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs | |
137 | wait_background pids | |
138 | return_code=$? | |
139 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
140 | ||
141 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
142 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
143 | } | |
144 | ||
145 | # | |
146 | # 1) add an object | |
147 | # 2) remove the corresponding file from a designated OSD | |
148 | # 3) repair the PG | |
149 | # 4) check that the file has been restored in the designated OSD | |
150 | # | |
151 | function corrupt_and_repair_one() { | |
152 | local dir=$1 | |
153 | local poolname=$2 | |
154 | local osd=$3 | |
155 | ||
156 | # | |
157 | # 1) remove the corresponding file from the OSD | |
158 | # | |
159 | objectstore_tool $dir $osd SOMETHING remove || return 1 | |
160 | # | |
161 | # 2) repair the PG | |
162 | # | |
163 | local pg=$(get_pg $poolname SOMETHING) | |
164 | repair $pg | |
165 | # | |
166 | # 3) The file must be back | |
167 | # | |
168 | objectstore_tool $dir $osd SOMETHING list-attrs || return 1 | |
169 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
170 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
171 | } | |
172 | ||
173 | function corrupt_and_repair_erasure_coded() { | |
174 | local dir=$1 | |
175 | local poolname=$2 | |
176 | ||
177 | add_something $dir $poolname || return 1 | |
178 | ||
179 | local primary=$(get_primary $poolname SOMETHING) | |
180 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
181 | local not_primary_first=${osds[0]} | |
182 | local not_primary_second=${osds[1]} | |
183 | ||
184 | # Reproduces http://tracker.ceph.com/issues/10017 | |
185 | corrupt_and_repair_one $dir $poolname $primary || return 1 | |
186 | # Reproduces http://tracker.ceph.com/issues/10409 | |
187 | corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 | |
188 | corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 | |
189 | corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 | |
190 | ||
191 | } | |
192 | ||
193 | function create_ec_pool() { | |
194 | local pool_name=$1 | |
195 | local allow_overwrites=$2 | |
196 | ||
224ce89b | 197 | ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1 |
7c673cae | 198 | |
b5b8bbf5 | 199 | create_pool "$poolname" 1 1 erasure myprofile || return 1 |
7c673cae FG |
200 | |
201 | if [ "$allow_overwrites" = "true" ]; then | |
202 | ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 | |
203 | fi | |
204 | ||
205 | wait_for_clean || return 1 | |
206 | return 0 | |
207 | } | |
208 | ||
209 | function auto_repair_erasure_coded() { | |
210 | local dir=$1 | |
211 | local allow_overwrites=$2 | |
212 | local poolname=ecpool | |
213 | ||
214 | # Launch a cluster with 5 seconds scrub interval | |
215 | setup $dir || return 1 | |
216 | run_mon $dir a || return 1 | |
217 | run_mgr $dir x || return 1 | |
218 | local ceph_osd_args="--osd-scrub-auto-repair=true \ | |
219 | --osd-deep-scrub-interval=5 \ | |
220 | --osd-scrub-max-interval=5 \ | |
221 | --osd-scrub-min-interval=5 \ | |
222 | --osd-scrub-interval-randomize-ratio=0" | |
223 | for id in $(seq 0 2) ; do | |
224 | if [ "$allow_overwrites" = "true" ]; then | |
225 | run_osd_bluestore $dir $id $ceph_osd_args || return 1 | |
226 | else | |
227 | run_osd $dir $id $ceph_osd_args || return 1 | |
228 | fi | |
229 | done | |
c07f9fc5 | 230 | create_rbd_pool || return 1 |
7c673cae FG |
231 | wait_for_clean || return 1 |
232 | ||
233 | # Create an EC pool | |
234 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
235 | ||
236 | # Put an object | |
237 | local payload=ABCDEF | |
238 | echo $payload > $dir/ORIGINAL | |
239 | rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 | |
240 | ||
241 | # Remove the object from one shard physically | |
242 | # Restarted osd get $ceph_osd_args passed | |
243 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 | |
244 | # Wait for auto repair | |
245 | local pgid=$(get_pg $poolname SOMETHING) | |
246 | wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" | |
247 | wait_for_clean || return 1 | |
248 | # Verify - the file should be back | |
249 | # Restarted osd get $ceph_osd_args passed | |
250 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 | |
251 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
252 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
253 | ||
254 | # Tear down | |
255 | teardown $dir || return 1 | |
256 | } | |
257 | ||
258 | function TEST_auto_repair_erasure_coded_appends() { | |
259 | auto_repair_erasure_coded $1 false | |
260 | } | |
261 | ||
262 | function TEST_auto_repair_erasure_coded_overwrites() { | |
224ce89b WB |
263 | if [ "$use_ec_overwrite" = "true" ]; then |
264 | auto_repair_erasure_coded $1 true | |
265 | fi | |
7c673cae FG |
266 | } |
267 | ||
268 | function corrupt_and_repair_jerasure() { | |
269 | local dir=$1 | |
270 | local allow_overwrites=$2 | |
271 | local poolname=ecpool | |
272 | ||
273 | setup $dir || return 1 | |
274 | run_mon $dir a || return 1 | |
275 | run_mgr $dir x || return 1 | |
276 | for id in $(seq 0 3) ; do | |
277 | if [ "$allow_overwrites" = "true" ]; then | |
278 | run_osd_bluestore $dir $id || return 1 | |
279 | else | |
280 | run_osd $dir $id || return 1 | |
281 | fi | |
282 | done | |
c07f9fc5 | 283 | create_rbd_pool || return 1 |
7c673cae FG |
284 | wait_for_clean || return 1 |
285 | ||
286 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
287 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
288 | ||
289 | teardown $dir || return 1 | |
290 | } | |
291 | ||
292 | function TEST_corrupt_and_repair_jerasure_appends() { | |
293 | corrupt_and_repair_jerasure $1 | |
294 | } | |
295 | ||
296 | function TEST_corrupt_and_repair_jerasure_overwrites() { | |
224ce89b WB |
297 | if [ "$use_ec_overwrite" = "true" ]; then |
298 | corrupt_and_repair_jerasure $1 true | |
299 | fi | |
7c673cae FG |
300 | } |
301 | ||
302 | function corrupt_and_repair_lrc() { | |
303 | local dir=$1 | |
304 | local allow_overwrites=$2 | |
305 | local poolname=ecpool | |
306 | ||
307 | setup $dir || return 1 | |
308 | run_mon $dir a || return 1 | |
309 | run_mgr $dir x || return 1 | |
310 | for id in $(seq 0 9) ; do | |
311 | if [ "$allow_overwrites" = "true" ]; then | |
312 | run_osd_bluestore $dir $id || return 1 | |
313 | else | |
314 | run_osd $dir $id || return 1 | |
315 | fi | |
316 | done | |
c07f9fc5 | 317 | create_rbd_pool || return 1 |
7c673cae FG |
318 | wait_for_clean || return 1 |
319 | ||
320 | create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 | |
321 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
322 | ||
323 | teardown $dir || return 1 | |
324 | } | |
325 | ||
326 | function TEST_corrupt_and_repair_lrc_appends() { | |
327 | corrupt_and_repair_jerasure $1 | |
328 | } | |
329 | ||
330 | function TEST_corrupt_and_repair_lrc_overwrites() { | |
224ce89b WB |
331 | if [ "$use_ec_overwrite" = "true" ]; then |
332 | corrupt_and_repair_jerasure $1 true | |
333 | fi | |
7c673cae FG |
334 | } |
335 | ||
336 | function unfound_erasure_coded() { | |
337 | local dir=$1 | |
338 | local allow_overwrites=$2 | |
339 | local poolname=ecpool | |
340 | local payload=ABCDEF | |
341 | ||
342 | setup $dir || return 1 | |
343 | run_mon $dir a || return 1 | |
344 | run_mgr $dir x || return 1 | |
345 | for id in $(seq 0 3) ; do | |
346 | if [ "$allow_overwrites" = "true" ]; then | |
347 | run_osd_bluestore $dir $id || return 1 | |
348 | else | |
349 | run_osd $dir $id || return 1 | |
350 | fi | |
351 | done | |
c07f9fc5 | 352 | create_rbd_pool || return 1 |
7c673cae FG |
353 | wait_for_clean || return 1 |
354 | ||
355 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
356 | ||
357 | add_something $dir $poolname || return 1 | |
358 | ||
359 | local primary=$(get_primary $poolname SOMETHING) | |
360 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
361 | local not_primary_first=${osds[0]} | |
362 | local not_primary_second=${osds[1]} | |
363 | local not_primary_third=${osds[2]} | |
364 | ||
365 | # | |
366 | # 1) remove the corresponding file from the OSDs | |
367 | # | |
368 | pids="" | |
369 | run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove | |
370 | run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove | |
371 | run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove | |
372 | wait_background pids | |
373 | return_code=$? | |
374 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
375 | ||
376 | # | |
377 | # 2) repair the PG | |
378 | # | |
379 | local pg=$(get_pg $poolname SOMETHING) | |
380 | repair $pg | |
381 | # | |
382 | # 3) check pg state | |
383 | # | |
384 | # it may take a bit to appear due to mon/mgr asynchrony | |
385 | for f in `seq 1 60`; do | |
b5b8bbf5 | 386 | ceph -s | grep "1/1 objects unfound" && break |
7c673cae FG |
387 | sleep 1 |
388 | done | |
389 | ceph -s|grep "4 osds: 4 up, 4 in" || return 1 | |
b5b8bbf5 | 390 | ceph -s|grep "1/1 objects unfound" || return 1 |
7c673cae FG |
391 | |
392 | teardown $dir || return 1 | |
393 | } | |
394 | ||
395 | function TEST_unfound_erasure_coded_appends() { | |
396 | unfound_erasure_coded $1 | |
397 | } | |
398 | ||
399 | function TEST_unfound_erasure_coded_overwrites() { | |
224ce89b WB |
400 | if [ "$use_ec_overwrite" = "true" ]; then |
401 | unfound_erasure_coded $1 true | |
402 | fi | |
7c673cae FG |
403 | } |
404 | ||
405 | # | |
406 | # list_missing for EC pool | |
407 | # | |
408 | function list_missing_erasure_coded() { | |
409 | local dir=$1 | |
410 | local allow_overwrites=$2 | |
411 | local poolname=ecpool | |
412 | ||
413 | setup $dir || return 1 | |
414 | run_mon $dir a || return 1 | |
415 | run_mgr $dir x || return 1 | |
416 | for id in $(seq 0 2) ; do | |
417 | if [ "$allow_overwrites" = "true" ]; then | |
418 | run_osd_bluestore $dir $id || return 1 | |
419 | else | |
420 | run_osd $dir $id || return 1 | |
421 | fi | |
422 | done | |
c07f9fc5 | 423 | create_rbd_pool || return 1 |
7c673cae FG |
424 | wait_for_clean || return 1 |
425 | ||
426 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
427 | ||
428 | # Put an object and remove the two shards (including primary) | |
429 | add_something $dir $poolname MOBJ0 || return 1 | |
430 | local -a osds0=($(get_osds $poolname MOBJ0)) | |
431 | ||
432 | # Put another object and remove two shards (excluding primary) | |
433 | add_something $dir $poolname MOBJ1 || return 1 | |
434 | local -a osds1=($(get_osds $poolname MOBJ1)) | |
435 | ||
436 | # Stop all osd daemons | |
437 | for id in $(seq 0 2) ; do | |
438 | kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 | |
439 | done | |
440 | ||
441 | id=${osds0[0]} | |
31f18b77 | 442 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
443 | MOBJ0 remove || return 1 |
444 | id=${osds0[1]} | |
31f18b77 | 445 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
446 | MOBJ0 remove || return 1 |
447 | ||
448 | id=${osds1[1]} | |
31f18b77 | 449 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
450 | MOBJ1 remove || return 1 |
451 | id=${osds1[2]} | |
31f18b77 | 452 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
453 | MOBJ1 remove || return 1 |
454 | ||
455 | for id in $(seq 0 2) ; do | |
456 | activate_osd $dir $id >&2 || return 1 | |
457 | done | |
c07f9fc5 | 458 | create_rbd_pool || return 1 |
7c673cae FG |
459 | wait_for_clean || return 1 |
460 | ||
461 | # Get get - both objects should in the same PG | |
462 | local pg=$(get_pg $poolname MOBJ0) | |
463 | ||
464 | # Repair the PG, which triggers the recovering, | |
465 | # and should mark the object as unfound | |
466 | repair $pg | |
c07f9fc5 | 467 | |
7c673cae FG |
468 | for i in $(seq 0 120) ; do |
469 | [ $i -lt 60 ] || return 1 | |
470 | matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) | |
471 | [ $matches -eq 2 ] && break | |
472 | done | |
473 | ||
474 | teardown $dir || return 1 | |
475 | } | |
476 | ||
477 | function TEST_list_missing_erasure_coded_appends() { | |
478 | list_missing_erasure_coded $1 false | |
479 | } | |
480 | ||
481 | function TEST_list_missing_erasure_coded_overwrites() { | |
224ce89b WB |
482 | if [ "$use_ec_overwrite" = "true" ]; then |
483 | list_missing_erasure_coded $1 true | |
484 | fi | |
7c673cae FG |
485 | } |
486 | ||
487 | # | |
488 | # Corrupt one copy of a replicated pool | |
489 | # | |
490 | function TEST_corrupt_scrub_replicated() { | |
491 | local dir=$1 | |
492 | local poolname=csr_pool | |
1adf2230 | 493 | local total_objs=18 |
7c673cae FG |
494 | |
495 | setup $dir || return 1 | |
496 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
497 | run_mgr $dir x || return 1 | |
498 | run_osd $dir 0 || return 1 | |
499 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 500 | create_rbd_pool || return 1 |
7c673cae FG |
501 | wait_for_clean || return 1 |
502 | ||
b5b8bbf5 FG |
503 | create_pool foo 1 || return 1 |
504 | create_pool $poolname 1 1 || return 1 | |
7c673cae FG |
505 | wait_for_clean || return 1 |
506 | ||
507 | for i in $(seq 1 $total_objs) ; do | |
508 | objname=ROBJ${i} | |
509 | add_something $dir $poolname $objname || return 1 | |
510 | ||
511 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
512 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
31f18b77 FG |
513 | done |
514 | ||
515 | local pg=$(get_pg $poolname ROBJ0) | |
91327a77 | 516 | local primary=$(get_primary $poolname ROBJ0) |
31f18b77 FG |
517 | |
518 | # Compute an old omap digest and save oi | |
c07f9fc5 | 519 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \ |
31f18b77 | 520 | config set osd_deep_scrub_update_digest_min_age 0 |
c07f9fc5 | 521 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \ |
31f18b77 FG |
522 | config set osd_deep_scrub_update_digest_min_age 0 |
523 | pg_deep_scrub $pg | |
524 | ||
525 | for i in $(seq 1 $total_objs) ; do | |
526 | objname=ROBJ${i} | |
7c673cae FG |
527 | |
528 | # Alternate corruption between osd.0 and osd.1 | |
529 | local osd=$(expr $i % 2) | |
530 | ||
531 | case $i in | |
532 | 1) | |
533 | # Size (deep scrub data_digest too) | |
534 | local payload=UVWXYZZZ | |
535 | echo $payload > $dir/CORRUPT | |
536 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
537 | ;; | |
538 | ||
539 | 2) | |
540 | # digest (deep scrub only) | |
541 | local payload=UVWXYZ | |
542 | echo $payload > $dir/CORRUPT | |
543 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
544 | ;; | |
545 | ||
546 | 3) | |
547 | # missing | |
548 | objectstore_tool $dir $osd $objname remove || return 1 | |
549 | ;; | |
550 | ||
551 | 4) | |
552 | # Modify omap value (deep scrub only) | |
553 | objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 | |
554 | ;; | |
555 | ||
556 | 5) | |
557 | # Delete omap key (deep scrub only) | |
558 | objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 | |
559 | ;; | |
560 | ||
561 | 6) | |
562 | # Add extra omap key (deep scrub only) | |
563 | echo extra > $dir/extra-val | |
564 | objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 | |
565 | rm $dir/extra-val | |
566 | ;; | |
567 | ||
568 | 7) | |
569 | # Modify omap header (deep scrub only) | |
570 | echo -n newheader > $dir/hdr | |
571 | objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 | |
572 | rm $dir/hdr | |
573 | ;; | |
574 | ||
575 | 8) | |
576 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
577 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
578 | ||
579 | # Break xattrs | |
580 | echo -n bad-val > $dir/bad-val | |
581 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
582 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
583 | echo -n val3-$objname > $dir/newval | |
584 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
585 | rm $dir/bad-val $dir/newval | |
586 | ;; | |
587 | ||
588 | 9) | |
589 | objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi | |
590 | echo -n D > $dir/change | |
591 | rados --pool $poolname put $objname $dir/change | |
592 | objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi | |
593 | rm $dir/oi $dir/change | |
594 | ;; | |
595 | ||
596 | # ROBJ10 must be handled after digests are re-computed by a deep scrub below | |
597 | # ROBJ11 must be handled with config change before deep scrub | |
598 | # ROBJ12 must be handled with config change before scrubs | |
599 | # ROBJ13 must be handled before scrubs | |
600 | ||
601 | 14) | |
602 | echo -n bad-val > $dir/bad-val | |
603 | objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 | |
604 | objectstore_tool $dir 1 $objname rm-attr _ || return 1 | |
605 | rm $dir/bad-val | |
606 | ;; | |
607 | ||
608 | 15) | |
609 | objectstore_tool $dir $osd $objname rm-attr _ || return 1 | |
b5b8bbf5 FG |
610 | ;; |
611 | ||
612 | 16) | |
613 | objectstore_tool $dir 0 $objname rm-attr snapset || return 1 | |
614 | echo -n bad-val > $dir/bad-val | |
615 | objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1 | |
1adf2230 AA |
616 | ;; |
617 | ||
618 | 17) | |
619 | # Deep-scrub only (all replicas are diffent than the object info | |
620 | local payload=ROBJ17 | |
621 | echo $payload > $dir/new.ROBJ17 | |
622 | objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ17 || return 1 | |
623 | objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ17 || return 1 | |
624 | ;; | |
625 | ||
626 | 18) | |
627 | # Deep-scrub only (all replicas are diffent than the object info | |
628 | local payload=ROBJ18 | |
629 | echo $payload > $dir/new.ROBJ18 | |
630 | objectstore_tool $dir 0 $objname set-bytes $dir/new.ROBJ18 || return 1 | |
631 | objectstore_tool $dir 1 $objname set-bytes $dir/new.ROBJ18 || return 1 | |
632 | # Make one replica have a different object info, so a full repair must happen too | |
633 | objectstore_tool $dir $osd $objname corrupt-info || return 1 | |
7c673cae FG |
634 | |
635 | esac | |
636 | done | |
637 | ||
638 | local pg=$(get_pg $poolname ROBJ0) | |
639 | ||
b5b8bbf5 FG |
640 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
641 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
642 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
643 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
644 | |
645 | pg_scrub $pg | |
646 | ||
91327a77 AA |
647 | ERRORS=0 |
648 | declare -a s_err_strings | |
649 | err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key" | |
650 | err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : object info inconsistent " | |
651 | err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset" | |
652 | err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key" | |
653 | err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info" | |
654 | err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'" | |
655 | err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error" | |
656 | err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key" | |
657 | err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info" | |
658 | err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info" | |
659 | err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch" | |
660 | err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0" | |
661 | err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error" | |
662 | err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing" | |
663 | err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : candidate size 1 info size 7 mismatch" | |
664 | err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent " | |
665 | err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr" | |
666 | err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97" | |
667 | err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 113/120 bytes, 0/0 hit_set_archive bytes." | |
668 | err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 1 missing, 7 inconsistent objects" | |
669 | err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 17 errors" | |
670 | ||
671 | for err_string in "${err_strings[@]}" | |
672 | do | |
673 | if ! grep -q "$err_string" $dir/osd.${primary}.log | |
674 | then | |
675 | echo "Missing log message '$err_string'" | |
676 | ERRORS=$(expr $ERRORS + 1) | |
677 | fi | |
678 | done | |
679 | ||
7c673cae FG |
680 | rados list-inconsistent-pg $poolname > $dir/json || return 1 |
681 | # Check pg count | |
682 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
683 | # Check pgid | |
684 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
685 | ||
686 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
687 | # Get epoch for repair-get requests | |
688 | epoch=$(jq .epoch $dir/json) | |
689 | ||
94b18763 | 690 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
691 | { |
692 | "inconsistents": [ | |
693 | { | |
694 | "shards": [ | |
695 | { | |
696 | "size": 7, | |
697 | "errors": [], | |
b5b8bbf5 FG |
698 | "osd": 0, |
699 | "primary": false | |
7c673cae FG |
700 | }, |
701 | { | |
94b18763 FG |
702 | "object_info": { |
703 | "oid": { | |
704 | "oid": "ROBJ1", | |
705 | "key": "", | |
706 | "snapid": -2, | |
707 | "hash": 1454963827, | |
708 | "max": 0, | |
709 | "pool": 3, | |
710 | "namespace": "" | |
711 | }, | |
712 | "version": "51'58", | |
713 | "prior_version": "21'3", | |
714 | "last_reqid": "osd.1.0:57", | |
715 | "user_version": 3, | |
716 | "size": 7, | |
717 | "mtime": "", | |
718 | "local_mtime": "", | |
719 | "lost": 0, | |
720 | "flags": [ | |
721 | "dirty", | |
722 | "omap", | |
723 | "data_digest", | |
724 | "omap_digest" | |
725 | ], | |
726 | "truncate_seq": 0, | |
727 | "truncate_size": 0, | |
728 | "data_digest": "0x2ddbf8f5", | |
729 | "omap_digest": "0xf5fba2c6", | |
730 | "expected_object_size": 0, | |
731 | "expected_write_size": 0, | |
732 | "alloc_hint_flags": 0, | |
733 | "manifest": { | |
734 | "type": 0 | |
735 | }, | |
736 | "watchers": {} | |
737 | }, | |
7c673cae FG |
738 | "size": 9, |
739 | "errors": [ | |
94b18763 FG |
740 | "size_mismatch_info", |
741 | "obj_size_info_mismatch" | |
7c673cae | 742 | ], |
b5b8bbf5 FG |
743 | "osd": 1, |
744 | "primary": true | |
7c673cae FG |
745 | } |
746 | ], | |
94b18763 FG |
747 | "selected_object_info": { |
748 | "oid": { | |
749 | "oid": "ROBJ1", | |
750 | "key": "", | |
751 | "snapid": -2, | |
752 | "hash": 1454963827, | |
753 | "max": 0, | |
754 | "pool": 3, | |
755 | "namespace": "" | |
756 | }, | |
757 | "version": "51'58", | |
758 | "prior_version": "21'3", | |
759 | "last_reqid": "osd.1.0:57", | |
760 | "user_version": 3, | |
761 | "size": 7, | |
762 | "mtime": "2018-04-05 14:33:19.804040", | |
763 | "local_mtime": "2018-04-05 14:33:19.804839", | |
764 | "lost": 0, | |
765 | "flags": [ | |
766 | "dirty", | |
767 | "omap", | |
768 | "data_digest", | |
769 | "omap_digest" | |
770 | ], | |
771 | "truncate_seq": 0, | |
772 | "truncate_size": 0, | |
773 | "data_digest": "0x2ddbf8f5", | |
774 | "omap_digest": "0xf5fba2c6", | |
775 | "expected_object_size": 0, | |
776 | "expected_write_size": 0, | |
777 | "alloc_hint_flags": 0, | |
778 | "manifest": { | |
779 | "type": 0 | |
780 | }, | |
781 | "watchers": {} | |
782 | }, | |
7c673cae | 783 | "union_shard_errors": [ |
94b18763 FG |
784 | "size_mismatch_info", |
785 | "obj_size_info_mismatch" | |
7c673cae FG |
786 | ], |
787 | "errors": [ | |
788 | "size_mismatch" | |
789 | ], | |
790 | "object": { | |
791 | "version": 3, | |
792 | "snap": "head", | |
793 | "locator": "", | |
794 | "nspace": "", | |
795 | "name": "ROBJ1" | |
796 | } | |
797 | }, | |
798 | { | |
799 | "shards": [ | |
800 | { | |
801 | "errors": [ | |
802 | "stat_error" | |
803 | ], | |
b5b8bbf5 FG |
804 | "osd": 0, |
805 | "primary": false | |
7c673cae FG |
806 | }, |
807 | { | |
808 | "size": 7, | |
809 | "errors": [], | |
b5b8bbf5 FG |
810 | "osd": 1, |
811 | "primary": true | |
7c673cae FG |
812 | } |
813 | ], | |
94b18763 FG |
814 | "selected_object_info": { |
815 | "oid": { | |
816 | "oid": "ROBJ12", | |
817 | "key": "", | |
818 | "snapid": -2, | |
819 | "hash": 3920199997, | |
820 | "max": 0, | |
821 | "pool": 3, | |
822 | "namespace": "" | |
823 | }, | |
824 | "version": "51'56", | |
825 | "prior_version": "43'36", | |
826 | "last_reqid": "osd.1.0:55", | |
827 | "user_version": 36, | |
828 | "size": 7, | |
829 | "mtime": "", | |
830 | "local_mtime": "", | |
831 | "lost": 0, | |
832 | "flags": [ | |
833 | "dirty", | |
834 | "omap", | |
835 | "data_digest", | |
836 | "omap_digest" | |
837 | ], | |
838 | "truncate_seq": 0, | |
839 | "truncate_size": 0, | |
840 | "data_digest": "0x2ddbf8f5", | |
841 | "omap_digest": "0x067f306a", | |
842 | "expected_object_size": 0, | |
843 | "expected_write_size": 0, | |
844 | "alloc_hint_flags": 0, | |
845 | "manifest": { | |
846 | "type": 0 | |
847 | }, | |
848 | "watchers": {} | |
849 | }, | |
7c673cae FG |
850 | "union_shard_errors": [ |
851 | "stat_error" | |
852 | ], | |
853 | "errors": [], | |
854 | "object": { | |
31f18b77 | 855 | "version": 36, |
7c673cae FG |
856 | "snap": "head", |
857 | "locator": "", | |
858 | "nspace": "", | |
859 | "name": "ROBJ12" | |
860 | } | |
861 | }, | |
862 | { | |
863 | "shards": [ | |
864 | { | |
865 | "errors": [ | |
866 | "stat_error" | |
867 | ], | |
b5b8bbf5 FG |
868 | "osd": 0, |
869 | "primary": false | |
7c673cae FG |
870 | }, |
871 | { | |
872 | "size": 7, | |
873 | "errors": [], | |
b5b8bbf5 FG |
874 | "osd": 1, |
875 | "primary": true | |
7c673cae FG |
876 | } |
877 | ], | |
94b18763 FG |
878 | "selected_object_info": { |
879 | "oid": { | |
880 | "oid": "ROBJ13", | |
881 | "key": "", | |
882 | "snapid": -2, | |
883 | "hash": 2682806379, | |
884 | "max": 0, | |
885 | "pool": 3, | |
886 | "namespace": "" | |
887 | }, | |
888 | "version": "51'59", | |
889 | "prior_version": "45'39", | |
890 | "last_reqid": "osd.1.0:58", | |
891 | "user_version": 39, | |
892 | "size": 7, | |
893 | "mtime": "", | |
894 | "local_mtime": "", | |
895 | "lost": 0, | |
896 | "flags": [ | |
897 | "dirty", | |
898 | "omap", | |
899 | "data_digest", | |
900 | "omap_digest" | |
901 | ], | |
902 | "truncate_seq": 0, | |
903 | "truncate_size": 0, | |
904 | "data_digest": "0x2ddbf8f5", | |
905 | "omap_digest": "0x6441854d", | |
906 | "expected_object_size": 0, | |
907 | "expected_write_size": 0, | |
908 | "alloc_hint_flags": 0, | |
909 | "manifest": { | |
910 | "type": 0 | |
911 | }, | |
912 | "watchers": {} | |
913 | }, | |
7c673cae FG |
914 | "union_shard_errors": [ |
915 | "stat_error" | |
916 | ], | |
917 | "errors": [], | |
918 | "object": { | |
31f18b77 | 919 | "version": 39, |
7c673cae FG |
920 | "snap": "head", |
921 | "locator": "", | |
922 | "nspace": "", | |
923 | "name": "ROBJ13" | |
924 | } | |
925 | }, | |
926 | { | |
927 | "shards": [ | |
928 | { | |
94b18763 | 929 | "object_info": "bad-val", |
7c673cae FG |
930 | "size": 7, |
931 | "errors": [ | |
94b18763 | 932 | "info_corrupted" |
7c673cae | 933 | ], |
b5b8bbf5 FG |
934 | "osd": 0, |
935 | "primary": false | |
7c673cae FG |
936 | }, |
937 | { | |
938 | "size": 7, | |
939 | "errors": [ | |
94b18763 | 940 | "info_missing" |
7c673cae | 941 | ], |
b5b8bbf5 FG |
942 | "osd": 1, |
943 | "primary": true | |
7c673cae FG |
944 | } |
945 | ], | |
946 | "union_shard_errors": [ | |
94b18763 FG |
947 | "info_missing", |
948 | "info_corrupted" | |
7c673cae FG |
949 | ], |
950 | "errors": [], | |
951 | "object": { | |
952 | "version": 0, | |
953 | "snap": "head", | |
954 | "locator": "", | |
955 | "nspace": "", | |
956 | "name": "ROBJ14" | |
957 | } | |
958 | }, | |
959 | { | |
960 | "shards": [ | |
961 | { | |
94b18763 FG |
962 | "object_info": { |
963 | "oid": { | |
964 | "oid": "ROBJ15", | |
965 | "key": "", | |
966 | "snapid": -2, | |
967 | "hash": 504996876, | |
968 | "max": 0, | |
969 | "pool": 3, | |
970 | "namespace": "" | |
7c673cae | 971 | }, |
94b18763 FG |
972 | "version": "51'49", |
973 | "prior_version": "49'45", | |
974 | "last_reqid": "osd.1.0:48", | |
975 | "user_version": 45, | |
976 | "size": 7, | |
977 | "mtime": "2018-04-05 14:33:29.498969", | |
978 | "local_mtime": "2018-04-05 14:33:29.499890", | |
979 | "lost": 0, | |
980 | "flags": [ | |
981 | "dirty", | |
982 | "omap", | |
983 | "data_digest", | |
984 | "omap_digest" | |
985 | ], | |
986 | "truncate_seq": 0, | |
987 | "truncate_size": 0, | |
988 | "data_digest": "0x2ddbf8f5", | |
989 | "omap_digest": "0x2d2a4d6e", | |
990 | "expected_object_size": 0, | |
991 | "expected_write_size": 0, | |
992 | "alloc_hint_flags": 0, | |
993 | "manifest": { | |
994 | "type": 0 | |
995 | }, | |
996 | "watchers": {} | |
997 | }, | |
7c673cae FG |
998 | "size": 7, |
999 | "errors": [], | |
b5b8bbf5 FG |
1000 | "osd": 0, |
1001 | "primary": false | |
7c673cae FG |
1002 | }, |
1003 | { | |
7c673cae FG |
1004 | "size": 7, |
1005 | "errors": [ | |
94b18763 | 1006 | "info_missing" |
7c673cae | 1007 | ], |
b5b8bbf5 FG |
1008 | "osd": 1, |
1009 | "primary": true | |
7c673cae FG |
1010 | } |
1011 | ], | |
94b18763 FG |
1012 | "selected_object_info": { |
1013 | "oid": { | |
1014 | "oid": "ROBJ15", | |
1015 | "key": "", | |
1016 | "snapid": -2, | |
1017 | "hash": 504996876, | |
1018 | "max": 0, | |
1019 | "pool": 3, | |
1020 | "namespace": "" | |
1021 | }, | |
1022 | "version": "51'49", | |
1023 | "prior_version": "49'45", | |
1024 | "last_reqid": "osd.1.0:48", | |
1025 | "user_version": 45, | |
1026 | "size": 7, | |
1027 | "mtime": "", | |
1028 | "local_mtime": "", | |
1029 | "lost": 0, | |
1030 | "flags": [ | |
1031 | "dirty", | |
1032 | "omap", | |
1033 | "data_digest", | |
1034 | "omap_digest" | |
1035 | ], | |
1036 | "truncate_seq": 0, | |
1037 | "truncate_size": 0, | |
1038 | "data_digest": "0x2ddbf8f5", | |
1039 | "omap_digest": "0x2d2a4d6e", | |
1040 | "expected_object_size": 0, | |
1041 | "expected_write_size": 0, | |
1042 | "alloc_hint_flags": 0, | |
1043 | "manifest": { | |
1044 | "type": 0 | |
1045 | }, | |
1046 | "watchers": {} | |
1047 | }, | |
7c673cae | 1048 | "union_shard_errors": [ |
94b18763 | 1049 | "info_missing" |
7c673cae | 1050 | ], |
b5b8bbf5 | 1051 | "errors": [], |
7c673cae | 1052 | "object": { |
31f18b77 | 1053 | "version": 45, |
7c673cae FG |
1054 | "snap": "head", |
1055 | "locator": "", | |
1056 | "nspace": "", | |
1057 | "name": "ROBJ15" | |
1058 | } | |
1059 | }, | |
b5b8bbf5 FG |
1060 | { |
1061 | "errors": [], | |
1062 | "object": { | |
1063 | "locator": "", | |
1064 | "name": "ROBJ16", | |
1065 | "nspace": "", | |
1066 | "snap": "head", | |
1067 | "version": 0 | |
1068 | }, | |
1069 | "shards": [ | |
1070 | { | |
b5b8bbf5 | 1071 | "errors": [ |
94b18763 | 1072 | "snapset_missing" |
b5b8bbf5 FG |
1073 | ], |
1074 | "osd": 0, | |
1075 | "primary": false, | |
1076 | "size": 7 | |
1077 | }, | |
1078 | { | |
b5b8bbf5 | 1079 | "errors": [ |
94b18763 | 1080 | "snapset_corrupted" |
b5b8bbf5 FG |
1081 | ], |
1082 | "osd": 1, | |
1083 | "primary": true, | |
94b18763 | 1084 | "snapset": "bad-val", |
b5b8bbf5 FG |
1085 | "size": 7 |
1086 | } | |
1087 | ], | |
1088 | "union_shard_errors": [ | |
94b18763 FG |
1089 | "snapset_missing", |
1090 | "snapset_corrupted" | |
b5b8bbf5 FG |
1091 | ] |
1092 | }, | |
7c673cae | 1093 | { |
1adf2230 AA |
1094 | "errors": [ |
1095 | "object_info_inconsistency" | |
1096 | ], | |
1097 | "object": { | |
1098 | "locator": "", | |
1099 | "name": "ROBJ18", | |
1100 | "nspace": "", | |
1101 | "snap": "head" | |
1102 | }, | |
1103 | "selected_object_info": { | |
1104 | "alloc_hint_flags": 255, | |
1105 | "data_digest": "0x2ddbf8f5", | |
1106 | "expected_object_size": 0, | |
1107 | "expected_write_size": 0, | |
1108 | "flags": [ | |
1109 | "dirty", | |
1110 | "omap", | |
1111 | "data_digest", | |
1112 | "omap_digest" | |
1113 | ], | |
1114 | "lost": 0, | |
1115 | "manifest": { | |
1116 | "type": 0 | |
1117 | }, | |
1118 | "oid": { | |
1119 | "hash": 1629828556, | |
1120 | "key": "", | |
1121 | "max": 0, | |
1122 | "namespace": "", | |
1123 | "oid": "ROBJ18", | |
1124 | "pool": 3, | |
1125 | "snapid": -2 | |
1126 | }, | |
1127 | "omap_digest": "0xddc3680f", | |
1128 | "size": 7, | |
1129 | "truncate_seq": 0, | |
1130 | "truncate_size": 0, | |
1131 | "user_version": 54, | |
1132 | "watchers": {} | |
1133 | }, | |
1134 | "shards": [ | |
1135 | { | |
1136 | "errors": [], | |
1137 | "object_info": { | |
1138 | "alloc_hint_flags": 0, | |
1139 | "data_digest": "0x2ddbf8f5", | |
1140 | "expected_object_size": 0, | |
1141 | "expected_write_size": 0, | |
1142 | "flags": [ | |
1143 | "dirty", | |
1144 | "omap", | |
1145 | "data_digest", | |
1146 | "omap_digest" | |
1147 | ], | |
1148 | "lost": 0, | |
1149 | "manifest": { | |
1150 | "type": 0 | |
1151 | }, | |
1152 | "oid": { | |
1153 | "hash": 1629828556, | |
1154 | "key": "", | |
1155 | "max": 0, | |
1156 | "namespace": "", | |
1157 | "oid": "ROBJ18", | |
1158 | "pool": 3, | |
1159 | "snapid": -2 | |
1160 | }, | |
1161 | "omap_digest": "0xddc3680f", | |
1162 | "size": 7, | |
1163 | "truncate_seq": 0, | |
1164 | "truncate_size": 0, | |
1165 | "user_version": 54, | |
1166 | "watchers": {} | |
1167 | }, | |
1168 | "osd": 0, | |
1169 | "primary": false, | |
1170 | "size": 7 | |
1171 | }, | |
1172 | { | |
1173 | "errors": [], | |
1174 | "object_info": { | |
1175 | "alloc_hint_flags": 255, | |
1176 | "data_digest": "0x2ddbf8f5", | |
1177 | "expected_object_size": 0, | |
1178 | "expected_write_size": 0, | |
1179 | "flags": [ | |
1180 | "dirty", | |
1181 | "omap", | |
1182 | "data_digest", | |
1183 | "omap_digest" | |
1184 | ], | |
1185 | "lost": 0, | |
1186 | "manifest": { | |
1187 | "type": 0 | |
1188 | }, | |
1189 | "oid": { | |
1190 | "hash": 1629828556, | |
1191 | "key": "", | |
1192 | "max": 0, | |
1193 | "namespace": "", | |
1194 | "oid": "ROBJ18", | |
1195 | "pool": 3, | |
1196 | "snapid": -2 | |
1197 | }, | |
1198 | "omap_digest": "0xddc3680f", | |
1199 | "size": 7, | |
1200 | "truncate_seq": 0, | |
1201 | "truncate_size": 0, | |
1202 | "user_version": 54, | |
1203 | "watchers": {} | |
1204 | }, | |
1205 | "osd": 1, | |
1206 | "primary": true, | |
1207 | "size": 7 | |
1208 | } | |
1209 | ], | |
1210 | "union_shard_errors": [] | |
1211 | }, | |
1212 | { | |
7c673cae FG |
1213 | "shards": [ |
1214 | { | |
1215 | "size": 7, | |
1216 | "errors": [], | |
b5b8bbf5 FG |
1217 | "osd": 0, |
1218 | "primary": false | |
7c673cae FG |
1219 | }, |
1220 | { | |
1221 | "errors": [ | |
1222 | "missing" | |
1223 | ], | |
b5b8bbf5 FG |
1224 | "osd": 1, |
1225 | "primary": true | |
7c673cae FG |
1226 | } |
1227 | ], | |
94b18763 FG |
1228 | "selected_object_info": { |
1229 | "oid": { | |
1230 | "oid": "ROBJ3", | |
1231 | "key": "", | |
1232 | "snapid": -2, | |
1233 | "hash": 625845583, | |
1234 | "max": 0, | |
1235 | "pool": 3, | |
1236 | "namespace": "" | |
1237 | }, | |
1238 | "version": "51'61", | |
1239 | "prior_version": "25'9", | |
1240 | "last_reqid": "osd.1.0:60", | |
1241 | "user_version": 9, | |
1242 | "size": 7, | |
1243 | "mtime": "", | |
1244 | "local_mtime": "", | |
1245 | "lost": 0, | |
1246 | "flags": [ | |
1247 | "dirty", | |
1248 | "omap", | |
1249 | "data_digest", | |
1250 | "omap_digest" | |
1251 | ], | |
1252 | "truncate_seq": 0, | |
1253 | "truncate_size": 0, | |
1254 | "data_digest": "0x2ddbf8f5", | |
1255 | "omap_digest": "0x00b35dfd", | |
1256 | "expected_object_size": 0, | |
1257 | "expected_write_size": 0, | |
1258 | "alloc_hint_flags": 0, | |
1259 | "manifest": { | |
1260 | "type": 0 | |
1261 | }, | |
1262 | "watchers": {} | |
1263 | }, | |
7c673cae FG |
1264 | "union_shard_errors": [ |
1265 | "missing" | |
1266 | ], | |
1267 | "errors": [], | |
1268 | "object": { | |
1269 | "version": 9, | |
1270 | "snap": "head", | |
1271 | "locator": "", | |
1272 | "nspace": "", | |
1273 | "name": "ROBJ3" | |
1274 | } | |
1275 | }, | |
1276 | { | |
1277 | "shards": [ | |
1278 | { | |
1279 | "attrs": [ | |
7c673cae FG |
1280 | { |
1281 | "Base64": false, | |
1282 | "value": "bad-val", | |
94b18763 | 1283 | "name": "key1-ROBJ8" |
7c673cae FG |
1284 | }, |
1285 | { | |
1286 | "Base64": false, | |
b32b8144 | 1287 | "value": "val2-ROBJ8", |
94b18763 | 1288 | "name": "key2-ROBJ8" |
7c673cae FG |
1289 | } |
1290 | ], | |
1291 | "size": 7, | |
1292 | "errors": [], | |
b5b8bbf5 FG |
1293 | "osd": 0, |
1294 | "primary": false | |
7c673cae FG |
1295 | }, |
1296 | { | |
1297 | "attrs": [ | |
7c673cae FG |
1298 | { |
1299 | "Base64": false, | |
1300 | "value": "val1-ROBJ8", | |
94b18763 | 1301 | "name": "key1-ROBJ8" |
7c673cae FG |
1302 | }, |
1303 | { | |
1304 | "Base64": false, | |
b32b8144 | 1305 | "value": "val3-ROBJ8", |
94b18763 | 1306 | "name": "key3-ROBJ8" |
7c673cae FG |
1307 | } |
1308 | ], | |
1309 | "size": 7, | |
1310 | "errors": [], | |
b5b8bbf5 FG |
1311 | "osd": 1, |
1312 | "primary": true | |
7c673cae FG |
1313 | } |
1314 | ], | |
94b18763 FG |
1315 | "selected_object_info": { |
1316 | "oid": { | |
1317 | "oid": "ROBJ8", | |
1318 | "key": "", | |
1319 | "snapid": -2, | |
1320 | "hash": 2359695969, | |
1321 | "max": 0, | |
1322 | "pool": 3, | |
1323 | "namespace": "" | |
1324 | }, | |
1325 | "version": "79'66", | |
1326 | "prior_version": "79'65", | |
1327 | "last_reqid": "client.4554.0:1", | |
1adf2230 | 1328 | "user_version": 74, |
94b18763 FG |
1329 | "size": 7, |
1330 | "mtime": "", | |
1331 | "local_mtime": "", | |
1332 | "lost": 0, | |
1333 | "flags": [ | |
1334 | "dirty", | |
1335 | "omap", | |
1336 | "data_digest", | |
1337 | "omap_digest" | |
1338 | ], | |
1339 | "truncate_seq": 0, | |
1340 | "truncate_size": 0, | |
1341 | "data_digest": "0x2ddbf8f5", | |
1342 | "omap_digest": "0xd6be81dc", | |
1343 | "expected_object_size": 0, | |
1344 | "expected_write_size": 0, | |
1345 | "alloc_hint_flags": 0, | |
1346 | "manifest": { | |
1347 | "type": 0 | |
1348 | }, | |
1349 | "watchers": {} | |
1350 | }, | |
7c673cae FG |
1351 | "union_shard_errors": [], |
1352 | "errors": [ | |
1353 | "attr_value_mismatch", | |
1354 | "attr_name_mismatch" | |
1355 | ], | |
1356 | "object": { | |
b5b8bbf5 | 1357 | "version": 66, |
7c673cae FG |
1358 | "snap": "head", |
1359 | "locator": "", | |
1360 | "nspace": "", | |
1361 | "name": "ROBJ8" | |
1362 | } | |
1363 | }, | |
1364 | { | |
1365 | "shards": [ | |
1366 | { | |
94b18763 FG |
1367 | "object_info": { |
1368 | "oid": { | |
1369 | "oid": "ROBJ9", | |
1370 | "key": "", | |
1371 | "snapid": -2, | |
1372 | "hash": 537189375, | |
1373 | "max": 0, | |
1374 | "pool": 3, | |
1375 | "namespace": "" | |
1376 | }, | |
1377 | "version": "95'67", | |
1378 | "prior_version": "51'64", | |
1379 | "last_reqid": "client.4649.0:1", | |
1adf2230 | 1380 | "user_version": 75, |
94b18763 FG |
1381 | "size": 1, |
1382 | "mtime": "", | |
1383 | "local_mtime": "", | |
1384 | "lost": 0, | |
1385 | "flags": [ | |
1386 | "dirty", | |
1387 | "omap", | |
1388 | "data_digest", | |
1389 | "omap_digest" | |
1390 | ], | |
1391 | "truncate_seq": 0, | |
1392 | "truncate_size": 0, | |
1393 | "data_digest": "0x2b63260d", | |
1394 | "omap_digest": "0x2eecc539", | |
1395 | "expected_object_size": 0, | |
1396 | "expected_write_size": 0, | |
1397 | "alloc_hint_flags": 0, | |
1398 | "manifest": { | |
1399 | "type": 0 | |
1400 | }, | |
1401 | "watchers": {} | |
1402 | }, | |
7c673cae FG |
1403 | "size": 1, |
1404 | "errors": [], | |
b5b8bbf5 FG |
1405 | "osd": 0, |
1406 | "primary": false | |
7c673cae FG |
1407 | }, |
1408 | { | |
94b18763 FG |
1409 | "object_info": { |
1410 | "oid": { | |
1411 | "oid": "ROBJ9", | |
1412 | "key": "", | |
1413 | "snapid": -2, | |
1414 | "hash": 537189375, | |
1415 | "max": 0, | |
1416 | "pool": 3, | |
1417 | "namespace": "" | |
1418 | }, | |
1419 | "version": "51'64", | |
1420 | "prior_version": "37'27", | |
1421 | "last_reqid": "osd.1.0:63", | |
1422 | "user_version": 27, | |
1423 | "size": 7, | |
1424 | "mtime": "2018-04-05 14:33:25.352485", | |
1425 | "local_mtime": "2018-04-05 14:33:25.353746", | |
1426 | "lost": 0, | |
1427 | "flags": [ | |
1428 | "dirty", | |
1429 | "omap", | |
1430 | "data_digest", | |
1431 | "omap_digest" | |
1432 | ], | |
1433 | "truncate_seq": 0, | |
1434 | "truncate_size": 0, | |
1435 | "data_digest": "0x2ddbf8f5", | |
1436 | "omap_digest": "0x2eecc539", | |
1437 | "expected_object_size": 0, | |
1438 | "expected_write_size": 0, | |
1439 | "alloc_hint_flags": 0, | |
1440 | "manifest": { | |
1441 | "type": 0 | |
1442 | }, | |
1443 | "watchers": {} | |
1444 | }, | |
7c673cae | 1445 | "size": 1, |
b5b8bbf5 | 1446 | "errors": [ |
94b18763 | 1447 | "obj_size_info_mismatch" |
b5b8bbf5 FG |
1448 | ], |
1449 | "osd": 1, | |
1450 | "primary": true | |
7c673cae FG |
1451 | } |
1452 | ], | |
94b18763 FG |
1453 | "selected_object_info": { |
1454 | "oid": { | |
1455 | "oid": "ROBJ9", | |
1456 | "key": "", | |
1457 | "snapid": -2, | |
1458 | "hash": 537189375, | |
1459 | "max": 0, | |
1460 | "pool": 3, | |
1461 | "namespace": "" | |
1462 | }, | |
1463 | "version": "95'67", | |
1464 | "prior_version": "51'64", | |
1465 | "last_reqid": "client.4649.0:1", | |
1adf2230 | 1466 | "user_version": 75, |
94b18763 FG |
1467 | "size": 1, |
1468 | "mtime": "", | |
1469 | "local_mtime": "", | |
1470 | "lost": 0, | |
1471 | "flags": [ | |
1472 | "dirty", | |
1473 | "omap", | |
1474 | "data_digest", | |
1475 | "omap_digest" | |
1476 | ], | |
1477 | "truncate_seq": 0, | |
1478 | "truncate_size": 0, | |
1479 | "data_digest": "0x2b63260d", | |
1480 | "omap_digest": "0x2eecc539", | |
1481 | "expected_object_size": 0, | |
1482 | "expected_write_size": 0, | |
1483 | "alloc_hint_flags": 0, | |
1484 | "manifest": { | |
1485 | "type": 0 | |
1486 | }, | |
1487 | "watchers": {} | |
1488 | }, | |
b5b8bbf5 | 1489 | "union_shard_errors": [ |
94b18763 | 1490 | "obj_size_info_mismatch" |
b5b8bbf5 | 1491 | ], |
7c673cae | 1492 | "errors": [ |
b5b8bbf5 | 1493 | "object_info_inconsistency" |
7c673cae FG |
1494 | ], |
1495 | "object": { | |
b5b8bbf5 | 1496 | "version": 67, |
7c673cae FG |
1497 | "snap": "head", |
1498 | "locator": "", | |
1499 | "nspace": "", | |
1500 | "name": "ROBJ9" | |
1501 | } | |
1502 | } | |
1503 | ], | |
1504 | "epoch": 0 | |
1505 | } | |
1506 | EOF | |
1507 | ||
94b18763 | 1508 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
1adf2230 | 1509 | multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
1510 | if test $getjson = "yes" |
1511 | then | |
1512 | jq '.' $dir/json > save1.json | |
1513 | fi | |
1514 | ||
94b18763 | 1515 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
1516 | then |
1517 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
1518 | fi | |
1519 | ||
7c673cae FG |
1520 | objname=ROBJ9 |
1521 | # Change data and size again because digest was recomputed | |
1522 | echo -n ZZZ > $dir/change | |
1523 | rados --pool $poolname put $objname $dir/change | |
1524 | # Set one to an even older value | |
1525 | objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi | |
1526 | rm $dir/oi $dir/change | |
1527 | ||
1528 | objname=ROBJ10 | |
1529 | objectstore_tool $dir 1 $objname get-attr _ > $dir/oi | |
1530 | rados --pool $poolname setomapval $objname key2-$objname val2-$objname | |
1531 | objectstore_tool $dir 0 $objname set-attr _ $dir/oi | |
1532 | objectstore_tool $dir 1 $objname set-attr _ $dir/oi | |
1533 | rm $dir/oi | |
1534 | ||
b5b8bbf5 FG |
1535 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
1536 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1537 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1538 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
1539 | pg_deep_scrub $pg |
1540 | ||
91327a77 AA |
1541 | err_strings=() |
1542 | err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:30259878:::ROBJ15:head : candidate had a missing info key" | |
1543 | err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)], object info inconsistent " | |
1544 | err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:33aca486:::ROBJ18:head : data_digest 0xbd89c912 != data_digest 0x2ddbf8f5 from auth oi 3:33aca486:::ROBJ18:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 54 dd 2ddbf8f5 od ddc3680f alloc_hint [[]0 0 255[]][)]" | |
1545 | err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:33aca486:::ROBJ18:head : failed to pick suitable auth object" | |
1546 | err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:5c7b2c47:::ROBJ16:head : candidate had a corrupt snapset" | |
1547 | err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:5c7b2c47:::ROBJ16:head : candidate had a missing snapset key" | |
1548 | err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:5c7b2c47:::ROBJ16:head : failed to pick suitable object info" | |
1549 | err_strings[7]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:86586531:::ROBJ8:head : attr value mismatch '_key1-ROBJ8', attr name mismatch '_key3-ROBJ8', attr name mismatch '_key2-ROBJ8'" | |
1550 | err_strings[8]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:87abbf36:::ROBJ11:head : candidate had a read error" | |
1551 | err_strings[9]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]" | |
1552 | err_strings[10]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8aa5320e:::ROBJ17:head : data_digest 0x5af0c3ef != data_digest 0x2ddbf8f5 from auth oi 3:8aa5320e:::ROBJ17:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 51 dd 2ddbf8f5 od e9572720 alloc_hint [[]0 0 0[]][)]" | |
1553 | err_strings[11]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8aa5320e:::ROBJ17:head : failed to pick suitable auth object" | |
1554 | err_strings[12]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0xefced57a != omap_digest 0x6a73cc07 from shard 1" | |
1555 | err_strings[13]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:8b55fa4b:::ROBJ7:head : omap_digest 0x6a73cc07 != omap_digest 0xefced57a from auth oi 3:8b55fa4b:::ROBJ7:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 21 dd 2ddbf8f5 od efced57a alloc_hint [[]0 0 0[]][)]" | |
1556 | err_strings[14]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:a53c12e8:::ROBJ6:head : omap_digest 0x689ee887 != omap_digest 0x179c919f from shard 1, omap_digest 0x689ee887 != omap_digest 0x179c919f from auth oi 3:a53c12e8:::ROBJ6:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 18 dd 2ddbf8f5 od 179c919f alloc_hint [[]0 0 0[]][)]" | |
1557 | err_strings[15]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]" | |
1558 | err_strings[16]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:b1f19cbd:::ROBJ10:head : omap_digest 0xa8dd5adc != omap_digest 0xc2025a24 from auth oi 3:b1f19cbd:::ROBJ10:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 30 dd 2ddbf8f5 od c2025a24 alloc_hint [[]0 0 0[]][)]" | |
1559 | err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:b1f19cbd:::ROBJ10:head : failed to pick suitable auth object" | |
1560 | err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:bc819597:::ROBJ12:head : candidate had a stat error" | |
1561 | err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:c0c86b1d:::ROBJ14:head : candidate had a missing info key" | |
1562 | err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:c0c86b1d:::ROBJ14:head : candidate had a corrupt info" | |
1563 | err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:c0c86b1d:::ROBJ14:head : failed to pick suitable object info" | |
1564 | err_strings[22]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : candidate size 9 info size 7 mismatch" | |
1565 | err_strings[23]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:ce3f1d6a:::ROBJ1:head : data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from shard 0, data_digest 0x2d4a11c2 != data_digest 0x2ddbf8f5 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:65 dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from auth oi 3:ce3f1d6a:::ROBJ1:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 3 dd 2ddbf8f5 od f5fba2c6 alloc_hint [[]0 0 0[]][)], size 9 != size 7 from shard 0" | |
1566 | err_strings[24]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:d60617f9:::ROBJ13:head : candidate had a read error" | |
1567 | err_strings[25]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:d60617f9:::ROBJ13:head : candidate had a stat error" | |
1568 | err_strings[26]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:d60617f9:::ROBJ13:head : failed to pick suitable object info" | |
1569 | err_strings[27]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:e97ce31e:::ROBJ2:head : data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from shard 1, data_digest 0x578a4830 != data_digest 0x2ddbf8f5 from auth oi 3:e97ce31e:::ROBJ2:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 6 dd 2ddbf8f5 od f8e11918 alloc_hint [[]0 0 0[]][)]" | |
1570 | err_strings[28]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 3:f2a5b2a4:::ROBJ3:head : missing" | |
1571 | err_strings[29]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:f4981d31:::ROBJ4:head : omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from shard 1, omap_digest 0xd7178dfe != omap_digest 0xe2d46ea4 from auth oi 3:f4981d31:::ROBJ4:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 12 dd 2ddbf8f5 od e2d46ea4 alloc_hint [[]0 0 0[]][)]" | |
1572 | err_strings[30]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x1a862a41 != omap_digest 0x6cac8f6 from shard 1" | |
1573 | err_strings[31]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 1 soid 3:f4bfd4d1:::ROBJ5:head : omap_digest 0x6cac8f6 != omap_digest 0x1a862a41 from auth oi 3:f4bfd4d1:::ROBJ5:head[(][0-9]*'[0-9]* osd.1.0:[0-9]* dirty|omap|data_digest|omap_digest s 7 uv 15 dd 2ddbf8f5 od 1a862a41 alloc_hint [[]0 0 0[]][)]" | |
1574 | err_strings[32]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : candidate size 3 info size 7 mismatch" | |
1575 | err_strings[33]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 shard 0 soid 3:ffdb2004:::ROBJ9:head : object info inconsistent " | |
1576 | err_strings[34]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:c0c86b1d:::ROBJ14:head : no '_' attr" | |
1577 | err_strings[35]="log_channel[(]cluster[)] log [[]ERR[]] : deep-scrub [0-9]*[.]0 3:5c7b2c47:::ROBJ16:head : can't decode 'snapset' attr buffer::malformed_input: .* no longer understand old encoding version 3 < 97" | |
1578 | err_strings[36]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub : stat mismatch, got 18/18 objects, 0/0 clones, 17/18 dirty, 17/18 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 115/116 bytes, 0/0 hit_set_archive bytes." | |
1579 | err_strings[37]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 1 missing, 11 inconsistent objects" | |
1580 | err_strings[38]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 deep-scrub 35 errors" | |
1581 | ||
1582 | for err_string in "${err_strings[@]}" | |
1583 | do | |
1584 | if ! grep -q "$err_string" $dir/osd.${primary}.log | |
1585 | then | |
1586 | echo "Missing log message '$err_string'" | |
1587 | ERRORS=$(expr $ERRORS + 1) | |
1588 | fi | |
1589 | done | |
1590 | ||
7c673cae FG |
1591 | rados list-inconsistent-pg $poolname > $dir/json || return 1 |
1592 | # Check pg count | |
1593 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
1594 | # Check pgid | |
1595 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
1596 | ||
1597 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
1598 | # Get epoch for repair-get requests | |
1599 | epoch=$(jq .epoch $dir/json) | |
1600 | ||
94b18763 | 1601 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
1602 | { |
1603 | "inconsistents": [ | |
1604 | { | |
1605 | "shards": [ | |
1606 | { | |
1607 | "data_digest": "0x2ddbf8f5", | |
1608 | "omap_digest": "0xf5fba2c6", | |
1609 | "size": 7, | |
1610 | "errors": [], | |
b5b8bbf5 FG |
1611 | "osd": 0, |
1612 | "primary": false | |
7c673cae FG |
1613 | }, |
1614 | { | |
94b18763 FG |
1615 | "object_info": { |
1616 | "oid": { | |
1617 | "oid": "ROBJ1", | |
1618 | "key": "", | |
1619 | "snapid": -2, | |
1620 | "hash": 1454963827, | |
1621 | "max": 0, | |
1622 | "pool": 3, | |
1623 | "namespace": "" | |
1624 | }, | |
1625 | "version": "51'58", | |
1626 | "prior_version": "21'3", | |
1627 | "last_reqid": "osd.1.0:57", | |
1628 | "user_version": 3, | |
1629 | "size": 7, | |
1630 | "mtime": "2018-04-05 14:33:19.804040", | |
1631 | "local_mtime": "2018-04-05 14:33:19.804839", | |
1632 | "lost": 0, | |
1633 | "flags": [ | |
1634 | "dirty", | |
1635 | "omap", | |
1636 | "data_digest", | |
1637 | "omap_digest" | |
1638 | ], | |
1639 | "truncate_seq": 0, | |
1640 | "truncate_size": 0, | |
1641 | "data_digest": "0x2ddbf8f5", | |
1642 | "omap_digest": "0xf5fba2c6", | |
1643 | "expected_object_size": 0, | |
1644 | "expected_write_size": 0, | |
1645 | "alloc_hint_flags": 0, | |
1646 | "manifest": { | |
1647 | "type": 0 | |
1648 | }, | |
1649 | "watchers": {} | |
1650 | }, | |
7c673cae FG |
1651 | "data_digest": "0x2d4a11c2", |
1652 | "omap_digest": "0xf5fba2c6", | |
1653 | "size": 9, | |
1654 | "errors": [ | |
94b18763 FG |
1655 | "data_digest_mismatch_info", |
1656 | "size_mismatch_info", | |
1657 | "obj_size_info_mismatch" | |
7c673cae | 1658 | ], |
b5b8bbf5 FG |
1659 | "osd": 1, |
1660 | "primary": true | |
7c673cae FG |
1661 | } |
1662 | ], | |
94b18763 FG |
1663 | "selected_object_info": { |
1664 | "oid": { | |
1665 | "oid": "ROBJ1", | |
1666 | "key": "", | |
1667 | "snapid": -2, | |
1668 | "hash": 1454963827, | |
1669 | "max": 0, | |
1670 | "pool": 3, | |
1671 | "namespace": "" | |
1672 | }, | |
1673 | "version": "51'58", | |
1674 | "prior_version": "21'3", | |
1675 | "last_reqid": "osd.1.0:57", | |
1676 | "user_version": 3, | |
1677 | "size": 7, | |
1678 | "mtime": "2018-04-05 14:33:19.804040", | |
1679 | "local_mtime": "2018-04-05 14:33:19.804839", | |
1680 | "lost": 0, | |
1681 | "flags": [ | |
1682 | "dirty", | |
1683 | "omap", | |
1684 | "data_digest", | |
1685 | "omap_digest" | |
1686 | ], | |
1687 | "truncate_seq": 0, | |
1688 | "truncate_size": 0, | |
1689 | "data_digest": "0x2ddbf8f5", | |
1690 | "omap_digest": "0xf5fba2c6", | |
1691 | "expected_object_size": 0, | |
1692 | "expected_write_size": 0, | |
1693 | "alloc_hint_flags": 0, | |
1694 | "manifest": { | |
1695 | "type": 0 | |
1696 | }, | |
1697 | "watchers": {} | |
1698 | }, | |
7c673cae | 1699 | "union_shard_errors": [ |
94b18763 FG |
1700 | "data_digest_mismatch_info", |
1701 | "size_mismatch_info", | |
1702 | "obj_size_info_mismatch" | |
7c673cae FG |
1703 | ], |
1704 | "errors": [ | |
1705 | "data_digest_mismatch", | |
1706 | "size_mismatch" | |
1707 | ], | |
1708 | "object": { | |
1709 | "version": 3, | |
1710 | "snap": "head", | |
1711 | "locator": "", | |
1712 | "nspace": "", | |
1713 | "name": "ROBJ1" | |
1714 | } | |
1715 | }, | |
1716 | { | |
1717 | "shards": [ | |
1718 | { | |
1719 | "data_digest": "0x2ddbf8f5", | |
1720 | "omap_digest": "0xa8dd5adc", | |
1721 | "size": 7, | |
1722 | "errors": [ | |
94b18763 | 1723 | "omap_digest_mismatch_info" |
7c673cae | 1724 | ], |
b5b8bbf5 FG |
1725 | "osd": 0, |
1726 | "primary": false | |
7c673cae FG |
1727 | }, |
1728 | { | |
1729 | "data_digest": "0x2ddbf8f5", | |
1730 | "omap_digest": "0xa8dd5adc", | |
1731 | "size": 7, | |
1732 | "errors": [ | |
94b18763 | 1733 | "omap_digest_mismatch_info" |
7c673cae | 1734 | ], |
b5b8bbf5 FG |
1735 | "osd": 1, |
1736 | "primary": true | |
7c673cae FG |
1737 | } |
1738 | ], | |
94b18763 FG |
1739 | "selected_object_info": { |
1740 | "alloc_hint_flags": 0, | |
1741 | "data_digest": "0x2ddbf8f5", | |
1742 | "expected_object_size": 0, | |
1743 | "expected_write_size": 0, | |
1744 | "flags": [ | |
1745 | "dirty", | |
1746 | "omap", | |
1747 | "data_digest", | |
1748 | "omap_digest" | |
1749 | ], | |
1750 | "last_reqid": "osd.0.0:50", | |
1751 | "local_mtime": "2018-04-05 14:33:26.762368", | |
1752 | "lost": 0, | |
1753 | "manifest": { | |
1754 | "type": 0 | |
1755 | }, | |
1756 | "mtime": "2018-04-05 14:33:26.762368", | |
1757 | "oid": { | |
1758 | "hash": 3174666125, | |
1759 | "key": "", | |
1760 | "max": 0, | |
1761 | "namespace": "", | |
1762 | "oid": "ROBJ10", | |
1763 | "pool": 3, | |
1764 | "snapid": -2 | |
1765 | }, | |
1766 | "omap_digest": "0xc2025a24", | |
1767 | "prior_version": "41'33", | |
1768 | "size": 7, | |
1769 | "truncate_seq": 0, | |
1770 | "truncate_size": 0, | |
1771 | "user_version": 30, | |
1772 | "version": "47'51", | |
1773 | "watchers": {} | |
1774 | }, | |
7c673cae | 1775 | "union_shard_errors": [ |
94b18763 | 1776 | "omap_digest_mismatch_info" |
7c673cae FG |
1777 | ], |
1778 | "errors": [], | |
1779 | "object": { | |
31f18b77 | 1780 | "version": 30, |
7c673cae FG |
1781 | "snap": "head", |
1782 | "locator": "", | |
1783 | "nspace": "", | |
1784 | "name": "ROBJ10" | |
1785 | } | |
1786 | }, | |
1787 | { | |
1788 | "shards": [ | |
1789 | { | |
1790 | "data_digest": "0x2ddbf8f5", | |
1791 | "omap_digest": "0xa03cef03", | |
1792 | "size": 7, | |
1793 | "errors": [], | |
b5b8bbf5 FG |
1794 | "osd": 0, |
1795 | "primary": false | |
7c673cae FG |
1796 | }, |
1797 | { | |
1798 | "size": 7, | |
1799 | "errors": [ | |
1800 | "read_error" | |
1801 | ], | |
b5b8bbf5 FG |
1802 | "osd": 1, |
1803 | "primary": true | |
7c673cae FG |
1804 | } |
1805 | ], | |
94b18763 FG |
1806 | "selected_object_info": { |
1807 | "oid": { | |
1808 | "oid": "ROBJ11", | |
1809 | "key": "", | |
1810 | "snapid": -2, | |
1811 | "hash": 1828574689, | |
1812 | "max": 0, | |
1813 | "pool": 3, | |
1814 | "namespace": "" | |
1815 | }, | |
1816 | "version": "51'52", | |
1817 | "prior_version": "41'33", | |
1818 | "last_reqid": "osd.1.0:51", | |
1819 | "user_version": 33, | |
1820 | "size": 7, | |
1821 | "mtime": "2018-04-05 14:33:26.761286", | |
1822 | "local_mtime": "2018-04-05 14:33:26.762368", | |
1823 | "lost": 0, | |
1824 | "flags": [ | |
1825 | "dirty", | |
1826 | "omap", | |
1827 | "data_digest", | |
1828 | "omap_digest" | |
1829 | ], | |
1830 | "truncate_seq": 0, | |
1831 | "truncate_size": 0, | |
1832 | "data_digest": "0x2ddbf8f5", | |
1833 | "omap_digest": "0xa03cef03", | |
1834 | "expected_object_size": 0, | |
1835 | "expected_write_size": 0, | |
1836 | "alloc_hint_flags": 0, | |
1837 | "manifest": { | |
1838 | "type": 0 | |
1839 | }, | |
1840 | "watchers": {} | |
1841 | }, | |
7c673cae FG |
1842 | "union_shard_errors": [ |
1843 | "read_error" | |
1844 | ], | |
1845 | "errors": [], | |
1846 | "object": { | |
31f18b77 | 1847 | "version": 33, |
7c673cae FG |
1848 | "snap": "head", |
1849 | "locator": "", | |
1850 | "nspace": "", | |
1851 | "name": "ROBJ11" | |
1852 | } | |
1853 | }, | |
1854 | { | |
1855 | "shards": [ | |
1856 | { | |
1857 | "errors": [ | |
1858 | "stat_error" | |
1859 | ], | |
b5b8bbf5 FG |
1860 | "osd": 0, |
1861 | "primary": false | |
7c673cae FG |
1862 | }, |
1863 | { | |
1864 | "data_digest": "0x2ddbf8f5", | |
1865 | "omap_digest": "0x067f306a", | |
1866 | "size": 7, | |
1867 | "errors": [], | |
b5b8bbf5 FG |
1868 | "osd": 1, |
1869 | "primary": true | |
7c673cae FG |
1870 | } |
1871 | ], | |
94b18763 FG |
1872 | "selected_object_info": { |
1873 | "oid": { | |
1874 | "oid": "ROBJ12", | |
1875 | "key": "", | |
1876 | "snapid": -2, | |
1877 | "hash": 3920199997, | |
1878 | "max": 0, | |
1879 | "pool": 3, | |
1880 | "namespace": "" | |
1881 | }, | |
1882 | "version": "51'56", | |
1883 | "prior_version": "43'36", | |
1884 | "last_reqid": "osd.1.0:55", | |
1885 | "user_version": 36, | |
1886 | "size": 7, | |
1887 | "mtime": "2018-04-05 14:33:27.460958", | |
1888 | "local_mtime": "2018-04-05 14:33:27.462109", | |
1889 | "lost": 0, | |
1890 | "flags": [ | |
1891 | "dirty", | |
1892 | "omap", | |
1893 | "data_digest", | |
1894 | "omap_digest" | |
1895 | ], | |
1896 | "truncate_seq": 0, | |
1897 | "truncate_size": 0, | |
1898 | "data_digest": "0x2ddbf8f5", | |
1899 | "omap_digest": "0x067f306a", | |
1900 | "expected_object_size": 0, | |
1901 | "expected_write_size": 0, | |
1902 | "alloc_hint_flags": 0, | |
1903 | "manifest": { | |
1904 | "type": 0 | |
1905 | }, | |
1906 | "watchers": {} | |
1907 | }, | |
7c673cae FG |
1908 | "union_shard_errors": [ |
1909 | "stat_error" | |
1910 | ], | |
1911 | "errors": [], | |
1912 | "object": { | |
31f18b77 | 1913 | "version": 36, |
7c673cae FG |
1914 | "snap": "head", |
1915 | "locator": "", | |
1916 | "nspace": "", | |
1917 | "name": "ROBJ12" | |
1918 | } | |
1919 | }, | |
1920 | { | |
1921 | "shards": [ | |
1922 | { | |
1923 | "errors": [ | |
1924 | "stat_error" | |
1925 | ], | |
b5b8bbf5 FG |
1926 | "osd": 0, |
1927 | "primary": false | |
7c673cae FG |
1928 | }, |
1929 | { | |
1930 | "size": 7, | |
1931 | "errors": [ | |
1932 | "read_error" | |
1933 | ], | |
b5b8bbf5 FG |
1934 | "osd": 1, |
1935 | "primary": true | |
7c673cae FG |
1936 | } |
1937 | ], | |
1938 | "union_shard_errors": [ | |
1939 | "stat_error", | |
1940 | "read_error" | |
1941 | ], | |
1942 | "errors": [], | |
1943 | "object": { | |
1944 | "version": 0, | |
1945 | "snap": "head", | |
1946 | "locator": "", | |
1947 | "nspace": "", | |
1948 | "name": "ROBJ13" | |
1949 | } | |
1950 | }, | |
1951 | { | |
1952 | "shards": [ | |
1953 | { | |
94b18763 | 1954 | "object_info": "bad-val", |
7c673cae FG |
1955 | "data_digest": "0x2ddbf8f5", |
1956 | "omap_digest": "0x4f14f849", | |
1957 | "size": 7, | |
1958 | "errors": [ | |
94b18763 | 1959 | "info_corrupted" |
7c673cae | 1960 | ], |
b5b8bbf5 FG |
1961 | "osd": 0, |
1962 | "primary": false | |
7c673cae FG |
1963 | }, |
1964 | { | |
1965 | "data_digest": "0x2ddbf8f5", | |
1966 | "omap_digest": "0x4f14f849", | |
1967 | "size": 7, | |
1968 | "errors": [ | |
94b18763 | 1969 | "info_missing" |
7c673cae | 1970 | ], |
b5b8bbf5 FG |
1971 | "osd": 1, |
1972 | "primary": true | |
7c673cae FG |
1973 | } |
1974 | ], | |
1975 | "union_shard_errors": [ | |
94b18763 FG |
1976 | "info_missing", |
1977 | "info_corrupted" | |
7c673cae FG |
1978 | ], |
1979 | "errors": [], | |
1980 | "object": { | |
1981 | "version": 0, | |
1982 | "snap": "head", | |
1983 | "locator": "", | |
1984 | "nspace": "", | |
1985 | "name": "ROBJ14" | |
1986 | } | |
1987 | }, | |
1988 | { | |
1989 | "shards": [ | |
1990 | { | |
94b18763 FG |
1991 | "object_info": { |
1992 | "oid": { | |
1993 | "oid": "ROBJ15", | |
1994 | "key": "", | |
1995 | "snapid": -2, | |
1996 | "hash": 504996876, | |
1997 | "max": 0, | |
1998 | "pool": 3, | |
1999 | "namespace": "" | |
7c673cae | 2000 | }, |
94b18763 FG |
2001 | "version": "51'49", |
2002 | "prior_version": "49'45", | |
2003 | "last_reqid": "osd.1.0:48", | |
2004 | "user_version": 45, | |
2005 | "size": 7, | |
2006 | "mtime": "2018-04-05 14:33:29.498969", | |
2007 | "local_mtime": "2018-04-05 14:33:29.499890", | |
2008 | "lost": 0, | |
2009 | "flags": [ | |
2010 | "dirty", | |
2011 | "omap", | |
2012 | "data_digest", | |
2013 | "omap_digest" | |
2014 | ], | |
2015 | "truncate_seq": 0, | |
2016 | "truncate_size": 0, | |
2017 | "data_digest": "0x2ddbf8f5", | |
2018 | "omap_digest": "0x2d2a4d6e", | |
2019 | "expected_object_size": 0, | |
2020 | "expected_write_size": 0, | |
2021 | "alloc_hint_flags": 0, | |
2022 | "manifest": { | |
2023 | "type": 0 | |
2024 | }, | |
2025 | "watchers": {} | |
2026 | }, | |
7c673cae FG |
2027 | "data_digest": "0x2ddbf8f5", |
2028 | "omap_digest": "0x2d2a4d6e", | |
2029 | "size": 7, | |
2030 | "errors": [], | |
b5b8bbf5 FG |
2031 | "osd": 0, |
2032 | "primary": false | |
7c673cae FG |
2033 | }, |
2034 | { | |
7c673cae FG |
2035 | "data_digest": "0x2ddbf8f5", |
2036 | "omap_digest": "0x2d2a4d6e", | |
2037 | "size": 7, | |
2038 | "errors": [ | |
94b18763 | 2039 | "info_missing" |
7c673cae | 2040 | ], |
b5b8bbf5 FG |
2041 | "osd": 1, |
2042 | "primary": true | |
7c673cae FG |
2043 | } |
2044 | ], | |
94b18763 FG |
2045 | "selected_object_info": { |
2046 | "oid": { | |
2047 | "oid": "ROBJ15", | |
2048 | "key": "", | |
2049 | "snapid": -2, | |
2050 | "hash": 504996876, | |
2051 | "max": 0, | |
2052 | "pool": 3, | |
2053 | "namespace": "" | |
2054 | }, | |
2055 | "version": "51'49", | |
2056 | "prior_version": "49'45", | |
2057 | "last_reqid": "osd.1.0:48", | |
2058 | "user_version": 45, | |
2059 | "size": 7, | |
2060 | "mtime": "2018-04-05 14:33:29.498969", | |
2061 | "local_mtime": "2018-04-05 14:33:29.499890", | |
2062 | "lost": 0, | |
2063 | "flags": [ | |
2064 | "dirty", | |
2065 | "omap", | |
2066 | "data_digest", | |
2067 | "omap_digest" | |
2068 | ], | |
2069 | "truncate_seq": 0, | |
2070 | "truncate_size": 0, | |
2071 | "data_digest": "0x2ddbf8f5", | |
2072 | "omap_digest": "0x2d2a4d6e", | |
2073 | "expected_object_size": 0, | |
2074 | "expected_write_size": 0, | |
2075 | "alloc_hint_flags": 0, | |
2076 | "manifest": { | |
2077 | "type": 0 | |
2078 | }, | |
2079 | "watchers": {} | |
2080 | }, | |
7c673cae | 2081 | "union_shard_errors": [ |
94b18763 | 2082 | "info_missing" |
7c673cae | 2083 | ], |
b5b8bbf5 | 2084 | "errors": [], |
7c673cae | 2085 | "object": { |
31f18b77 | 2086 | "version": 45, |
7c673cae FG |
2087 | "snap": "head", |
2088 | "locator": "", | |
2089 | "nspace": "", | |
2090 | "name": "ROBJ15" | |
2091 | } | |
2092 | }, | |
b5b8bbf5 FG |
2093 | { |
2094 | "errors": [], | |
2095 | "object": { | |
2096 | "locator": "", | |
2097 | "name": "ROBJ16", | |
2098 | "nspace": "", | |
2099 | "snap": "head", | |
2100 | "version": 0 | |
2101 | }, | |
2102 | "shards": [ | |
2103 | { | |
b5b8bbf5 FG |
2104 | "data_digest": "0x2ddbf8f5", |
2105 | "errors": [ | |
94b18763 | 2106 | "snapset_missing" |
b5b8bbf5 FG |
2107 | ], |
2108 | "omap_digest": "0x8b699207", | |
2109 | "osd": 0, | |
2110 | "primary": false, | |
2111 | "size": 7 | |
2112 | }, | |
2113 | { | |
94b18763 | 2114 | "snapset": "bad-val", |
b5b8bbf5 FG |
2115 | "data_digest": "0x2ddbf8f5", |
2116 | "errors": [ | |
94b18763 | 2117 | "snapset_corrupted" |
b5b8bbf5 FG |
2118 | ], |
2119 | "omap_digest": "0x8b699207", | |
2120 | "osd": 1, | |
2121 | "primary": true, | |
2122 | "size": 7 | |
2123 | } | |
2124 | ], | |
2125 | "union_shard_errors": [ | |
94b18763 FG |
2126 | "snapset_missing", |
2127 | "snapset_corrupted" | |
b5b8bbf5 FG |
2128 | ] |
2129 | }, | |
7c673cae | 2130 | { |
1adf2230 AA |
2131 | "errors": [], |
2132 | "object": { | |
2133 | "locator": "", | |
2134 | "name": "ROBJ17", | |
2135 | "nspace": "", | |
2136 | "snap": "head" | |
2137 | }, | |
2138 | "selected_object_info": { | |
2139 | "alloc_hint_flags": 0, | |
2140 | "data_digest": "0x2ddbf8f5", | |
2141 | "expected_object_size": 0, | |
2142 | "expected_write_size": 0, | |
2143 | "flags": [ | |
2144 | "dirty", | |
2145 | "omap", | |
2146 | "data_digest", | |
2147 | "omap_digest" | |
2148 | ], | |
2149 | "lost": 0, | |
2150 | "manifest": { | |
2151 | "type": 0 | |
2152 | }, | |
2153 | "oid": { | |
2154 | "hash": 1884071249, | |
2155 | "key": "", | |
2156 | "max": 0, | |
2157 | "namespace": "", | |
2158 | "oid": "ROBJ17", | |
2159 | "pool": 3, | |
2160 | "snapid": -2 | |
2161 | }, | |
2162 | "omap_digest": "0xe9572720", | |
2163 | "size": 7, | |
2164 | "truncate_seq": 0, | |
2165 | "truncate_size": 0, | |
2166 | "user_version": 51, | |
2167 | "watchers": {} | |
2168 | }, | |
2169 | "shards": [ | |
2170 | { | |
2171 | "data_digest": "0x5af0c3ef", | |
2172 | "errors": [ | |
2173 | "data_digest_mismatch_info" | |
2174 | ], | |
2175 | "omap_digest": "0xe9572720", | |
2176 | "osd": 0, | |
2177 | "primary": false, | |
2178 | "size": 7 | |
2179 | }, | |
2180 | { | |
2181 | "data_digest": "0x5af0c3ef", | |
2182 | "errors": [ | |
2183 | "data_digest_mismatch_info" | |
2184 | ], | |
2185 | "omap_digest": "0xe9572720", | |
2186 | "osd": 1, | |
2187 | "primary": true, | |
2188 | "size": 7 | |
2189 | } | |
2190 | ], | |
2191 | "union_shard_errors": [ | |
2192 | "data_digest_mismatch_info" | |
2193 | ] | |
2194 | }, | |
2195 | { | |
2196 | "errors": [ | |
2197 | "object_info_inconsistency" | |
2198 | ], | |
2199 | "object": { | |
2200 | "locator": "", | |
2201 | "name": "ROBJ18", | |
2202 | "nspace": "", | |
2203 | "snap": "head" | |
2204 | }, | |
2205 | "selected_object_info": { | |
2206 | "alloc_hint_flags": 255, | |
2207 | "data_digest": "0x2ddbf8f5", | |
2208 | "expected_object_size": 0, | |
2209 | "expected_write_size": 0, | |
2210 | "flags": [ | |
2211 | "dirty", | |
2212 | "omap", | |
2213 | "data_digest", | |
2214 | "omap_digest" | |
2215 | ], | |
2216 | "lost": 0, | |
2217 | "manifest": { | |
2218 | "type": 0 | |
2219 | }, | |
2220 | "oid": { | |
2221 | "hash": 1629828556, | |
2222 | "key": "", | |
2223 | "max": 0, | |
2224 | "namespace": "", | |
2225 | "oid": "ROBJ18", | |
2226 | "pool": 3, | |
2227 | "snapid": -2 | |
2228 | }, | |
2229 | "omap_digest": "0xddc3680f", | |
2230 | "size": 7, | |
2231 | "truncate_seq": 0, | |
2232 | "truncate_size": 0, | |
2233 | "user_version": 54, | |
2234 | "watchers": {} | |
2235 | }, | |
2236 | "shards": [ | |
2237 | { | |
2238 | "data_digest": "0xbd89c912", | |
2239 | "errors": [ | |
2240 | "data_digest_mismatch_info" | |
2241 | ], | |
2242 | "object_info": { | |
2243 | "alloc_hint_flags": 0, | |
2244 | "data_digest": "0x2ddbf8f5", | |
2245 | "expected_object_size": 0, | |
2246 | "expected_write_size": 0, | |
2247 | "flags": [ | |
2248 | "dirty", | |
2249 | "omap", | |
2250 | "data_digest", | |
2251 | "omap_digest" | |
2252 | ], | |
2253 | "lost": 0, | |
2254 | "manifest": { | |
2255 | "type": 0 | |
2256 | }, | |
2257 | "oid": { | |
2258 | "hash": 1629828556, | |
2259 | "key": "", | |
2260 | "max": 0, | |
2261 | "namespace": "", | |
2262 | "oid": "ROBJ18", | |
2263 | "pool": 3, | |
2264 | "snapid": -2 | |
2265 | }, | |
2266 | "omap_digest": "0xddc3680f", | |
2267 | "size": 7, | |
2268 | "truncate_seq": 0, | |
2269 | "truncate_size": 0, | |
2270 | "user_version": 54, | |
2271 | "watchers": {} | |
2272 | }, | |
2273 | "omap_digest": "0xddc3680f", | |
2274 | "osd": 0, | |
2275 | "primary": false, | |
2276 | "size": 7 | |
2277 | }, | |
2278 | { | |
2279 | "data_digest": "0xbd89c912", | |
2280 | "errors": [ | |
2281 | "data_digest_mismatch_info" | |
2282 | ], | |
2283 | "object_info": { | |
2284 | "alloc_hint_flags": 255, | |
2285 | "data_digest": "0x2ddbf8f5", | |
2286 | "expected_object_size": 0, | |
2287 | "expected_write_size": 0, | |
2288 | "flags": [ | |
2289 | "dirty", | |
2290 | "omap", | |
2291 | "data_digest", | |
2292 | "omap_digest" | |
2293 | ], | |
2294 | "lost": 0, | |
2295 | "manifest": { | |
2296 | "type": 0 | |
2297 | }, | |
2298 | "oid": { | |
2299 | "hash": 1629828556, | |
2300 | "key": "", | |
2301 | "max": 0, | |
2302 | "namespace": "", | |
2303 | "oid": "ROBJ18", | |
2304 | "pool": 3, | |
2305 | "snapid": -2 | |
2306 | }, | |
2307 | "omap_digest": "0xddc3680f", | |
2308 | "size": 7, | |
2309 | "truncate_seq": 0, | |
2310 | "truncate_size": 0, | |
2311 | "user_version": 54, | |
2312 | "watchers": {} | |
2313 | }, | |
2314 | "omap_digest": "0xddc3680f", | |
2315 | "osd": 1, | |
2316 | "primary": true, | |
2317 | "size": 7 | |
2318 | } | |
2319 | ], | |
2320 | "union_shard_errors": [ | |
2321 | "data_digest_mismatch_info" | |
2322 | ] | |
2323 | }, | |
2324 | { | |
2325 | "shards": [ | |
7c673cae FG |
2326 | { |
2327 | "data_digest": "0x578a4830", | |
2328 | "omap_digest": "0xf8e11918", | |
2329 | "size": 7, | |
2330 | "errors": [ | |
94b18763 | 2331 | "data_digest_mismatch_info" |
7c673cae | 2332 | ], |
b5b8bbf5 FG |
2333 | "osd": 0, |
2334 | "primary": false | |
7c673cae FG |
2335 | }, |
2336 | { | |
2337 | "data_digest": "0x2ddbf8f5", | |
2338 | "omap_digest": "0xf8e11918", | |
2339 | "size": 7, | |
2340 | "errors": [], | |
b5b8bbf5 FG |
2341 | "osd": 1, |
2342 | "primary": true | |
7c673cae FG |
2343 | } |
2344 | ], | |
94b18763 FG |
2345 | "selected_object_info": { |
2346 | "oid": { | |
2347 | "oid": "ROBJ2", | |
2348 | "key": "", | |
2349 | "snapid": -2, | |
2350 | "hash": 2026323607, | |
2351 | "max": 0, | |
2352 | "pool": 3, | |
2353 | "namespace": "" | |
2354 | }, | |
2355 | "version": "51'60", | |
2356 | "prior_version": "23'6", | |
2357 | "last_reqid": "osd.1.0:59", | |
2358 | "user_version": 6, | |
2359 | "size": 7, | |
2360 | "mtime": "2018-04-05 14:33:20.498756", | |
2361 | "local_mtime": "2018-04-05 14:33:20.499704", | |
2362 | "lost": 0, | |
2363 | "flags": [ | |
2364 | "dirty", | |
2365 | "omap", | |
2366 | "data_digest", | |
2367 | "omap_digest" | |
2368 | ], | |
2369 | "truncate_seq": 0, | |
2370 | "truncate_size": 0, | |
2371 | "data_digest": "0x2ddbf8f5", | |
2372 | "omap_digest": "0xf8e11918", | |
2373 | "expected_object_size": 0, | |
2374 | "expected_write_size": 0, | |
2375 | "alloc_hint_flags": 0, | |
2376 | "manifest": { | |
2377 | "type": 0 | |
2378 | }, | |
2379 | "watchers": {} | |
2380 | }, | |
7c673cae | 2381 | "union_shard_errors": [ |
94b18763 | 2382 | "data_digest_mismatch_info" |
7c673cae FG |
2383 | ], |
2384 | "errors": [ | |
2385 | "data_digest_mismatch" | |
2386 | ], | |
2387 | "object": { | |
2388 | "version": 6, | |
2389 | "snap": "head", | |
2390 | "locator": "", | |
2391 | "nspace": "", | |
2392 | "name": "ROBJ2" | |
2393 | } | |
2394 | }, | |
2395 | { | |
2396 | "shards": [ | |
2397 | { | |
2398 | "data_digest": "0x2ddbf8f5", | |
2399 | "omap_digest": "0x00b35dfd", | |
2400 | "size": 7, | |
2401 | "errors": [], | |
b5b8bbf5 FG |
2402 | "osd": 0, |
2403 | "primary": false | |
7c673cae FG |
2404 | }, |
2405 | { | |
2406 | "errors": [ | |
2407 | "missing" | |
2408 | ], | |
b5b8bbf5 FG |
2409 | "osd": 1, |
2410 | "primary": true | |
7c673cae FG |
2411 | } |
2412 | ], | |
94b18763 FG |
2413 | "selected_object_info": { |
2414 | "oid": { | |
2415 | "oid": "ROBJ3", | |
2416 | "key": "", | |
2417 | "snapid": -2, | |
2418 | "hash": 625845583, | |
2419 | "max": 0, | |
2420 | "pool": 3, | |
2421 | "namespace": "" | |
2422 | }, | |
2423 | "version": "51'61", | |
2424 | "prior_version": "25'9", | |
2425 | "last_reqid": "osd.1.0:60", | |
2426 | "user_version": 9, | |
2427 | "size": 7, | |
2428 | "mtime": "2018-04-05 14:33:21.189382", | |
2429 | "local_mtime": "2018-04-05 14:33:21.190446", | |
2430 | "lost": 0, | |
2431 | "flags": [ | |
2432 | "dirty", | |
2433 | "omap", | |
2434 | "data_digest", | |
2435 | "omap_digest" | |
2436 | ], | |
2437 | "truncate_seq": 0, | |
2438 | "truncate_size": 0, | |
2439 | "data_digest": "0x2ddbf8f5", | |
2440 | "omap_digest": "0x00b35dfd", | |
2441 | "expected_object_size": 0, | |
2442 | "expected_write_size": 0, | |
2443 | "alloc_hint_flags": 0, | |
2444 | "manifest": { | |
2445 | "type": 0 | |
2446 | }, | |
2447 | "watchers": {} | |
2448 | }, | |
7c673cae FG |
2449 | "union_shard_errors": [ |
2450 | "missing" | |
2451 | ], | |
2452 | "errors": [], | |
2453 | "object": { | |
2454 | "version": 9, | |
2455 | "snap": "head", | |
2456 | "locator": "", | |
2457 | "nspace": "", | |
2458 | "name": "ROBJ3" | |
2459 | } | |
2460 | }, | |
2461 | { | |
2462 | "shards": [ | |
2463 | { | |
2464 | "data_digest": "0x2ddbf8f5", | |
2465 | "omap_digest": "0xd7178dfe", | |
2466 | "size": 7, | |
31f18b77 | 2467 | "errors": [ |
94b18763 | 2468 | "omap_digest_mismatch_info" |
31f18b77 | 2469 | ], |
b5b8bbf5 FG |
2470 | "osd": 0, |
2471 | "primary": false | |
7c673cae FG |
2472 | }, |
2473 | { | |
2474 | "data_digest": "0x2ddbf8f5", | |
2475 | "omap_digest": "0xe2d46ea4", | |
2476 | "size": 7, | |
31f18b77 | 2477 | "errors": [], |
b5b8bbf5 FG |
2478 | "osd": 1, |
2479 | "primary": true | |
7c673cae FG |
2480 | } |
2481 | ], | |
94b18763 FG |
2482 | "selected_object_info": { |
2483 | "oid": { | |
2484 | "oid": "ROBJ4", | |
2485 | "key": "", | |
2486 | "snapid": -2, | |
2487 | "hash": 2360875311, | |
2488 | "max": 0, | |
2489 | "pool": 3, | |
2490 | "namespace": "" | |
2491 | }, | |
2492 | "version": "51'62", | |
2493 | "prior_version": "27'12", | |
2494 | "last_reqid": "osd.1.0:61", | |
2495 | "user_version": 12, | |
2496 | "size": 7, | |
2497 | "mtime": "2018-04-05 14:33:21.862313", | |
2498 | "local_mtime": "2018-04-05 14:33:21.863261", | |
2499 | "lost": 0, | |
2500 | "flags": [ | |
2501 | "dirty", | |
2502 | "omap", | |
2503 | "data_digest", | |
2504 | "omap_digest" | |
2505 | ], | |
2506 | "truncate_seq": 0, | |
2507 | "truncate_size": 0, | |
2508 | "data_digest": "0x2ddbf8f5", | |
2509 | "omap_digest": "0xe2d46ea4", | |
2510 | "expected_object_size": 0, | |
2511 | "expected_write_size": 0, | |
2512 | "alloc_hint_flags": 0, | |
2513 | "manifest": { | |
2514 | "type": 0 | |
2515 | }, | |
2516 | "watchers": {} | |
2517 | }, | |
7c673cae | 2518 | "union_shard_errors": [ |
94b18763 | 2519 | "omap_digest_mismatch_info" |
7c673cae FG |
2520 | ], |
2521 | "errors": [ | |
2522 | "omap_digest_mismatch" | |
2523 | ], | |
2524 | "object": { | |
2525 | "version": 12, | |
2526 | "snap": "head", | |
2527 | "locator": "", | |
2528 | "nspace": "", | |
2529 | "name": "ROBJ4" | |
2530 | } | |
2531 | }, | |
2532 | { | |
2533 | "shards": [ | |
2534 | { | |
2535 | "data_digest": "0x2ddbf8f5", | |
2536 | "omap_digest": "0x1a862a41", | |
2537 | "size": 7, | |
2538 | "errors": [], | |
b5b8bbf5 FG |
2539 | "osd": 0, |
2540 | "primary": false | |
7c673cae FG |
2541 | }, |
2542 | { | |
2543 | "data_digest": "0x2ddbf8f5", | |
2544 | "omap_digest": "0x06cac8f6", | |
2545 | "size": 7, | |
2546 | "errors": [ | |
94b18763 | 2547 | "omap_digest_mismatch_info" |
7c673cae | 2548 | ], |
b5b8bbf5 FG |
2549 | "osd": 1, |
2550 | "primary": true | |
7c673cae FG |
2551 | } |
2552 | ], | |
94b18763 FG |
2553 | "selected_object_info": { |
2554 | "oid": { | |
2555 | "oid": "ROBJ5", | |
2556 | "key": "", | |
2557 | "snapid": -2, | |
2558 | "hash": 2334915887, | |
2559 | "max": 0, | |
2560 | "pool": 3, | |
2561 | "namespace": "" | |
2562 | }, | |
2563 | "version": "51'63", | |
2564 | "prior_version": "29'15", | |
2565 | "last_reqid": "osd.1.0:62", | |
2566 | "user_version": 15, | |
2567 | "size": 7, | |
2568 | "mtime": "2018-04-05 14:33:22.589300", | |
2569 | "local_mtime": "2018-04-05 14:33:22.590376", | |
2570 | "lost": 0, | |
2571 | "flags": [ | |
2572 | "dirty", | |
2573 | "omap", | |
2574 | "data_digest", | |
2575 | "omap_digest" | |
2576 | ], | |
2577 | "truncate_seq": 0, | |
2578 | "truncate_size": 0, | |
2579 | "data_digest": "0x2ddbf8f5", | |
2580 | "omap_digest": "0x1a862a41", | |
2581 | "expected_object_size": 0, | |
2582 | "expected_write_size": 0, | |
2583 | "alloc_hint_flags": 0, | |
2584 | "manifest": { | |
2585 | "type": 0 | |
2586 | }, | |
2587 | "watchers": {} | |
2588 | }, | |
7c673cae | 2589 | "union_shard_errors": [ |
94b18763 | 2590 | "omap_digest_mismatch_info" |
7c673cae FG |
2591 | ], |
2592 | "errors": [ | |
2593 | "omap_digest_mismatch" | |
2594 | ], | |
2595 | "object": { | |
2596 | "version": 15, | |
2597 | "snap": "head", | |
2598 | "locator": "", | |
2599 | "nspace": "", | |
2600 | "name": "ROBJ5" | |
2601 | } | |
2602 | }, | |
2603 | { | |
2604 | "shards": [ | |
2605 | { | |
2606 | "data_digest": "0x2ddbf8f5", | |
2607 | "omap_digest": "0x689ee887", | |
2608 | "size": 7, | |
31f18b77 | 2609 | "errors": [ |
94b18763 | 2610 | "omap_digest_mismatch_info" |
31f18b77 | 2611 | ], |
b5b8bbf5 FG |
2612 | "osd": 0, |
2613 | "primary": false | |
7c673cae FG |
2614 | }, |
2615 | { | |
2616 | "data_digest": "0x2ddbf8f5", | |
2617 | "omap_digest": "0x179c919f", | |
2618 | "size": 7, | |
31f18b77 | 2619 | "errors": [], |
b5b8bbf5 FG |
2620 | "osd": 1, |
2621 | "primary": true | |
7c673cae FG |
2622 | } |
2623 | ], | |
94b18763 FG |
2624 | "selected_object_info": { |
2625 | "oid": { | |
2626 | "oid": "ROBJ6", | |
2627 | "key": "", | |
2628 | "snapid": -2, | |
2629 | "hash": 390610085, | |
2630 | "max": 0, | |
2631 | "pool": 3, | |
2632 | "namespace": "" | |
2633 | }, | |
2634 | "version": "51'54", | |
2635 | "prior_version": "31'18", | |
2636 | "last_reqid": "osd.1.0:53", | |
2637 | "user_version": 18, | |
2638 | "size": 7, | |
2639 | "mtime": "2018-04-05 14:33:23.289188", | |
2640 | "local_mtime": "2018-04-05 14:33:23.290130", | |
2641 | "lost": 0, | |
2642 | "flags": [ | |
2643 | "dirty", | |
2644 | "omap", | |
2645 | "data_digest", | |
2646 | "omap_digest" | |
2647 | ], | |
2648 | "truncate_seq": 0, | |
2649 | "truncate_size": 0, | |
2650 | "data_digest": "0x2ddbf8f5", | |
2651 | "omap_digest": "0x179c919f", | |
2652 | "expected_object_size": 0, | |
2653 | "expected_write_size": 0, | |
2654 | "alloc_hint_flags": 0, | |
2655 | "manifest": { | |
2656 | "type": 0 | |
2657 | }, | |
2658 | "watchers": {} | |
2659 | }, | |
7c673cae | 2660 | "union_shard_errors": [ |
94b18763 | 2661 | "omap_digest_mismatch_info" |
7c673cae FG |
2662 | ], |
2663 | "errors": [ | |
2664 | "omap_digest_mismatch" | |
2665 | ], | |
2666 | "object": { | |
2667 | "version": 18, | |
2668 | "snap": "head", | |
2669 | "locator": "", | |
2670 | "nspace": "", | |
2671 | "name": "ROBJ6" | |
2672 | } | |
2673 | }, | |
2674 | { | |
2675 | "shards": [ | |
2676 | { | |
2677 | "data_digest": "0x2ddbf8f5", | |
2678 | "omap_digest": "0xefced57a", | |
2679 | "size": 7, | |
2680 | "errors": [], | |
b5b8bbf5 FG |
2681 | "osd": 0, |
2682 | "primary": false | |
7c673cae FG |
2683 | }, |
2684 | { | |
2685 | "data_digest": "0x2ddbf8f5", | |
2686 | "omap_digest": "0x6a73cc07", | |
2687 | "size": 7, | |
2688 | "errors": [ | |
94b18763 | 2689 | "omap_digest_mismatch_info" |
7c673cae | 2690 | ], |
b5b8bbf5 FG |
2691 | "osd": 1, |
2692 | "primary": true | |
7c673cae FG |
2693 | } |
2694 | ], | |
94b18763 FG |
2695 | "selected_object_info": { |
2696 | "oid": { | |
2697 | "oid": "ROBJ7", | |
2698 | "key": "", | |
2699 | "snapid": -2, | |
2700 | "hash": 3529485009, | |
2701 | "max": 0, | |
2702 | "pool": 3, | |
2703 | "namespace": "" | |
2704 | }, | |
2705 | "version": "51'53", | |
2706 | "prior_version": "33'21", | |
2707 | "last_reqid": "osd.1.0:52", | |
2708 | "user_version": 21, | |
2709 | "size": 7, | |
2710 | "mtime": "2018-04-05 14:33:23.979658", | |
2711 | "local_mtime": "2018-04-05 14:33:23.980731", | |
2712 | "lost": 0, | |
2713 | "flags": [ | |
2714 | "dirty", | |
2715 | "omap", | |
2716 | "data_digest", | |
2717 | "omap_digest" | |
2718 | ], | |
2719 | "truncate_seq": 0, | |
2720 | "truncate_size": 0, | |
2721 | "data_digest": "0x2ddbf8f5", | |
2722 | "omap_digest": "0xefced57a", | |
2723 | "expected_object_size": 0, | |
2724 | "expected_write_size": 0, | |
2725 | "alloc_hint_flags": 0, | |
2726 | "manifest": { | |
2727 | "type": 0 | |
2728 | }, | |
2729 | "watchers": {} | |
2730 | }, | |
7c673cae | 2731 | "union_shard_errors": [ |
94b18763 | 2732 | "omap_digest_mismatch_info" |
7c673cae FG |
2733 | ], |
2734 | "errors": [ | |
2735 | "omap_digest_mismatch" | |
2736 | ], | |
2737 | "object": { | |
2738 | "version": 21, | |
2739 | "snap": "head", | |
2740 | "locator": "", | |
2741 | "nspace": "", | |
2742 | "name": "ROBJ7" | |
2743 | } | |
2744 | }, | |
2745 | { | |
2746 | "shards": [ | |
2747 | { | |
2748 | "attrs": [ | |
7c673cae FG |
2749 | { |
2750 | "Base64": false, | |
2751 | "value": "bad-val", | |
94b18763 | 2752 | "name": "key1-ROBJ8" |
7c673cae FG |
2753 | }, |
2754 | { | |
2755 | "Base64": false, | |
b32b8144 | 2756 | "value": "val2-ROBJ8", |
94b18763 | 2757 | "name": "key2-ROBJ8" |
7c673cae FG |
2758 | } |
2759 | ], | |
2760 | "data_digest": "0x2ddbf8f5", | |
2761 | "omap_digest": "0xd6be81dc", | |
2762 | "size": 7, | |
2763 | "errors": [], | |
b5b8bbf5 FG |
2764 | "osd": 0, |
2765 | "primary": false | |
7c673cae FG |
2766 | }, |
2767 | { | |
2768 | "attrs": [ | |
7c673cae FG |
2769 | { |
2770 | "Base64": false, | |
2771 | "value": "val1-ROBJ8", | |
94b18763 | 2772 | "name": "key1-ROBJ8" |
7c673cae FG |
2773 | }, |
2774 | { | |
2775 | "Base64": false, | |
b32b8144 | 2776 | "value": "val3-ROBJ8", |
94b18763 | 2777 | "name": "key3-ROBJ8" |
7c673cae FG |
2778 | } |
2779 | ], | |
2780 | "data_digest": "0x2ddbf8f5", | |
2781 | "omap_digest": "0xd6be81dc", | |
2782 | "size": 7, | |
2783 | "errors": [], | |
b5b8bbf5 FG |
2784 | "osd": 1, |
2785 | "primary": true | |
7c673cae FG |
2786 | } |
2787 | ], | |
94b18763 FG |
2788 | "selected_object_info": { |
2789 | "oid": { | |
2790 | "oid": "ROBJ8", | |
2791 | "key": "", | |
2792 | "snapid": -2, | |
2793 | "hash": 2359695969, | |
2794 | "max": 0, | |
2795 | "pool": 3, | |
2796 | "namespace": "" | |
2797 | }, | |
2798 | "version": "79'66", | |
2799 | "prior_version": "79'65", | |
2800 | "last_reqid": "client.4554.0:1", | |
1adf2230 | 2801 | "user_version": 74, |
94b18763 FG |
2802 | "size": 7, |
2803 | "mtime": "2018-04-05 14:34:05.598688", | |
2804 | "local_mtime": "2018-04-05 14:34:05.599698", | |
2805 | "lost": 0, | |
2806 | "flags": [ | |
2807 | "dirty", | |
2808 | "omap", | |
2809 | "data_digest", | |
2810 | "omap_digest" | |
2811 | ], | |
2812 | "truncate_seq": 0, | |
2813 | "truncate_size": 0, | |
2814 | "data_digest": "0x2ddbf8f5", | |
2815 | "omap_digest": "0xd6be81dc", | |
2816 | "expected_object_size": 0, | |
2817 | "expected_write_size": 0, | |
2818 | "alloc_hint_flags": 0, | |
2819 | "manifest": { | |
2820 | "type": 0 | |
2821 | }, | |
2822 | "watchers": {} | |
2823 | }, | |
7c673cae FG |
2824 | "union_shard_errors": [], |
2825 | "errors": [ | |
2826 | "attr_value_mismatch", | |
2827 | "attr_name_mismatch" | |
2828 | ], | |
2829 | "object": { | |
b5b8bbf5 | 2830 | "version": 66, |
7c673cae FG |
2831 | "snap": "head", |
2832 | "locator": "", | |
2833 | "nspace": "", | |
2834 | "name": "ROBJ8" | |
2835 | } | |
2836 | }, | |
2837 | { | |
2838 | "shards": [ | |
2839 | { | |
94b18763 FG |
2840 | "object_info": { |
2841 | "oid": { | |
2842 | "oid": "ROBJ9", | |
2843 | "key": "", | |
2844 | "snapid": -2, | |
2845 | "hash": 537189375, | |
2846 | "max": 0, | |
2847 | "pool": 3, | |
2848 | "namespace": "" | |
2849 | }, | |
2850 | "version": "51'64", | |
2851 | "prior_version": "37'27", | |
2852 | "last_reqid": "osd.1.0:63", | |
2853 | "user_version": 27, | |
2854 | "size": 7, | |
2855 | "mtime": "2018-04-05 14:33:25.352485", | |
2856 | "local_mtime": "2018-04-05 14:33:25.353746", | |
2857 | "lost": 0, | |
2858 | "flags": [ | |
2859 | "dirty", | |
2860 | "omap", | |
2861 | "data_digest", | |
2862 | "omap_digest" | |
2863 | ], | |
2864 | "truncate_seq": 0, | |
2865 | "truncate_size": 0, | |
2866 | "data_digest": "0x2ddbf8f5", | |
2867 | "omap_digest": "0x2eecc539", | |
2868 | "expected_object_size": 0, | |
2869 | "expected_write_size": 0, | |
2870 | "alloc_hint_flags": 0, | |
2871 | "manifest": { | |
2872 | "type": 0 | |
2873 | }, | |
2874 | "watchers": {} | |
2875 | }, | |
7c673cae FG |
2876 | "data_digest": "0x1f26fb26", |
2877 | "omap_digest": "0x2eecc539", | |
2878 | "size": 3, | |
b5b8bbf5 | 2879 | "errors": [ |
94b18763 | 2880 | "obj_size_info_mismatch" |
b5b8bbf5 FG |
2881 | ], |
2882 | "osd": 0, | |
2883 | "primary": false | |
7c673cae FG |
2884 | }, |
2885 | { | |
94b18763 FG |
2886 | "object_info": { |
2887 | "oid": { | |
2888 | "oid": "ROBJ9", | |
2889 | "key": "", | |
2890 | "snapid": -2, | |
2891 | "hash": 537189375, | |
2892 | "max": 0, | |
2893 | "pool": 3, | |
2894 | "namespace": "" | |
2895 | }, | |
2896 | "version": "119'68", | |
2897 | "prior_version": "51'64", | |
2898 | "last_reqid": "client.4834.0:1", | |
1adf2230 | 2899 | "user_version": 76, |
94b18763 FG |
2900 | "size": 3, |
2901 | "mtime": "2018-04-05 14:35:01.500659", | |
2902 | "local_mtime": "2018-04-05 14:35:01.502117", | |
2903 | "lost": 0, | |
2904 | "flags": [ | |
2905 | "dirty", | |
2906 | "omap", | |
2907 | "data_digest", | |
2908 | "omap_digest" | |
2909 | ], | |
2910 | "truncate_seq": 0, | |
2911 | "truncate_size": 0, | |
2912 | "data_digest": "0x1f26fb26", | |
2913 | "omap_digest": "0x2eecc539", | |
2914 | "expected_object_size": 0, | |
2915 | "expected_write_size": 0, | |
2916 | "alloc_hint_flags": 0, | |
2917 | "manifest": { | |
2918 | "type": 0 | |
2919 | }, | |
2920 | "watchers": {} | |
2921 | }, | |
7c673cae FG |
2922 | "data_digest": "0x1f26fb26", |
2923 | "omap_digest": "0x2eecc539", | |
2924 | "size": 3, | |
2925 | "errors": [], | |
b5b8bbf5 FG |
2926 | "osd": 1, |
2927 | "primary": true | |
7c673cae FG |
2928 | } |
2929 | ], | |
94b18763 FG |
2930 | "selected_object_info": { |
2931 | "oid": { | |
2932 | "oid": "ROBJ9", | |
2933 | "key": "", | |
2934 | "snapid": -2, | |
2935 | "hash": 537189375, | |
2936 | "max": 0, | |
2937 | "pool": 3, | |
2938 | "namespace": "" | |
2939 | }, | |
2940 | "version": "119'68", | |
2941 | "prior_version": "51'64", | |
2942 | "last_reqid": "client.4834.0:1", | |
1adf2230 | 2943 | "user_version": 76, |
94b18763 FG |
2944 | "size": 3, |
2945 | "mtime": "2018-04-05 14:35:01.500659", | |
2946 | "local_mtime": "2018-04-05 14:35:01.502117", | |
2947 | "lost": 0, | |
2948 | "flags": [ | |
2949 | "dirty", | |
2950 | "omap", | |
2951 | "data_digest", | |
2952 | "omap_digest" | |
2953 | ], | |
2954 | "truncate_seq": 0, | |
2955 | "truncate_size": 0, | |
2956 | "data_digest": "0x1f26fb26", | |
2957 | "omap_digest": "0x2eecc539", | |
2958 | "expected_object_size": 0, | |
2959 | "expected_write_size": 0, | |
2960 | "alloc_hint_flags": 0, | |
2961 | "manifest": { | |
2962 | "type": 0 | |
2963 | }, | |
2964 | "watchers": {} | |
2965 | }, | |
b5b8bbf5 | 2966 | "union_shard_errors": [ |
94b18763 | 2967 | "obj_size_info_mismatch" |
b5b8bbf5 | 2968 | ], |
7c673cae | 2969 | "errors": [ |
b5b8bbf5 | 2970 | "object_info_inconsistency" |
7c673cae FG |
2971 | ], |
2972 | "object": { | |
b5b8bbf5 | 2973 | "version": 68, |
7c673cae FG |
2974 | "snap": "head", |
2975 | "locator": "", | |
2976 | "nspace": "", | |
2977 | "name": "ROBJ9" | |
2978 | } | |
2979 | } | |
2980 | ], | |
2981 | "epoch": 0 | |
2982 | } | |
2983 | EOF | |
2984 | ||
94b18763 | 2985 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
1adf2230 | 2986 | multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
2987 | if test $getjson = "yes" |
2988 | then | |
2989 | jq '.' $dir/json > save2.json | |
2990 | fi | |
2991 | ||
94b18763 | 2992 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
2993 | then |
2994 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
2995 | fi | |
2996 | ||
1adf2230 AA |
2997 | repair $pg |
2998 | wait_for_clean | |
2999 | ||
3000 | # This hangs if the repair doesn't work | |
3001 | timeout 30 rados -p $poolname get ROBJ17 $dir/robj17.out || return 1 | |
3002 | timeout 30 rados -p $poolname get ROBJ18 $dir/robj18.out || return 1 | |
3003 | # Even though we couldn't repair all of the introduced errors, we can fix ROBJ17 | |
3004 | diff -q $dir/new.ROBJ17 $dir/robj17.out || return 1 | |
3005 | rm -f $dir/new.ROBJ17 $dir/robj17.out || return 1 | |
3006 | diff -q $dir/new.ROBJ18 $dir/robj18.out || return 1 | |
3007 | rm -f $dir/new.ROBJ18 $dir/robj18.out || return 1 | |
3008 | ||
91327a77 AA |
3009 | if [ $ERRORS != "0" ]; |
3010 | then | |
3011 | echo "TEST FAILED WITH $ERRORS ERRORS" | |
3012 | return 1 | |
3013 | fi | |
3014 | ||
7c673cae FG |
3015 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it |
3016 | teardown $dir || return 1 | |
3017 | } | |
3018 | ||
3019 | ||
3020 | # | |
3021 | # Test scrub errors for an erasure coded pool | |
3022 | # | |
3023 | function corrupt_scrub_erasure() { | |
3024 | local dir=$1 | |
3025 | local allow_overwrites=$2 | |
3026 | local poolname=ecpool | |
94b18763 | 3027 | local total_objs=7 |
7c673cae FG |
3028 | |
3029 | setup $dir || return 1 | |
3030 | run_mon $dir a || return 1 | |
3031 | run_mgr $dir x || return 1 | |
3032 | for id in $(seq 0 2) ; do | |
3033 | if [ "$allow_overwrites" = "true" ]; then | |
3034 | run_osd_bluestore $dir $id || return 1 | |
3035 | else | |
3036 | run_osd $dir $id || return 1 | |
3037 | fi | |
3038 | done | |
c07f9fc5 | 3039 | create_rbd_pool || return 1 |
b5b8bbf5 | 3040 | create_pool foo 1 |
7c673cae FG |
3041 | |
3042 | create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 | |
c07f9fc5 | 3043 | wait_for_clean || return 1 |
7c673cae FG |
3044 | |
3045 | for i in $(seq 1 $total_objs) ; do | |
3046 | objname=EOBJ${i} | |
3047 | add_something $dir $poolname $objname || return 1 | |
3048 | ||
3049 | local osd=$(expr $i % 2) | |
3050 | ||
3051 | case $i in | |
3052 | 1) | |
3053 | # Size (deep scrub data_digest too) | |
3054 | local payload=UVWXYZZZ | |
3055 | echo $payload > $dir/CORRUPT | |
3056 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
3057 | ;; | |
3058 | ||
3059 | 2) | |
3060 | # Corrupt EC shard | |
3061 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 | |
3062 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
3063 | ;; | |
3064 | ||
3065 | 3) | |
3066 | # missing | |
3067 | objectstore_tool $dir $osd $objname remove || return 1 | |
3068 | ;; | |
3069 | ||
3070 | 4) | |
3071 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
3072 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
3073 | ||
3074 | # Break xattrs | |
3075 | echo -n bad-val > $dir/bad-val | |
3076 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
3077 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
3078 | echo -n val3-$objname > $dir/newval | |
3079 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
3080 | rm $dir/bad-val $dir/newval | |
3081 | ;; | |
3082 | ||
3083 | 5) | |
3084 | # Corrupt EC shard | |
3085 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 | |
3086 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
3087 | ;; | |
3088 | ||
94b18763 FG |
3089 | 6) |
3090 | objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1 | |
3091 | echo -n bad-val > $dir/bad-val | |
3092 | objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1 | |
3093 | ;; | |
3094 | ||
3095 | 7) | |
3096 | local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS | |
3097 | echo $payload > $dir/DIFFERENT | |
3098 | rados --pool $poolname put $objname $dir/DIFFERENT || return 1 | |
3099 | ||
3100 | # Get hinfo_key from EOBJ1 | |
3101 | objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo | |
3102 | objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1 | |
3103 | rm -f $dir/hinfo | |
3104 | ;; | |
3105 | ||
7c673cae FG |
3106 | esac |
3107 | done | |
3108 | ||
3109 | local pg=$(get_pg $poolname EOBJ0) | |
3110 | ||
3111 | pg_scrub $pg | |
3112 | ||
3113 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
3114 | # Check pg count | |
3115 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
3116 | # Check pgid | |
3117 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
3118 | ||
3119 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
3120 | # Get epoch for repair-get requests | |
3121 | epoch=$(jq .epoch $dir/json) | |
3122 | ||
94b18763 | 3123 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
3124 | { |
3125 | "inconsistents": [ | |
3126 | { | |
3127 | "shards": [ | |
3128 | { | |
3129 | "size": 2048, | |
3130 | "errors": [], | |
3131 | "shard": 2, | |
b5b8bbf5 FG |
3132 | "osd": 0, |
3133 | "primary": false | |
7c673cae FG |
3134 | }, |
3135 | { | |
94b18763 FG |
3136 | "object_info": { |
3137 | "oid": { | |
3138 | "oid": "EOBJ1", | |
3139 | "key": "", | |
3140 | "snapid": -2, | |
3141 | "hash": 560836233, | |
3142 | "max": 0, | |
3143 | "pool": 3, | |
3144 | "namespace": "" | |
3145 | }, | |
3146 | "version": "27'1", | |
3147 | "prior_version": "0'0", | |
3148 | "last_reqid": "client.4184.0:1", | |
3149 | "user_version": 1, | |
3150 | "size": 7, | |
3151 | "mtime": "", | |
3152 | "local_mtime": "", | |
3153 | "lost": 0, | |
3154 | "flags": [ | |
3155 | "dirty", | |
28e407b8 | 3156 | "data_digest" |
94b18763 FG |
3157 | ], |
3158 | "truncate_seq": 0, | |
3159 | "truncate_size": 0, | |
3160 | "data_digest": "0x2ddbf8f5", | |
3161 | "omap_digest": "0xffffffff", | |
3162 | "expected_object_size": 0, | |
3163 | "expected_write_size": 0, | |
3164 | "alloc_hint_flags": 0, | |
3165 | "manifest": { | |
3166 | "type": 0 | |
3167 | }, | |
3168 | "watchers": {} | |
3169 | }, | |
7c673cae | 3170 | "size": 9, |
224ce89b | 3171 | "shard": 0, |
7c673cae | 3172 | "errors": [ |
94b18763 FG |
3173 | "size_mismatch_info", |
3174 | "obj_size_info_mismatch" | |
7c673cae | 3175 | ], |
b5b8bbf5 FG |
3176 | "osd": 1, |
3177 | "primary": true | |
7c673cae FG |
3178 | }, |
3179 | { | |
3180 | "size": 2048, | |
224ce89b | 3181 | "shard": 1, |
7c673cae | 3182 | "errors": [], |
b5b8bbf5 FG |
3183 | "osd": 2, |
3184 | "primary": false | |
7c673cae FG |
3185 | } |
3186 | ], | |
94b18763 FG |
3187 | "selected_object_info": { |
3188 | "oid": { | |
3189 | "oid": "EOBJ1", | |
3190 | "key": "", | |
3191 | "snapid": -2, | |
3192 | "hash": 560836233, | |
3193 | "max": 0, | |
3194 | "pool": 3, | |
3195 | "namespace": "" | |
3196 | }, | |
3197 | "version": "27'1", | |
3198 | "prior_version": "0'0", | |
3199 | "last_reqid": "client.4184.0:1", | |
3200 | "user_version": 1, | |
3201 | "size": 7, | |
3202 | "mtime": "", | |
3203 | "local_mtime": "", | |
3204 | "lost": 0, | |
3205 | "flags": [ | |
3206 | "dirty", | |
28e407b8 | 3207 | "data_digest" |
94b18763 FG |
3208 | ], |
3209 | "truncate_seq": 0, | |
3210 | "truncate_size": 0, | |
3211 | "data_digest": "0x2ddbf8f5", | |
3212 | "omap_digest": "0xffffffff", | |
3213 | "expected_object_size": 0, | |
3214 | "expected_write_size": 0, | |
3215 | "alloc_hint_flags": 0, | |
3216 | "manifest": { | |
3217 | "type": 0 | |
3218 | }, | |
3219 | "watchers": {} | |
3220 | }, | |
7c673cae | 3221 | "union_shard_errors": [ |
94b18763 FG |
3222 | "size_mismatch_info", |
3223 | "obj_size_info_mismatch" | |
7c673cae FG |
3224 | ], |
3225 | "errors": [ | |
3226 | "size_mismatch" | |
3227 | ], | |
3228 | "object": { | |
3229 | "version": 1, | |
3230 | "snap": "head", | |
3231 | "locator": "", | |
3232 | "nspace": "", | |
3233 | "name": "EOBJ1" | |
3234 | } | |
3235 | }, | |
3236 | { | |
3237 | "shards": [ | |
3238 | { | |
3239 | "size": 2048, | |
3240 | "errors": [], | |
3241 | "shard": 2, | |
b5b8bbf5 FG |
3242 | "osd": 0, |
3243 | "primary": false | |
7c673cae FG |
3244 | }, |
3245 | { | |
224ce89b | 3246 | "shard": 0, |
7c673cae FG |
3247 | "errors": [ |
3248 | "missing" | |
3249 | ], | |
b5b8bbf5 FG |
3250 | "osd": 1, |
3251 | "primary": true | |
7c673cae FG |
3252 | }, |
3253 | { | |
3254 | "size": 2048, | |
224ce89b | 3255 | "shard": 1, |
7c673cae | 3256 | "errors": [], |
b5b8bbf5 FG |
3257 | "osd": 2, |
3258 | "primary": false | |
7c673cae FG |
3259 | } |
3260 | ], | |
94b18763 FG |
3261 | "selected_object_info": { |
3262 | "oid": { | |
3263 | "oid": "EOBJ3", | |
3264 | "key": "", | |
3265 | "snapid": -2, | |
3266 | "hash": 3125668237, | |
3267 | "max": 0, | |
3268 | "pool": 3, | |
3269 | "namespace": "" | |
3270 | }, | |
3271 | "version": "39'3", | |
3272 | "prior_version": "0'0", | |
3273 | "last_reqid": "client.4252.0:1", | |
3274 | "user_version": 3, | |
3275 | "size": 7, | |
3276 | "mtime": "", | |
3277 | "local_mtime": "", | |
3278 | "lost": 0, | |
3279 | "flags": [ | |
3280 | "dirty", | |
28e407b8 | 3281 | "data_digest" |
94b18763 FG |
3282 | ], |
3283 | "truncate_seq": 0, | |
3284 | "truncate_size": 0, | |
3285 | "data_digest": "0x2ddbf8f5", | |
3286 | "omap_digest": "0xffffffff", | |
3287 | "expected_object_size": 0, | |
3288 | "expected_write_size": 0, | |
3289 | "alloc_hint_flags": 0, | |
3290 | "manifest": { | |
3291 | "type": 0 | |
3292 | }, | |
3293 | "watchers": {} | |
3294 | }, | |
7c673cae FG |
3295 | "union_shard_errors": [ |
3296 | "missing" | |
3297 | ], | |
3298 | "errors": [], | |
3299 | "object": { | |
3300 | "version": 3, | |
3301 | "snap": "head", | |
3302 | "locator": "", | |
3303 | "nspace": "", | |
3304 | "name": "EOBJ3" | |
3305 | } | |
3306 | }, | |
3307 | { | |
3308 | "shards": [ | |
3309 | { | |
3310 | "attrs": [ | |
7c673cae FG |
3311 | { |
3312 | "Base64": false, | |
3313 | "value": "bad-val", | |
94b18763 | 3314 | "name": "key1-EOBJ4" |
7c673cae FG |
3315 | }, |
3316 | { | |
3317 | "Base64": false, | |
b32b8144 | 3318 | "value": "val2-EOBJ4", |
94b18763 | 3319 | "name": "key2-EOBJ4" |
7c673cae FG |
3320 | } |
3321 | ], | |
3322 | "size": 2048, | |
3323 | "errors": [], | |
3324 | "shard": 2, | |
b5b8bbf5 FG |
3325 | "osd": 0, |
3326 | "primary": false | |
7c673cae FG |
3327 | }, |
3328 | { | |
224ce89b | 3329 | "osd": 1, |
b5b8bbf5 | 3330 | "primary": true, |
224ce89b WB |
3331 | "shard": 0, |
3332 | "errors": [], | |
3333 | "size": 2048, | |
7c673cae | 3334 | "attrs": [ |
7c673cae FG |
3335 | { |
3336 | "Base64": false, | |
3337 | "value": "val1-EOBJ4", | |
94b18763 | 3338 | "name": "key1-EOBJ4" |
7c673cae FG |
3339 | }, |
3340 | { | |
3341 | "Base64": false, | |
3342 | "value": "val2-EOBJ4", | |
94b18763 | 3343 | "name": "key2-EOBJ4" |
7c673cae | 3344 | } |
224ce89b | 3345 | ] |
7c673cae FG |
3346 | }, |
3347 | { | |
224ce89b | 3348 | "osd": 2, |
b5b8bbf5 | 3349 | "primary": false, |
224ce89b WB |
3350 | "shard": 1, |
3351 | "errors": [], | |
3352 | "size": 2048, | |
7c673cae | 3353 | "attrs": [ |
7c673cae FG |
3354 | { |
3355 | "Base64": false, | |
3356 | "value": "val1-EOBJ4", | |
94b18763 | 3357 | "name": "key1-EOBJ4" |
7c673cae FG |
3358 | }, |
3359 | { | |
3360 | "Base64": false, | |
b32b8144 | 3361 | "value": "val3-EOBJ4", |
94b18763 | 3362 | "name": "key3-EOBJ4" |
7c673cae | 3363 | } |
224ce89b | 3364 | ] |
7c673cae FG |
3365 | } |
3366 | ], | |
94b18763 FG |
3367 | "selected_object_info": { |
3368 | "oid": { | |
3369 | "oid": "EOBJ4", | |
3370 | "key": "", | |
3371 | "snapid": -2, | |
3372 | "hash": 1618759290, | |
3373 | "max": 0, | |
3374 | "pool": 3, | |
3375 | "namespace": "" | |
3376 | }, | |
3377 | "version": "45'6", | |
3378 | "prior_version": "45'5", | |
3379 | "last_reqid": "client.4294.0:1", | |
3380 | "user_version": 6, | |
3381 | "size": 7, | |
3382 | "mtime": "", | |
3383 | "local_mtime": "", | |
3384 | "lost": 0, | |
3385 | "flags": [ | |
3386 | "dirty", | |
28e407b8 | 3387 | "data_digest" |
94b18763 FG |
3388 | ], |
3389 | "truncate_seq": 0, | |
3390 | "truncate_size": 0, | |
3391 | "data_digest": "0x2ddbf8f5", | |
3392 | "omap_digest": "0xffffffff", | |
3393 | "expected_object_size": 0, | |
3394 | "expected_write_size": 0, | |
3395 | "alloc_hint_flags": 0, | |
3396 | "manifest": { | |
3397 | "type": 0 | |
3398 | }, | |
3399 | "watchers": {} | |
3400 | }, | |
7c673cae FG |
3401 | "union_shard_errors": [], |
3402 | "errors": [ | |
3403 | "attr_value_mismatch", | |
3404 | "attr_name_mismatch" | |
3405 | ], | |
3406 | "object": { | |
3407 | "version": 6, | |
3408 | "snap": "head", | |
3409 | "locator": "", | |
3410 | "nspace": "", | |
3411 | "name": "EOBJ4" | |
3412 | } | |
3413 | }, | |
3414 | { | |
3415 | "shards": [ | |
3416 | { | |
3417 | "size": 2048, | |
3418 | "errors": [], | |
3419 | "shard": 2, | |
b5b8bbf5 FG |
3420 | "osd": 0, |
3421 | "primary": false | |
7c673cae FG |
3422 | }, |
3423 | { | |
94b18763 FG |
3424 | "object_info": { |
3425 | "oid": { | |
3426 | "oid": "EOBJ5", | |
3427 | "key": "", | |
3428 | "snapid": -2, | |
3429 | "hash": 2918945441, | |
3430 | "max": 0, | |
3431 | "pool": 3, | |
3432 | "namespace": "" | |
3433 | }, | |
3434 | "version": "59'7", | |
3435 | "prior_version": "0'0", | |
3436 | "last_reqid": "client.4382.0:1", | |
3437 | "user_version": 7, | |
3438 | "size": 7, | |
3439 | "mtime": "", | |
3440 | "local_mtime": "", | |
3441 | "lost": 0, | |
3442 | "flags": [ | |
3443 | "dirty", | |
28e407b8 | 3444 | "data_digest" |
94b18763 FG |
3445 | ], |
3446 | "truncate_seq": 0, | |
3447 | "truncate_size": 0, | |
3448 | "data_digest": "0x2ddbf8f5", | |
3449 | "omap_digest": "0xffffffff", | |
3450 | "expected_object_size": 0, | |
3451 | "expected_write_size": 0, | |
3452 | "alloc_hint_flags": 0, | |
3453 | "manifest": { | |
3454 | "type": 0 | |
3455 | }, | |
3456 | "watchers": {} | |
3457 | }, | |
7c673cae | 3458 | "size": 4096, |
224ce89b | 3459 | "shard": 0, |
7c673cae | 3460 | "errors": [ |
94b18763 FG |
3461 | "size_mismatch_info", |
3462 | "obj_size_info_mismatch" | |
7c673cae | 3463 | ], |
b5b8bbf5 FG |
3464 | "osd": 1, |
3465 | "primary": true | |
7c673cae FG |
3466 | }, |
3467 | { | |
3468 | "size": 2048, | |
224ce89b | 3469 | "shard": 1, |
7c673cae | 3470 | "errors": [], |
b5b8bbf5 FG |
3471 | "osd": 2, |
3472 | "primary": false | |
7c673cae FG |
3473 | } |
3474 | ], | |
94b18763 FG |
3475 | "selected_object_info": { |
3476 | "oid": { | |
3477 | "oid": "EOBJ5", | |
3478 | "key": "", | |
3479 | "snapid": -2, | |
3480 | "hash": 2918945441, | |
3481 | "max": 0, | |
3482 | "pool": 3, | |
3483 | "namespace": "" | |
3484 | }, | |
3485 | "version": "59'7", | |
3486 | "prior_version": "0'0", | |
3487 | "last_reqid": "client.4382.0:1", | |
3488 | "user_version": 7, | |
3489 | "size": 7, | |
3490 | "mtime": "", | |
3491 | "local_mtime": "", | |
3492 | "lost": 0, | |
3493 | "flags": [ | |
3494 | "dirty", | |
28e407b8 | 3495 | "data_digest" |
94b18763 FG |
3496 | ], |
3497 | "truncate_seq": 0, | |
3498 | "truncate_size": 0, | |
3499 | "data_digest": "0x2ddbf8f5", | |
3500 | "omap_digest": "0xffffffff", | |
3501 | "expected_object_size": 0, | |
3502 | "expected_write_size": 0, | |
3503 | "alloc_hint_flags": 0, | |
3504 | "manifest": { | |
3505 | "type": 0 | |
3506 | }, | |
3507 | "watchers": {} | |
3508 | }, | |
7c673cae | 3509 | "union_shard_errors": [ |
94b18763 FG |
3510 | "size_mismatch_info", |
3511 | "obj_size_info_mismatch" | |
7c673cae FG |
3512 | ], |
3513 | "errors": [ | |
3514 | "size_mismatch" | |
3515 | ], | |
3516 | "object": { | |
3517 | "version": 7, | |
3518 | "snap": "head", | |
3519 | "locator": "", | |
3520 | "nspace": "", | |
3521 | "name": "EOBJ5" | |
3522 | } | |
94b18763 FG |
3523 | }, |
3524 | { | |
3525 | "errors": [], | |
3526 | "object": { | |
3527 | "locator": "", | |
3528 | "name": "EOBJ6", | |
3529 | "nspace": "", | |
3530 | "snap": "head", | |
3531 | "version": 8 | |
3532 | }, | |
3533 | "selected_object_info": { | |
3534 | "oid": { | |
3535 | "oid": "EOBJ6", | |
3536 | "key": "", | |
3537 | "snapid": -2, | |
3538 | "hash": 3050890866, | |
3539 | "max": 0, | |
3540 | "pool": 3, | |
3541 | "namespace": "" | |
3542 | }, | |
3543 | "version": "65'8", | |
3544 | "prior_version": "0'0", | |
3545 | "last_reqid": "client.4418.0:1", | |
3546 | "user_version": 8, | |
3547 | "size": 7, | |
3548 | "mtime": "", | |
3549 | "local_mtime": "", | |
3550 | "lost": 0, | |
3551 | "flags": [ | |
3552 | "dirty", | |
28e407b8 | 3553 | "data_digest" |
94b18763 FG |
3554 | ], |
3555 | "truncate_seq": 0, | |
3556 | "truncate_size": 0, | |
3557 | "data_digest": "0x2ddbf8f5", | |
3558 | "omap_digest": "0xffffffff", | |
3559 | "expected_object_size": 0, | |
3560 | "expected_write_size": 0, | |
3561 | "alloc_hint_flags": 0, | |
3562 | "manifest": { | |
3563 | "type": 0 | |
3564 | }, | |
3565 | "watchers": {} | |
3566 | }, | |
3567 | "shards": [ | |
3568 | { | |
3569 | "errors": [ | |
3570 | "hinfo_missing" | |
3571 | ], | |
3572 | "osd": 0, | |
3573 | "primary": false, | |
3574 | "shard": 2, | |
3575 | "size": 2048 | |
3576 | }, | |
3577 | { | |
3578 | "errors": [ | |
3579 | "hinfo_corrupted" | |
3580 | ], | |
3581 | "osd": 1, | |
3582 | "primary": true, | |
3583 | "shard": 0, | |
3584 | "hashinfo": "bad-val", | |
3585 | "size": 2048 | |
3586 | }, | |
3587 | { | |
3588 | "errors": [], | |
3589 | "osd": 2, | |
3590 | "primary": false, | |
3591 | "shard": 1, | |
3592 | "size": 2048, | |
3593 | "hashinfo": { | |
3594 | "cumulative_shard_hashes": [ | |
3595 | { | |
3596 | "hash": 80717615, | |
3597 | "shard": 0 | |
3598 | }, | |
3599 | { | |
3600 | "hash": 1534491824, | |
3601 | "shard": 1 | |
3602 | }, | |
3603 | { | |
3604 | "hash": 80717615, | |
3605 | "shard": 2 | |
3606 | } | |
3607 | ], | |
3608 | "total_chunk_size": 2048 | |
3609 | } | |
3610 | } | |
3611 | ], | |
3612 | "union_shard_errors": [ | |
3613 | "hinfo_missing", | |
3614 | "hinfo_corrupted" | |
3615 | ] | |
3616 | }, | |
3617 | { | |
3618 | "errors": [ | |
3619 | "hinfo_inconsistency" | |
3620 | ], | |
3621 | "object": { | |
3622 | "locator": "", | |
3623 | "name": "EOBJ7", | |
3624 | "nspace": "", | |
3625 | "snap": "head", | |
3626 | "version": 10 | |
3627 | }, | |
3628 | "selected_object_info": { | |
3629 | "oid": { | |
3630 | "oid": "EOBJ7", | |
3631 | "key": "", | |
3632 | "snapid": -2, | |
3633 | "hash": 3258066308, | |
3634 | "max": 0, | |
3635 | "pool": 3, | |
3636 | "namespace": "" | |
3637 | }, | |
3638 | "version": "75'10", | |
3639 | "prior_version": "75'9", | |
3640 | "last_reqid": "client.4482.0:1", | |
3641 | "user_version": 10, | |
3642 | "size": 34, | |
3643 | "mtime": "", | |
3644 | "local_mtime": "", | |
3645 | "lost": 0, | |
3646 | "flags": [ | |
3647 | "dirty", | |
28e407b8 | 3648 | "data_digest" |
94b18763 FG |
3649 | ], |
3650 | "truncate_seq": 0, | |
3651 | "truncate_size": 0, | |
3652 | "data_digest": "0x136e4e27", | |
3653 | "omap_digest": "0xffffffff", | |
3654 | "expected_object_size": 0, | |
3655 | "expected_write_size": 0, | |
3656 | "alloc_hint_flags": 0, | |
3657 | "manifest": { | |
3658 | "type": 0 | |
3659 | }, | |
3660 | "watchers": {} | |
3661 | }, | |
3662 | "shards": [ | |
3663 | { | |
3664 | "hashinfo": { | |
3665 | "cumulative_shard_hashes": [ | |
3666 | { | |
3667 | "hash": 80717615, | |
3668 | "shard": 0 | |
3669 | }, | |
3670 | { | |
3671 | "hash": 1534491824, | |
3672 | "shard": 1 | |
3673 | }, | |
3674 | { | |
3675 | "hash": 80717615, | |
3676 | "shard": 2 | |
3677 | } | |
3678 | ], | |
3679 | "total_chunk_size": 2048 | |
3680 | }, | |
3681 | "errors": [], | |
3682 | "osd": 0, | |
3683 | "primary": false, | |
3684 | "shard": 2, | |
3685 | "size": 2048 | |
3686 | }, | |
3687 | { | |
3688 | "hashinfo": { | |
3689 | "cumulative_shard_hashes": [ | |
3690 | { | |
3691 | "hash": 1534350760, | |
3692 | "shard": 0 | |
3693 | }, | |
3694 | { | |
3695 | "hash": 1534491824, | |
3696 | "shard": 1 | |
3697 | }, | |
3698 | { | |
3699 | "hash": 1534350760, | |
3700 | "shard": 2 | |
3701 | } | |
3702 | ], | |
3703 | "total_chunk_size": 2048 | |
3704 | }, | |
3705 | "errors": [], | |
3706 | "osd": 1, | |
3707 | "primary": true, | |
3708 | "shard": 0, | |
3709 | "size": 2048 | |
3710 | }, | |
3711 | { | |
3712 | "hashinfo": { | |
3713 | "cumulative_shard_hashes": [ | |
3714 | { | |
3715 | "hash": 1534350760, | |
3716 | "shard": 0 | |
3717 | }, | |
3718 | { | |
3719 | "hash": 1534491824, | |
3720 | "shard": 1 | |
3721 | }, | |
3722 | { | |
3723 | "hash": 1534350760, | |
3724 | "shard": 2 | |
3725 | } | |
3726 | ], | |
3727 | "total_chunk_size": 2048 | |
3728 | }, | |
3729 | "errors": [], | |
3730 | "osd": 2, | |
3731 | "primary": false, | |
3732 | "shard": 1, | |
3733 | "size": 2048 | |
3734 | } | |
3735 | ], | |
3736 | "union_shard_errors": [] | |
3737 | } | |
3738 | ], | |
3739 | "epoch": 0 | |
3740 | } | |
3741 | EOF | |
3742 | ||
3743 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson | |
1adf2230 | 3744 | multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
94b18763 FG |
3745 | if test $getjson = "yes" |
3746 | then | |
3747 | jq '.' $dir/json > save3.json | |
3748 | fi | |
3749 | ||
3750 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; | |
3751 | then | |
3752 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
3753 | fi | |
3754 | ||
3755 | pg_deep_scrub $pg | |
3756 | ||
7c673cae FG |
3757 | rados list-inconsistent-pg $poolname > $dir/json || return 1 |
3758 | # Check pg count | |
3759 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
3760 | # Check pgid | |
3761 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
3762 | ||
3763 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
3764 | # Get epoch for repair-get requests | |
3765 | epoch=$(jq .epoch $dir/json) | |
3766 | ||
31f18b77 FG |
3767 | if [ "$allow_overwrites" = "true" ] |
3768 | then | |
94b18763 | 3769 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
31f18b77 FG |
3770 | { |
3771 | "inconsistents": [ | |
3772 | { | |
3773 | "shards": [ | |
3774 | { | |
3775 | "data_digest": "0x00000000", | |
3776 | "omap_digest": "0xffffffff", | |
3777 | "size": 2048, | |
3778 | "errors": [], | |
3779 | "shard": 2, | |
b5b8bbf5 FG |
3780 | "osd": 0, |
3781 | "primary": false | |
31f18b77 FG |
3782 | }, |
3783 | { | |
94b18763 FG |
3784 | "object_info": { |
3785 | "oid": { | |
3786 | "oid": "EOBJ1", | |
3787 | "key": "", | |
3788 | "snapid": -2, | |
3789 | "hash": 560836233, | |
3790 | "max": 0, | |
3791 | "pool": 3, | |
3792 | "namespace": "" | |
3793 | }, | |
3794 | "version": "27'1", | |
3795 | "prior_version": "0'0", | |
3796 | "last_reqid": "client.4184.0:1", | |
3797 | "user_version": 1, | |
3798 | "size": 7, | |
3799 | "mtime": "2018-04-05 14:31:33.837147", | |
3800 | "local_mtime": "2018-04-05 14:31:33.840763", | |
3801 | "lost": 0, | |
3802 | "flags": [ | |
3803 | "dirty", | |
28e407b8 | 3804 | "data_digest" |
94b18763 FG |
3805 | ], |
3806 | "truncate_seq": 0, | |
3807 | "truncate_size": 0, | |
3808 | "data_digest": "0x2ddbf8f5", | |
3809 | "omap_digest": "0xffffffff", | |
3810 | "expected_object_size": 0, | |
3811 | "expected_write_size": 0, | |
3812 | "alloc_hint_flags": 0, | |
3813 | "manifest": { | |
3814 | "type": 0 | |
3815 | }, | |
3816 | "watchers": {} | |
3817 | }, | |
31f18b77 | 3818 | "size": 9, |
224ce89b | 3819 | "shard": 0, |
31f18b77 FG |
3820 | "errors": [ |
3821 | "read_error", | |
94b18763 FG |
3822 | "size_mismatch_info", |
3823 | "obj_size_info_mismatch" | |
31f18b77 | 3824 | ], |
b5b8bbf5 FG |
3825 | "osd": 1, |
3826 | "primary": true | |
31f18b77 FG |
3827 | }, |
3828 | { | |
3829 | "data_digest": "0x00000000", | |
3830 | "omap_digest": "0xffffffff", | |
3831 | "size": 2048, | |
224ce89b | 3832 | "shard": 1, |
31f18b77 | 3833 | "errors": [], |
b5b8bbf5 FG |
3834 | "osd": 2, |
3835 | "primary": false | |
31f18b77 FG |
3836 | } |
3837 | ], | |
94b18763 FG |
3838 | "selected_object_info": { |
3839 | "oid": { | |
3840 | "oid": "EOBJ1", | |
3841 | "key": "", | |
3842 | "snapid": -2, | |
3843 | "hash": 560836233, | |
3844 | "max": 0, | |
3845 | "pool": 3, | |
3846 | "namespace": "" | |
3847 | }, | |
3848 | "version": "27'1", | |
3849 | "prior_version": "0'0", | |
3850 | "last_reqid": "client.4184.0:1", | |
3851 | "user_version": 1, | |
3852 | "size": 7, | |
3853 | "mtime": "2018-04-05 14:31:33.837147", | |
3854 | "local_mtime": "2018-04-05 14:31:33.840763", | |
3855 | "lost": 0, | |
3856 | "flags": [ | |
3857 | "dirty", | |
28e407b8 | 3858 | "data_digest" |
94b18763 FG |
3859 | ], |
3860 | "truncate_seq": 0, | |
3861 | "truncate_size": 0, | |
3862 | "data_digest": "0x2ddbf8f5", | |
3863 | "omap_digest": "0xffffffff", | |
3864 | "expected_object_size": 0, | |
3865 | "expected_write_size": 0, | |
3866 | "alloc_hint_flags": 0, | |
3867 | "manifest": { | |
3868 | "type": 0 | |
3869 | }, | |
3870 | "watchers": {} | |
3871 | }, | |
31f18b77 FG |
3872 | "union_shard_errors": [ |
3873 | "read_error", | |
94b18763 FG |
3874 | "size_mismatch_info", |
3875 | "obj_size_info_mismatch" | |
31f18b77 FG |
3876 | ], |
3877 | "errors": [ | |
3878 | "size_mismatch" | |
3879 | ], | |
3880 | "object": { | |
3881 | "version": 1, | |
3882 | "snap": "head", | |
3883 | "locator": "", | |
3884 | "nspace": "", | |
3885 | "name": "EOBJ1" | |
3886 | } | |
3887 | }, | |
3888 | { | |
3889 | "shards": [ | |
3890 | { | |
3891 | "data_digest": "0x00000000", | |
3892 | "omap_digest": "0xffffffff", | |
3893 | "size": 2048, | |
3894 | "errors": [], | |
3895 | "shard": 2, | |
b5b8bbf5 FG |
3896 | "osd": 0, |
3897 | "primary": false | |
31f18b77 FG |
3898 | }, |
3899 | { | |
224ce89b | 3900 | "shard": 0, |
31f18b77 FG |
3901 | "errors": [ |
3902 | "missing" | |
3903 | ], | |
b5b8bbf5 FG |
3904 | "osd": 1, |
3905 | "primary": true | |
31f18b77 FG |
3906 | }, |
3907 | { | |
3908 | "data_digest": "0x00000000", | |
3909 | "omap_digest": "0xffffffff", | |
3910 | "size": 2048, | |
224ce89b | 3911 | "shard": 1, |
31f18b77 | 3912 | "errors": [], |
b5b8bbf5 FG |
3913 | "osd": 2, |
3914 | "primary": false | |
31f18b77 FG |
3915 | } |
3916 | ], | |
94b18763 FG |
3917 | "selected_object_info": { |
3918 | "oid": { | |
3919 | "oid": "EOBJ3", | |
3920 | "key": "", | |
3921 | "snapid": -2, | |
3922 | "hash": 3125668237, | |
3923 | "max": 0, | |
3924 | "pool": 3, | |
3925 | "namespace": "" | |
3926 | }, | |
3927 | "version": "39'3", | |
3928 | "prior_version": "0'0", | |
3929 | "last_reqid": "client.4252.0:1", | |
3930 | "user_version": 3, | |
3931 | "size": 7, | |
3932 | "mtime": "2018-04-05 14:31:46.841145", | |
3933 | "local_mtime": "2018-04-05 14:31:46.844996", | |
3934 | "lost": 0, | |
3935 | "flags": [ | |
3936 | "dirty", | |
28e407b8 | 3937 | "data_digest" |
94b18763 FG |
3938 | ], |
3939 | "truncate_seq": 0, | |
3940 | "truncate_size": 0, | |
3941 | "data_digest": "0x2ddbf8f5", | |
3942 | "omap_digest": "0xffffffff", | |
3943 | "expected_object_size": 0, | |
3944 | "expected_write_size": 0, | |
3945 | "alloc_hint_flags": 0, | |
3946 | "manifest": { | |
3947 | "type": 0 | |
3948 | }, | |
3949 | "watchers": {} | |
3950 | }, | |
31f18b77 FG |
3951 | "union_shard_errors": [ |
3952 | "missing" | |
3953 | ], | |
3954 | "errors": [], | |
3955 | "object": { | |
3956 | "version": 3, | |
3957 | "snap": "head", | |
3958 | "locator": "", | |
3959 | "nspace": "", | |
3960 | "name": "EOBJ3" | |
3961 | } | |
3962 | }, | |
3963 | { | |
3964 | "shards": [ | |
3965 | { | |
3966 | "attrs": [ | |
31f18b77 FG |
3967 | { |
3968 | "Base64": false, | |
3969 | "value": "bad-val", | |
94b18763 | 3970 | "name": "key1-EOBJ4" |
31f18b77 FG |
3971 | }, |
3972 | { | |
3973 | "Base64": false, | |
b32b8144 | 3974 | "value": "val2-EOBJ4", |
94b18763 | 3975 | "name": "key2-EOBJ4" |
31f18b77 FG |
3976 | } |
3977 | ], | |
3978 | "data_digest": "0x00000000", | |
3979 | "omap_digest": "0xffffffff", | |
3980 | "size": 2048, | |
3981 | "errors": [], | |
3982 | "shard": 2, | |
b5b8bbf5 FG |
3983 | "osd": 0, |
3984 | "primary": false | |
31f18b77 FG |
3985 | }, |
3986 | { | |
3987 | "attrs": [ | |
31f18b77 FG |
3988 | { |
3989 | "Base64": false, | |
3990 | "value": "val1-EOBJ4", | |
94b18763 | 3991 | "name": "key1-EOBJ4" |
31f18b77 FG |
3992 | }, |
3993 | { | |
3994 | "Base64": false, | |
3995 | "value": "val2-EOBJ4", | |
94b18763 | 3996 | "name": "key2-EOBJ4" |
31f18b77 FG |
3997 | } |
3998 | ], | |
3999 | "data_digest": "0x00000000", | |
4000 | "omap_digest": "0xffffffff", | |
4001 | "size": 2048, | |
4002 | "errors": [], | |
224ce89b | 4003 | "shard": 0, |
b5b8bbf5 FG |
4004 | "osd": 1, |
4005 | "primary": true | |
31f18b77 FG |
4006 | }, |
4007 | { | |
4008 | "attrs": [ | |
31f18b77 FG |
4009 | { |
4010 | "Base64": false, | |
4011 | "value": "val1-EOBJ4", | |
94b18763 | 4012 | "name": "key1-EOBJ4" |
31f18b77 FG |
4013 | }, |
4014 | { | |
4015 | "Base64": false, | |
b32b8144 | 4016 | "value": "val3-EOBJ4", |
94b18763 | 4017 | "name": "key3-EOBJ4" |
31f18b77 FG |
4018 | } |
4019 | ], | |
4020 | "data_digest": "0x00000000", | |
4021 | "omap_digest": "0xffffffff", | |
4022 | "size": 2048, | |
4023 | "errors": [], | |
224ce89b | 4024 | "shard": 1, |
b5b8bbf5 FG |
4025 | "osd": 2, |
4026 | "primary": false | |
31f18b77 FG |
4027 | } |
4028 | ], | |
94b18763 FG |
4029 | "selected_object_info": { |
4030 | "oid": { | |
4031 | "oid": "EOBJ4", | |
4032 | "key": "", | |
4033 | "snapid": -2, | |
4034 | "hash": 1618759290, | |
4035 | "max": 0, | |
4036 | "pool": 3, | |
4037 | "namespace": "" | |
4038 | }, | |
4039 | "version": "45'6", | |
4040 | "prior_version": "45'5", | |
4041 | "last_reqid": "client.4294.0:1", | |
4042 | "user_version": 6, | |
4043 | "size": 7, | |
4044 | "mtime": "2018-04-05 14:31:54.663622", | |
4045 | "local_mtime": "2018-04-05 14:31:54.664527", | |
4046 | "lost": 0, | |
4047 | "flags": [ | |
4048 | "dirty", | |
28e407b8 | 4049 | "data_digest" |
94b18763 FG |
4050 | ], |
4051 | "truncate_seq": 0, | |
4052 | "truncate_size": 0, | |
4053 | "data_digest": "0x2ddbf8f5", | |
4054 | "omap_digest": "0xffffffff", | |
4055 | "expected_object_size": 0, | |
4056 | "expected_write_size": 0, | |
4057 | "alloc_hint_flags": 0, | |
4058 | "manifest": { | |
4059 | "type": 0 | |
4060 | }, | |
4061 | "watchers": {} | |
4062 | }, | |
31f18b77 FG |
4063 | "union_shard_errors": [], |
4064 | "errors": [ | |
4065 | "attr_value_mismatch", | |
4066 | "attr_name_mismatch" | |
4067 | ], | |
4068 | "object": { | |
4069 | "version": 6, | |
4070 | "snap": "head", | |
4071 | "locator": "", | |
4072 | "nspace": "", | |
4073 | "name": "EOBJ4" | |
4074 | } | |
4075 | }, | |
4076 | { | |
4077 | "shards": [ | |
4078 | { | |
4079 | "data_digest": "0x00000000", | |
4080 | "omap_digest": "0xffffffff", | |
4081 | "size": 2048, | |
4082 | "errors": [], | |
4083 | "shard": 2, | |
b5b8bbf5 FG |
4084 | "osd": 0, |
4085 | "primary": false | |
31f18b77 FG |
4086 | }, |
4087 | { | |
4088 | "data_digest": "0x00000000", | |
4089 | "omap_digest": "0xffffffff", | |
94b18763 FG |
4090 | "object_info": { |
4091 | "oid": { | |
4092 | "oid": "EOBJ5", | |
4093 | "key": "", | |
4094 | "snapid": -2, | |
4095 | "hash": 2918945441, | |
4096 | "max": 0, | |
4097 | "pool": 3, | |
4098 | "namespace": "" | |
4099 | }, | |
4100 | "version": "59'7", | |
4101 | "prior_version": "0'0", | |
4102 | "last_reqid": "client.4382.0:1", | |
4103 | "user_version": 7, | |
4104 | "size": 7, | |
4105 | "mtime": "2018-04-05 14:32:12.929161", | |
4106 | "local_mtime": "2018-04-05 14:32:12.934707", | |
4107 | "lost": 0, | |
4108 | "flags": [ | |
4109 | "dirty", | |
28e407b8 | 4110 | "data_digest" |
94b18763 FG |
4111 | ], |
4112 | "truncate_seq": 0, | |
4113 | "truncate_size": 0, | |
4114 | "data_digest": "0x2ddbf8f5", | |
4115 | "omap_digest": "0xffffffff", | |
4116 | "expected_object_size": 0, | |
4117 | "expected_write_size": 0, | |
4118 | "alloc_hint_flags": 0, | |
4119 | "manifest": { | |
4120 | "type": 0 | |
4121 | }, | |
4122 | "watchers": {} | |
4123 | }, | |
31f18b77 FG |
4124 | "size": 4096, |
4125 | "errors": [ | |
94b18763 FG |
4126 | "size_mismatch_info", |
4127 | "obj_size_info_mismatch" | |
31f18b77 | 4128 | ], |
224ce89b | 4129 | "shard": 0, |
b5b8bbf5 FG |
4130 | "osd": 1, |
4131 | "primary": true | |
31f18b77 FG |
4132 | }, |
4133 | { | |
4134 | "data_digest": "0x00000000", | |
4135 | "omap_digest": "0xffffffff", | |
4136 | "size": 2048, | |
4137 | "errors": [], | |
224ce89b | 4138 | "shard": 1, |
b5b8bbf5 FG |
4139 | "osd": 2, |
4140 | "primary": false | |
31f18b77 FG |
4141 | } |
4142 | ], | |
94b18763 FG |
4143 | "selected_object_info": { |
4144 | "oid": { | |
4145 | "oid": "EOBJ5", | |
4146 | "key": "", | |
4147 | "snapid": -2, | |
4148 | "hash": 2918945441, | |
4149 | "max": 0, | |
4150 | "pool": 3, | |
4151 | "namespace": "" | |
4152 | }, | |
4153 | "version": "59'7", | |
4154 | "prior_version": "0'0", | |
4155 | "last_reqid": "client.4382.0:1", | |
4156 | "user_version": 7, | |
4157 | "size": 7, | |
4158 | "mtime": "2018-04-05 14:32:12.929161", | |
4159 | "local_mtime": "2018-04-05 14:32:12.934707", | |
4160 | "lost": 0, | |
4161 | "flags": [ | |
4162 | "dirty", | |
28e407b8 | 4163 | "data_digest" |
94b18763 FG |
4164 | ], |
4165 | "truncate_seq": 0, | |
4166 | "truncate_size": 0, | |
4167 | "data_digest": "0x2ddbf8f5", | |
4168 | "omap_digest": "0xffffffff", | |
4169 | "expected_object_size": 0, | |
4170 | "expected_write_size": 0, | |
4171 | "alloc_hint_flags": 0, | |
4172 | "manifest": { | |
4173 | "type": 0 | |
4174 | }, | |
4175 | "watchers": {} | |
4176 | }, | |
31f18b77 | 4177 | "union_shard_errors": [ |
94b18763 FG |
4178 | "size_mismatch_info", |
4179 | "obj_size_info_mismatch" | |
31f18b77 FG |
4180 | ], |
4181 | "errors": [ | |
4182 | "size_mismatch" | |
4183 | ], | |
4184 | "object": { | |
4185 | "version": 7, | |
4186 | "snap": "head", | |
4187 | "locator": "", | |
4188 | "nspace": "", | |
4189 | "name": "EOBJ5" | |
4190 | } | |
94b18763 FG |
4191 | }, |
4192 | { | |
4193 | "object": { | |
4194 | "name": "EOBJ6", | |
4195 | "nspace": "", | |
4196 | "locator": "", | |
4197 | "snap": "head", | |
4198 | "version": 8 | |
4199 | }, | |
4200 | "errors": [], | |
4201 | "union_shard_errors": [ | |
4202 | "read_error", | |
4203 | "hinfo_missing", | |
4204 | "hinfo_corrupted" | |
4205 | ], | |
4206 | "selected_object_info": { | |
4207 | "oid": { | |
4208 | "oid": "EOBJ6", | |
4209 | "key": "", | |
4210 | "snapid": -2, | |
4211 | "hash": 3050890866, | |
4212 | "max": 0, | |
4213 | "pool": 3, | |
4214 | "namespace": "" | |
4215 | }, | |
4216 | "version": "65'8", | |
4217 | "prior_version": "0'0", | |
4218 | "last_reqid": "client.4418.0:1", | |
4219 | "user_version": 8, | |
4220 | "size": 7, | |
4221 | "mtime": "2018-04-05 14:32:20.634116", | |
4222 | "local_mtime": "2018-04-05 14:32:20.637999", | |
4223 | "lost": 0, | |
4224 | "flags": [ | |
4225 | "dirty", | |
28e407b8 | 4226 | "data_digest" |
94b18763 FG |
4227 | ], |
4228 | "truncate_seq": 0, | |
4229 | "truncate_size": 0, | |
4230 | "data_digest": "0x2ddbf8f5", | |
4231 | "omap_digest": "0xffffffff", | |
4232 | "expected_object_size": 0, | |
4233 | "expected_write_size": 0, | |
4234 | "alloc_hint_flags": 0, | |
4235 | "manifest": { | |
4236 | "type": 0 | |
4237 | }, | |
4238 | "watchers": {} | |
4239 | }, | |
4240 | "shards": [ | |
4241 | { | |
4242 | "osd": 0, | |
4243 | "primary": false, | |
4244 | "shard": 2, | |
4245 | "errors": [ | |
4246 | "read_error", | |
4247 | "hinfo_missing" | |
4248 | ], | |
4249 | "size": 2048 | |
4250 | }, | |
4251 | { | |
4252 | "osd": 1, | |
4253 | "primary": true, | |
4254 | "shard": 0, | |
4255 | "errors": [ | |
4256 | "read_error", | |
4257 | "hinfo_corrupted" | |
4258 | ], | |
4259 | "size": 2048, | |
4260 | "hashinfo": "bad-val" | |
4261 | }, | |
4262 | { | |
4263 | "osd": 2, | |
4264 | "primary": false, | |
4265 | "shard": 1, | |
4266 | "errors": [], | |
4267 | "size": 2048, | |
4268 | "omap_digest": "0xffffffff", | |
4269 | "data_digest": "0x00000000", | |
4270 | "hashinfo": { | |
4271 | "cumulative_shard_hashes": [ | |
4272 | { | |
4273 | "hash": 80717615, | |
4274 | "shard": 0 | |
4275 | }, | |
4276 | { | |
4277 | "hash": 1534491824, | |
4278 | "shard": 1 | |
4279 | }, | |
4280 | { | |
4281 | "hash": 80717615, | |
4282 | "shard": 2 | |
4283 | } | |
4284 | ], | |
4285 | "total_chunk_size": 2048 | |
4286 | } | |
4287 | } | |
4288 | ] | |
4289 | }, | |
4290 | { | |
4291 | "object": { | |
4292 | "name": "EOBJ7", | |
4293 | "nspace": "", | |
4294 | "locator": "", | |
4295 | "snap": "head", | |
4296 | "version": 10 | |
4297 | }, | |
4298 | "errors": [ | |
4299 | "hinfo_inconsistency" | |
4300 | ], | |
4301 | "union_shard_errors": [], | |
4302 | "selected_object_info": { | |
4303 | "oid": { | |
4304 | "oid": "EOBJ7", | |
4305 | "key": "", | |
4306 | "snapid": -2, | |
4307 | "hash": 3258066308, | |
4308 | "max": 0, | |
4309 | "pool": 3, | |
4310 | "namespace": "" | |
4311 | }, | |
4312 | "version": "75'10", | |
4313 | "prior_version": "75'9", | |
4314 | "last_reqid": "client.4482.0:1", | |
4315 | "user_version": 10, | |
4316 | "size": 34, | |
4317 | "mtime": "2018-04-05 14:32:33.058782", | |
4318 | "local_mtime": "2018-04-05 14:32:33.059679", | |
4319 | "lost": 0, | |
4320 | "flags": [ | |
4321 | "dirty", | |
28e407b8 | 4322 | "data_digest" |
94b18763 FG |
4323 | ], |
4324 | "truncate_seq": 0, | |
4325 | "truncate_size": 0, | |
4326 | "data_digest": "0x136e4e27", | |
4327 | "omap_digest": "0xffffffff", | |
4328 | "expected_object_size": 0, | |
4329 | "expected_write_size": 0, | |
4330 | "alloc_hint_flags": 0, | |
4331 | "manifest": { | |
4332 | "type": 0 | |
4333 | }, | |
4334 | "watchers": {} | |
4335 | }, | |
4336 | "shards": [ | |
4337 | { | |
4338 | "osd": 0, | |
4339 | "primary": false, | |
4340 | "shard": 2, | |
4341 | "errors": [], | |
4342 | "size": 2048, | |
4343 | "omap_digest": "0xffffffff", | |
4344 | "data_digest": "0x00000000", | |
4345 | "hashinfo": { | |
4346 | "cumulative_shard_hashes": [ | |
4347 | { | |
4348 | "hash": 80717615, | |
4349 | "shard": 0 | |
4350 | }, | |
4351 | { | |
4352 | "hash": 1534491824, | |
4353 | "shard": 1 | |
4354 | }, | |
4355 | { | |
4356 | "hash": 80717615, | |
4357 | "shard": 2 | |
4358 | } | |
4359 | ], | |
4360 | "total_chunk_size": 2048 | |
4361 | } | |
4362 | }, | |
4363 | { | |
4364 | "osd": 1, | |
4365 | "primary": true, | |
4366 | "shard": 0, | |
4367 | "errors": [], | |
4368 | "size": 2048, | |
4369 | "omap_digest": "0xffffffff", | |
4370 | "data_digest": "0x00000000", | |
4371 | "hashinfo": { | |
4372 | "cumulative_shard_hashes": [ | |
4373 | { | |
4374 | "hash": 1534350760, | |
4375 | "shard": 0 | |
4376 | }, | |
4377 | { | |
4378 | "hash": 1534491824, | |
4379 | "shard": 1 | |
4380 | }, | |
4381 | { | |
4382 | "hash": 1534350760, | |
4383 | "shard": 2 | |
4384 | } | |
4385 | ], | |
4386 | "total_chunk_size": 2048 | |
4387 | } | |
4388 | }, | |
4389 | { | |
4390 | "osd": 2, | |
4391 | "primary": false, | |
4392 | "shard": 1, | |
4393 | "errors": [], | |
4394 | "size": 2048, | |
4395 | "omap_digest": "0xffffffff", | |
4396 | "data_digest": "0x00000000", | |
4397 | "hashinfo": { | |
4398 | "cumulative_shard_hashes": [ | |
4399 | { | |
4400 | "hash": 1534350760, | |
4401 | "shard": 0 | |
4402 | }, | |
4403 | { | |
4404 | "hash": 1534491824, | |
4405 | "shard": 1 | |
4406 | }, | |
4407 | { | |
4408 | "hash": 1534350760, | |
4409 | "shard": 2 | |
4410 | } | |
4411 | ], | |
4412 | "total_chunk_size": 2048 | |
4413 | } | |
4414 | } | |
4415 | ] | |
31f18b77 FG |
4416 | } |
4417 | ], | |
4418 | "epoch": 0 | |
4419 | } | |
4420 | EOF | |
4421 | ||
4422 | else | |
4423 | ||
94b18763 | 4424 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
4425 | { |
4426 | "inconsistents": [ | |
4427 | { | |
4428 | "shards": [ | |
4429 | { | |
4430 | "data_digest": "0x04cfa72f", | |
4431 | "omap_digest": "0xffffffff", | |
4432 | "size": 2048, | |
4433 | "errors": [], | |
4434 | "shard": 2, | |
b5b8bbf5 FG |
4435 | "osd": 0, |
4436 | "primary": false | |
7c673cae FG |
4437 | }, |
4438 | { | |
94b18763 FG |
4439 | "object_info": { |
4440 | "oid": { | |
4441 | "oid": "EOBJ1", | |
4442 | "key": "", | |
4443 | "snapid": -2, | |
4444 | "hash": 560836233, | |
4445 | "max": 0, | |
4446 | "pool": 3, | |
4447 | "namespace": "" | |
4448 | }, | |
4449 | "version": "27'1", | |
4450 | "prior_version": "0'0", | |
4451 | "last_reqid": "client.4192.0:1", | |
4452 | "user_version": 1, | |
4453 | "size": 7, | |
4454 | "mtime": "2018-04-05 14:30:10.688009", | |
4455 | "local_mtime": "2018-04-05 14:30:10.691774", | |
4456 | "lost": 0, | |
4457 | "flags": [ | |
4458 | "dirty", | |
28e407b8 | 4459 | "data_digest" |
94b18763 FG |
4460 | ], |
4461 | "truncate_seq": 0, | |
4462 | "truncate_size": 0, | |
4463 | "data_digest": "0x2ddbf8f5", | |
4464 | "omap_digest": "0xffffffff", | |
4465 | "expected_object_size": 0, | |
4466 | "expected_write_size": 0, | |
4467 | "alloc_hint_flags": 0, | |
4468 | "manifest": { | |
4469 | "type": 0 | |
4470 | }, | |
4471 | "watchers": {} | |
4472 | }, | |
7c673cae | 4473 | "size": 9, |
224ce89b | 4474 | "shard": 0, |
7c673cae FG |
4475 | "errors": [ |
4476 | "read_error", | |
94b18763 FG |
4477 | "size_mismatch_info", |
4478 | "obj_size_info_mismatch" | |
7c673cae | 4479 | ], |
b5b8bbf5 FG |
4480 | "osd": 1, |
4481 | "primary": true | |
7c673cae FG |
4482 | }, |
4483 | { | |
4484 | "data_digest": "0x04cfa72f", | |
4485 | "omap_digest": "0xffffffff", | |
4486 | "size": 2048, | |
224ce89b | 4487 | "shard": 1, |
7c673cae | 4488 | "errors": [], |
b5b8bbf5 FG |
4489 | "osd": 2, |
4490 | "primary": false | |
7c673cae FG |
4491 | } |
4492 | ], | |
94b18763 FG |
4493 | "selected_object_info": { |
4494 | "oid": { | |
4495 | "oid": "EOBJ1", | |
4496 | "key": "", | |
4497 | "snapid": -2, | |
4498 | "hash": 560836233, | |
4499 | "max": 0, | |
4500 | "pool": 3, | |
4501 | "namespace": "" | |
4502 | }, | |
4503 | "version": "27'1", | |
4504 | "prior_version": "0'0", | |
4505 | "last_reqid": "client.4192.0:1", | |
4506 | "user_version": 1, | |
4507 | "size": 7, | |
4508 | "mtime": "2018-04-05 14:30:10.688009", | |
4509 | "local_mtime": "2018-04-05 14:30:10.691774", | |
4510 | "lost": 0, | |
4511 | "flags": [ | |
4512 | "dirty", | |
28e407b8 | 4513 | "data_digest" |
94b18763 FG |
4514 | ], |
4515 | "truncate_seq": 0, | |
4516 | "truncate_size": 0, | |
4517 | "data_digest": "0x2ddbf8f5", | |
4518 | "omap_digest": "0xffffffff", | |
4519 | "expected_object_size": 0, | |
4520 | "expected_write_size": 0, | |
4521 | "alloc_hint_flags": 0, | |
4522 | "manifest": { | |
4523 | "type": 0 | |
4524 | }, | |
4525 | "watchers": {} | |
4526 | }, | |
7c673cae FG |
4527 | "union_shard_errors": [ |
4528 | "read_error", | |
94b18763 FG |
4529 | "size_mismatch_info", |
4530 | "obj_size_info_mismatch" | |
7c673cae FG |
4531 | ], |
4532 | "errors": [ | |
4533 | "size_mismatch" | |
4534 | ], | |
4535 | "object": { | |
4536 | "version": 1, | |
4537 | "snap": "head", | |
4538 | "locator": "", | |
4539 | "nspace": "", | |
4540 | "name": "EOBJ1" | |
4541 | } | |
4542 | }, | |
4543 | { | |
4544 | "shards": [ | |
4545 | { | |
4546 | "size": 2048, | |
4547 | "errors": [ | |
4548 | "ec_hash_error" | |
4549 | ], | |
4550 | "shard": 2, | |
b5b8bbf5 FG |
4551 | "osd": 0, |
4552 | "primary": false | |
7c673cae FG |
4553 | }, |
4554 | { | |
4555 | "data_digest": "0x04cfa72f", | |
4556 | "omap_digest": "0xffffffff", | |
4557 | "size": 2048, | |
4558 | "errors": [], | |
224ce89b | 4559 | "shard": 0, |
b5b8bbf5 FG |
4560 | "osd": 1, |
4561 | "primary": true | |
7c673cae FG |
4562 | }, |
4563 | { | |
4564 | "data_digest": "0x04cfa72f", | |
4565 | "omap_digest": "0xffffffff", | |
4566 | "size": 2048, | |
4567 | "errors": [], | |
224ce89b | 4568 | "shard": 1, |
b5b8bbf5 FG |
4569 | "osd": 2, |
4570 | "primary": false | |
7c673cae FG |
4571 | } |
4572 | ], | |
94b18763 FG |
4573 | "selected_object_info": { |
4574 | "oid": { | |
4575 | "oid": "EOBJ2", | |
4576 | "key": "", | |
4577 | "snapid": -2, | |
4578 | "hash": 562812377, | |
4579 | "max": 0, | |
4580 | "pool": 3, | |
4581 | "namespace": "" | |
4582 | }, | |
4583 | "version": "33'2", | |
4584 | "prior_version": "0'0", | |
4585 | "last_reqid": "client.4224.0:1", | |
4586 | "user_version": 2, | |
4587 | "size": 7, | |
4588 | "mtime": "2018-04-05 14:30:14.152945", | |
4589 | "local_mtime": "2018-04-05 14:30:14.154014", | |
4590 | "lost": 0, | |
4591 | "flags": [ | |
4592 | "dirty", | |
28e407b8 | 4593 | "data_digest" |
94b18763 FG |
4594 | ], |
4595 | "truncate_seq": 0, | |
4596 | "truncate_size": 0, | |
4597 | "data_digest": "0x2ddbf8f5", | |
4598 | "omap_digest": "0xffffffff", | |
4599 | "expected_object_size": 0, | |
4600 | "expected_write_size": 0, | |
4601 | "alloc_hint_flags": 0, | |
4602 | "manifest": { | |
4603 | "type": 0 | |
4604 | }, | |
4605 | "watchers": {} | |
4606 | }, | |
7c673cae FG |
4607 | "union_shard_errors": [ |
4608 | "ec_hash_error" | |
4609 | ], | |
4610 | "errors": [], | |
4611 | "object": { | |
4612 | "version": 2, | |
4613 | "snap": "head", | |
4614 | "locator": "", | |
4615 | "nspace": "", | |
4616 | "name": "EOBJ2" | |
4617 | } | |
4618 | }, | |
4619 | { | |
4620 | "shards": [ | |
4621 | { | |
4622 | "data_digest": "0x04cfa72f", | |
4623 | "omap_digest": "0xffffffff", | |
4624 | "size": 2048, | |
4625 | "errors": [], | |
4626 | "shard": 2, | |
b5b8bbf5 FG |
4627 | "osd": 0, |
4628 | "primary": false | |
7c673cae FG |
4629 | }, |
4630 | { | |
224ce89b | 4631 | "osd": 1, |
b5b8bbf5 | 4632 | "primary": true, |
224ce89b | 4633 | "shard": 0, |
7c673cae FG |
4634 | "errors": [ |
4635 | "missing" | |
224ce89b | 4636 | ] |
7c673cae FG |
4637 | }, |
4638 | { | |
4639 | "data_digest": "0x04cfa72f", | |
4640 | "omap_digest": "0xffffffff", | |
4641 | "size": 2048, | |
224ce89b | 4642 | "shard": 1, |
7c673cae | 4643 | "errors": [], |
b5b8bbf5 FG |
4644 | "osd": 2, |
4645 | "primary": false | |
7c673cae FG |
4646 | } |
4647 | ], | |
94b18763 FG |
4648 | "selected_object_info": { |
4649 | "oid": { | |
4650 | "oid": "EOBJ3", | |
4651 | "key": "", | |
4652 | "snapid": -2, | |
4653 | "hash": 3125668237, | |
4654 | "max": 0, | |
4655 | "pool": 3, | |
4656 | "namespace": "" | |
4657 | }, | |
4658 | "version": "39'3", | |
4659 | "prior_version": "0'0", | |
4660 | "last_reqid": "client.4258.0:1", | |
4661 | "user_version": 3, | |
4662 | "size": 7, | |
4663 | "mtime": "2018-04-05 14:30:18.875544", | |
4664 | "local_mtime": "2018-04-05 14:30:18.880153", | |
4665 | "lost": 0, | |
4666 | "flags": [ | |
4667 | "dirty", | |
28e407b8 | 4668 | "data_digest" |
94b18763 FG |
4669 | ], |
4670 | "truncate_seq": 0, | |
4671 | "truncate_size": 0, | |
4672 | "data_digest": "0x2ddbf8f5", | |
4673 | "omap_digest": "0xffffffff", | |
4674 | "expected_object_size": 0, | |
4675 | "expected_write_size": 0, | |
4676 | "alloc_hint_flags": 0, | |
4677 | "manifest": { | |
4678 | "type": 0 | |
4679 | }, | |
4680 | "watchers": {} | |
4681 | }, | |
7c673cae FG |
4682 | "union_shard_errors": [ |
4683 | "missing" | |
4684 | ], | |
4685 | "errors": [], | |
4686 | "object": { | |
4687 | "version": 3, | |
4688 | "snap": "head", | |
4689 | "locator": "", | |
4690 | "nspace": "", | |
4691 | "name": "EOBJ3" | |
4692 | } | |
4693 | }, | |
4694 | { | |
4695 | "shards": [ | |
4696 | { | |
4697 | "attrs": [ | |
7c673cae FG |
4698 | { |
4699 | "Base64": false, | |
4700 | "value": "bad-val", | |
94b18763 | 4701 | "name": "key1-EOBJ4" |
7c673cae FG |
4702 | }, |
4703 | { | |
4704 | "Base64": false, | |
b32b8144 | 4705 | "value": "val2-EOBJ4", |
94b18763 | 4706 | "name": "key2-EOBJ4" |
7c673cae FG |
4707 | } |
4708 | ], | |
4709 | "data_digest": "0x04cfa72f", | |
4710 | "omap_digest": "0xffffffff", | |
4711 | "size": 2048, | |
4712 | "errors": [], | |
4713 | "shard": 2, | |
b5b8bbf5 FG |
4714 | "osd": 0, |
4715 | "primary": false | |
7c673cae FG |
4716 | }, |
4717 | { | |
224ce89b | 4718 | "osd": 1, |
b5b8bbf5 | 4719 | "primary": true, |
224ce89b WB |
4720 | "shard": 0, |
4721 | "errors": [], | |
4722 | "size": 2048, | |
4723 | "omap_digest": "0xffffffff", | |
4724 | "data_digest": "0x04cfa72f", | |
7c673cae | 4725 | "attrs": [ |
7c673cae FG |
4726 | { |
4727 | "Base64": false, | |
4728 | "value": "val1-EOBJ4", | |
94b18763 | 4729 | "name": "key1-EOBJ4" |
7c673cae FG |
4730 | }, |
4731 | { | |
4732 | "Base64": false, | |
4733 | "value": "val2-EOBJ4", | |
94b18763 | 4734 | "name": "key2-EOBJ4" |
7c673cae | 4735 | } |
224ce89b | 4736 | ] |
7c673cae FG |
4737 | }, |
4738 | { | |
224ce89b | 4739 | "osd": 2, |
b5b8bbf5 | 4740 | "primary": false, |
224ce89b WB |
4741 | "shard": 1, |
4742 | "errors": [], | |
4743 | "size": 2048, | |
4744 | "omap_digest": "0xffffffff", | |
4745 | "data_digest": "0x04cfa72f", | |
7c673cae | 4746 | "attrs": [ |
7c673cae FG |
4747 | { |
4748 | "Base64": false, | |
4749 | "value": "val1-EOBJ4", | |
94b18763 | 4750 | "name": "key1-EOBJ4" |
7c673cae FG |
4751 | }, |
4752 | { | |
4753 | "Base64": false, | |
b32b8144 | 4754 | "value": "val3-EOBJ4", |
94b18763 | 4755 | "name": "key3-EOBJ4" |
7c673cae | 4756 | } |
224ce89b | 4757 | ] |
7c673cae FG |
4758 | } |
4759 | ], | |
94b18763 FG |
4760 | "selected_object_info": { |
4761 | "oid": { | |
4762 | "oid": "EOBJ4", | |
4763 | "key": "", | |
4764 | "snapid": -2, | |
4765 | "hash": 1618759290, | |
4766 | "max": 0, | |
4767 | "pool": 3, | |
4768 | "namespace": "" | |
4769 | }, | |
4770 | "version": "45'6", | |
4771 | "prior_version": "45'5", | |
4772 | "last_reqid": "client.4296.0:1", | |
4773 | "user_version": 6, | |
4774 | "size": 7, | |
4775 | "mtime": "2018-04-05 14:30:22.271983", | |
4776 | "local_mtime": "2018-04-05 14:30:22.272840", | |
4777 | "lost": 0, | |
4778 | "flags": [ | |
4779 | "dirty", | |
28e407b8 | 4780 | "data_digest" |
94b18763 FG |
4781 | ], |
4782 | "truncate_seq": 0, | |
4783 | "truncate_size": 0, | |
4784 | "data_digest": "0x2ddbf8f5", | |
4785 | "omap_digest": "0xffffffff", | |
4786 | "expected_object_size": 0, | |
4787 | "expected_write_size": 0, | |
4788 | "alloc_hint_flags": 0, | |
4789 | "manifest": { | |
4790 | "type": 0 | |
4791 | }, | |
4792 | "watchers": {} | |
4793 | }, | |
7c673cae FG |
4794 | "union_shard_errors": [], |
4795 | "errors": [ | |
4796 | "attr_value_mismatch", | |
4797 | "attr_name_mismatch" | |
4798 | ], | |
4799 | "object": { | |
4800 | "version": 6, | |
4801 | "snap": "head", | |
4802 | "locator": "", | |
4803 | "nspace": "", | |
4804 | "name": "EOBJ4" | |
4805 | } | |
4806 | }, | |
4807 | { | |
4808 | "shards": [ | |
4809 | { | |
4810 | "data_digest": "0x04cfa72f", | |
4811 | "omap_digest": "0xffffffff", | |
4812 | "size": 2048, | |
4813 | "errors": [], | |
4814 | "shard": 2, | |
b5b8bbf5 FG |
4815 | "osd": 0, |
4816 | "primary": false | |
7c673cae FG |
4817 | }, |
4818 | { | |
94b18763 FG |
4819 | "object_info": { |
4820 | "oid": { | |
4821 | "oid": "EOBJ5", | |
4822 | "key": "", | |
4823 | "snapid": -2, | |
4824 | "hash": 2918945441, | |
4825 | "max": 0, | |
4826 | "pool": 3, | |
4827 | "namespace": "" | |
4828 | }, | |
4829 | "version": "59'7", | |
4830 | "prior_version": "0'0", | |
4831 | "last_reqid": "client.4384.0:1", | |
4832 | "user_version": 7, | |
4833 | "size": 7, | |
4834 | "mtime": "2018-04-05 14:30:35.162395", | |
4835 | "local_mtime": "2018-04-05 14:30:35.166390", | |
4836 | "lost": 0, | |
4837 | "flags": [ | |
4838 | "dirty", | |
28e407b8 | 4839 | "data_digest" |
94b18763 FG |
4840 | ], |
4841 | "truncate_seq": 0, | |
4842 | "truncate_size": 0, | |
4843 | "data_digest": "0x2ddbf8f5", | |
4844 | "omap_digest": "0xffffffff", | |
4845 | "expected_object_size": 0, | |
4846 | "expected_write_size": 0, | |
4847 | "alloc_hint_flags": 0, | |
4848 | "manifest": { | |
4849 | "type": 0 | |
4850 | }, | |
4851 | "watchers": {} | |
4852 | }, | |
7c673cae | 4853 | "size": 4096, |
224ce89b | 4854 | "shard": 0, |
7c673cae | 4855 | "errors": [ |
94b18763 | 4856 | "size_mismatch_info", |
b5b8bbf5 | 4857 | "ec_size_error", |
94b18763 | 4858 | "obj_size_info_mismatch" |
7c673cae | 4859 | ], |
b5b8bbf5 FG |
4860 | "osd": 1, |
4861 | "primary": true | |
7c673cae FG |
4862 | }, |
4863 | { | |
4864 | "data_digest": "0x04cfa72f", | |
4865 | "omap_digest": "0xffffffff", | |
4866 | "size": 2048, | |
224ce89b | 4867 | "shard": 1, |
7c673cae | 4868 | "errors": [], |
b5b8bbf5 FG |
4869 | "osd": 2, |
4870 | "primary": false | |
7c673cae FG |
4871 | } |
4872 | ], | |
94b18763 FG |
4873 | "selected_object_info": { |
4874 | "oid": { | |
4875 | "oid": "EOBJ5", | |
4876 | "key": "", | |
4877 | "snapid": -2, | |
4878 | "hash": 2918945441, | |
4879 | "max": 0, | |
4880 | "pool": 3, | |
4881 | "namespace": "" | |
4882 | }, | |
4883 | "version": "59'7", | |
4884 | "prior_version": "0'0", | |
4885 | "last_reqid": "client.4384.0:1", | |
4886 | "user_version": 7, | |
4887 | "size": 7, | |
4888 | "mtime": "2018-04-05 14:30:35.162395", | |
4889 | "local_mtime": "2018-04-05 14:30:35.166390", | |
4890 | "lost": 0, | |
4891 | "flags": [ | |
4892 | "dirty", | |
28e407b8 | 4893 | "data_digest" |
94b18763 FG |
4894 | ], |
4895 | "truncate_seq": 0, | |
4896 | "truncate_size": 0, | |
4897 | "data_digest": "0x2ddbf8f5", | |
4898 | "omap_digest": "0xffffffff", | |
4899 | "expected_object_size": 0, | |
4900 | "expected_write_size": 0, | |
4901 | "alloc_hint_flags": 0, | |
4902 | "manifest": { | |
4903 | "type": 0 | |
4904 | }, | |
4905 | "watchers": {} | |
4906 | }, | |
7c673cae | 4907 | "union_shard_errors": [ |
94b18763 | 4908 | "size_mismatch_info", |
b5b8bbf5 | 4909 | "ec_size_error", |
94b18763 | 4910 | "obj_size_info_mismatch" |
7c673cae FG |
4911 | ], |
4912 | "errors": [ | |
4913 | "size_mismatch" | |
4914 | ], | |
4915 | "object": { | |
4916 | "version": 7, | |
4917 | "snap": "head", | |
4918 | "locator": "", | |
4919 | "nspace": "", | |
4920 | "name": "EOBJ5" | |
4921 | } | |
94b18763 FG |
4922 | }, |
4923 | { | |
4924 | "object": { | |
4925 | "name": "EOBJ6", | |
4926 | "nspace": "", | |
4927 | "locator": "", | |
4928 | "snap": "head", | |
4929 | "version": 8 | |
4930 | }, | |
4931 | "errors": [], | |
4932 | "union_shard_errors": [ | |
4933 | "read_error", | |
4934 | "hinfo_missing", | |
4935 | "hinfo_corrupted" | |
4936 | ], | |
4937 | "selected_object_info": { | |
4938 | "oid": { | |
4939 | "oid": "EOBJ6", | |
4940 | "key": "", | |
4941 | "snapid": -2, | |
4942 | "hash": 3050890866, | |
4943 | "max": 0, | |
4944 | "pool": 3, | |
4945 | "namespace": "" | |
4946 | }, | |
4947 | "version": "65'8", | |
4948 | "prior_version": "0'0", | |
4949 | "last_reqid": "client.4420.0:1", | |
4950 | "user_version": 8, | |
4951 | "size": 7, | |
4952 | "mtime": "2018-04-05 14:30:40.914673", | |
4953 | "local_mtime": "2018-04-05 14:30:40.917705", | |
4954 | "lost": 0, | |
4955 | "flags": [ | |
4956 | "dirty", | |
28e407b8 | 4957 | "data_digest" |
94b18763 FG |
4958 | ], |
4959 | "truncate_seq": 0, | |
4960 | "truncate_size": 0, | |
4961 | "data_digest": "0x2ddbf8f5", | |
4962 | "omap_digest": "0xffffffff", | |
4963 | "expected_object_size": 0, | |
4964 | "expected_write_size": 0, | |
4965 | "alloc_hint_flags": 0, | |
4966 | "manifest": { | |
4967 | "type": 0 | |
4968 | }, | |
4969 | "watchers": {} | |
4970 | }, | |
4971 | "shards": [ | |
4972 | { | |
4973 | "osd": 0, | |
4974 | "primary": false, | |
4975 | "shard": 2, | |
4976 | "errors": [ | |
4977 | "read_error", | |
4978 | "hinfo_missing" | |
4979 | ], | |
4980 | "size": 2048 | |
4981 | }, | |
4982 | { | |
4983 | "osd": 1, | |
4984 | "primary": true, | |
4985 | "shard": 0, | |
4986 | "errors": [ | |
4987 | "read_error", | |
4988 | "hinfo_corrupted" | |
4989 | ], | |
4990 | "size": 2048, | |
4991 | "hashinfo": "bad-val" | |
4992 | }, | |
4993 | { | |
4994 | "osd": 2, | |
4995 | "primary": false, | |
4996 | "shard": 1, | |
4997 | "errors": [], | |
4998 | "size": 2048, | |
4999 | "omap_digest": "0xffffffff", | |
5000 | "data_digest": "0x04cfa72f", | |
5001 | "hashinfo": { | |
5002 | "cumulative_shard_hashes": [ | |
5003 | { | |
5004 | "hash": 80717615, | |
5005 | "shard": 0 | |
5006 | }, | |
5007 | { | |
5008 | "hash": 1534491824, | |
5009 | "shard": 1 | |
5010 | }, | |
5011 | { | |
5012 | "hash": 80717615, | |
5013 | "shard": 2 | |
5014 | } | |
5015 | ], | |
5016 | "total_chunk_size": 2048 | |
5017 | } | |
5018 | } | |
5019 | ] | |
5020 | }, | |
5021 | { | |
5022 | "object": { | |
5023 | "name": "EOBJ7", | |
5024 | "nspace": "", | |
5025 | "locator": "", | |
5026 | "snap": "head", | |
5027 | "version": 10 | |
5028 | }, | |
5029 | "errors": [ | |
5030 | "hinfo_inconsistency" | |
5031 | ], | |
5032 | "union_shard_errors": [ | |
5033 | "ec_hash_error" | |
5034 | ], | |
5035 | "selected_object_info": { | |
5036 | "oid": { | |
5037 | "oid": "EOBJ7", | |
5038 | "key": "", | |
5039 | "snapid": -2, | |
5040 | "hash": 3258066308, | |
5041 | "max": 0, | |
5042 | "pool": 3, | |
5043 | "namespace": "" | |
5044 | }, | |
5045 | "version": "75'10", | |
5046 | "prior_version": "75'9", | |
5047 | "last_reqid": "client.4486.0:1", | |
5048 | "user_version": 10, | |
5049 | "size": 34, | |
5050 | "mtime": "2018-04-05 14:30:50.995009", | |
5051 | "local_mtime": "2018-04-05 14:30:50.996112", | |
5052 | "lost": 0, | |
5053 | "flags": [ | |
5054 | "dirty", | |
28e407b8 | 5055 | "data_digest" |
94b18763 FG |
5056 | ], |
5057 | "truncate_seq": 0, | |
5058 | "truncate_size": 0, | |
5059 | "data_digest": "0x136e4e27", | |
5060 | "omap_digest": "0xffffffff", | |
5061 | "expected_object_size": 0, | |
5062 | "expected_write_size": 0, | |
5063 | "alloc_hint_flags": 0, | |
5064 | "manifest": { | |
5065 | "type": 0 | |
5066 | }, | |
5067 | "watchers": {} | |
5068 | }, | |
5069 | "shards": [ | |
5070 | { | |
5071 | "osd": 0, | |
5072 | "primary": false, | |
5073 | "shard": 2, | |
5074 | "errors": [ | |
5075 | "ec_hash_error" | |
5076 | ], | |
5077 | "size": 2048, | |
5078 | "hashinfo": { | |
5079 | "cumulative_shard_hashes": [ | |
5080 | { | |
5081 | "hash": 80717615, | |
5082 | "shard": 0 | |
5083 | }, | |
5084 | { | |
5085 | "hash": 1534491824, | |
5086 | "shard": 1 | |
5087 | }, | |
5088 | { | |
5089 | "hash": 80717615, | |
5090 | "shard": 2 | |
5091 | } | |
5092 | ], | |
5093 | "total_chunk_size": 2048 | |
5094 | } | |
5095 | }, | |
5096 | { | |
5097 | "osd": 1, | |
5098 | "primary": true, | |
5099 | "shard": 0, | |
5100 | "errors": [], | |
5101 | "size": 2048, | |
5102 | "omap_digest": "0xffffffff", | |
5103 | "data_digest": "0x5b7455a8", | |
5104 | "hashinfo": { | |
5105 | "cumulative_shard_hashes": [ | |
5106 | { | |
5107 | "hash": 1534350760, | |
5108 | "shard": 0 | |
5109 | }, | |
5110 | { | |
5111 | "hash": 1534491824, | |
5112 | "shard": 1 | |
5113 | }, | |
5114 | { | |
5115 | "hash": 1534350760, | |
5116 | "shard": 2 | |
5117 | } | |
5118 | ], | |
5119 | "total_chunk_size": 2048 | |
5120 | } | |
5121 | }, | |
5122 | { | |
5123 | "osd": 2, | |
5124 | "primary": false, | |
5125 | "shard": 1, | |
5126 | "errors": [], | |
5127 | "size": 2048, | |
5128 | "omap_digest": "0xffffffff", | |
5129 | "data_digest": "0x5b7455a8", | |
5130 | "hashinfo": { | |
5131 | "cumulative_shard_hashes": [ | |
5132 | { | |
5133 | "hash": 1534350760, | |
5134 | "shard": 0 | |
5135 | }, | |
5136 | { | |
5137 | "hash": 1534491824, | |
5138 | "shard": 1 | |
5139 | }, | |
5140 | { | |
5141 | "hash": 1534350760, | |
5142 | "shard": 2 | |
5143 | } | |
5144 | ], | |
5145 | "total_chunk_size": 2048 | |
5146 | } | |
5147 | } | |
5148 | ] | |
7c673cae FG |
5149 | } |
5150 | ], | |
5151 | "epoch": 0 | |
5152 | } | |
5153 | EOF | |
5154 | ||
7c673cae | 5155 | fi |
31f18b77 | 5156 | |
94b18763 | 5157 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
1adf2230 | 5158 | multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
5159 | if test $getjson = "yes" |
5160 | then | |
31f18b77 FG |
5161 | if [ "$allow_overwrites" = "true" ] |
5162 | then | |
5163 | num=4 | |
5164 | else | |
5165 | num=5 | |
5166 | fi | |
5167 | jq '.' $dir/json > save${num}.json | |
7c673cae FG |
5168 | fi |
5169 | ||
94b18763 | 5170 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
5171 | then |
5172 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
5173 | fi | |
5174 | ||
5175 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
5176 | teardown $dir || return 1 | |
5177 | } | |
5178 | ||
5179 | function TEST_corrupt_scrub_erasure_appends() { | |
5180 | corrupt_scrub_erasure $1 false | |
5181 | } | |
5182 | ||
5183 | function TEST_corrupt_scrub_erasure_overwrites() { | |
224ce89b WB |
5184 | if [ "$use_ec_overwrite" = "true" ]; then |
5185 | corrupt_scrub_erasure $1 true | |
5186 | fi | |
7c673cae FG |
5187 | } |
5188 | ||
5189 | # | |
5190 | # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost | |
5191 | # | |
5192 | function TEST_periodic_scrub_replicated() { | |
5193 | local dir=$1 | |
5194 | local poolname=psr_pool | |
5195 | local objname=POBJ | |
5196 | ||
5197 | setup $dir || return 1 | |
5198 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
5199 | run_mgr $dir x || return 1 | |
b5b8bbf5 FG |
5200 | local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 " |
5201 | ceph_osd_args+="--osd_scrub_backoff_ratio=0" | |
7c673cae FG |
5202 | run_osd $dir 0 $ceph_osd_args || return 1 |
5203 | run_osd $dir 1 $ceph_osd_args || return 1 | |
c07f9fc5 | 5204 | create_rbd_pool || return 1 |
7c673cae FG |
5205 | wait_for_clean || return 1 |
5206 | ||
b5b8bbf5 | 5207 | create_pool $poolname 1 1 || return 1 |
7c673cae FG |
5208 | wait_for_clean || return 1 |
5209 | ||
5210 | local osd=0 | |
5211 | add_something $dir $poolname $objname scrub || return 1 | |
5212 | local primary=$(get_primary $poolname $objname) | |
5213 | local pg=$(get_pg $poolname $objname) | |
5214 | ||
5215 | # Add deep-scrub only error | |
5216 | local payload=UVWXYZ | |
5217 | echo $payload > $dir/CORRUPT | |
5218 | # Uses $ceph_osd_args for osd restart | |
5219 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
5220 | ||
5221 | # No scrub information available, so expect failure | |
5222 | set -o pipefail | |
5223 | ! rados list-inconsistent-obj $pg | jq '.' || return 1 | |
5224 | set +o pipefail | |
5225 | ||
5226 | pg_deep_scrub $pg || return 1 | |
5227 | ||
5228 | # Make sure bad object found | |
5229 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
5230 | ||
b5b8bbf5 | 5231 | flush_pg_stats |
7c673cae FG |
5232 | local last_scrub=$(get_last_scrub_stamp $pg) |
5233 | # Fake a schedule scrub | |
c07f9fc5 | 5234 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
5235 | trigger_scrub $pg || return 1 |
5236 | # Wait for schedule regular scrub | |
5237 | wait_for_scrub $pg "$last_scrub" | |
5238 | ||
5239 | # It needed to be upgraded | |
5240 | grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 | |
5241 | ||
5242 | # Bad object still known | |
5243 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
5244 | ||
5245 | # Can't upgrade with this set | |
5246 | ceph osd set nodeep-scrub | |
224ce89b | 5247 | # Let map change propagate to OSDs |
b5b8bbf5 FG |
5248 | flush pg_stats |
5249 | sleep 5 | |
7c673cae FG |
5250 | |
5251 | # Fake a schedule scrub | |
c07f9fc5 | 5252 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
5253 | trigger_scrub $pg || return 1 |
5254 | # Wait for schedule regular scrub | |
5255 | # to notice scrub and skip it | |
5256 | local found=false | |
5257 | for i in $(seq 14 -1 0) | |
5258 | do | |
5259 | sleep 1 | |
5260 | ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } | |
5261 | echo Time left: $i seconds | |
5262 | done | |
5263 | test $found = "true" || return 1 | |
5264 | ||
5265 | # Bad object still known | |
5266 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
5267 | ||
b5b8bbf5 | 5268 | flush_pg_stats |
7c673cae | 5269 | # Request a regular scrub and it will be done |
7c673cae | 5270 | pg_scrub $pg |
c07f9fc5 | 5271 | grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1 |
7c673cae FG |
5272 | |
5273 | # deep-scrub error is no longer present | |
5274 | rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 | |
5275 | } | |
5276 | ||
3a9019d9 FG |
5277 | # |
5278 | # Corrupt snapset in replicated pool | |
5279 | # | |
5280 | function TEST_corrupt_snapset_scrub_rep() { | |
5281 | local dir=$1 | |
5282 | local poolname=csr_pool | |
5283 | local total_objs=2 | |
5284 | ||
5285 | setup $dir || return 1 | |
5286 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
5287 | run_mgr $dir x || return 1 | |
5288 | run_osd $dir 0 || return 1 | |
5289 | run_osd $dir 1 || return 1 | |
5290 | create_rbd_pool || return 1 | |
5291 | wait_for_clean || return 1 | |
5292 | ||
5293 | create_pool foo 1 || return 1 | |
5294 | create_pool $poolname 1 1 || return 1 | |
5295 | wait_for_clean || return 1 | |
5296 | ||
5297 | for i in $(seq 1 $total_objs) ; do | |
5298 | objname=ROBJ${i} | |
5299 | add_something $dir $poolname $objname || return 1 | |
5300 | ||
5301 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
5302 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
5303 | done | |
5304 | ||
5305 | local pg=$(get_pg $poolname ROBJ0) | |
91327a77 | 5306 | local primary=$(get_primary $poolname ROBJ0) |
3a9019d9 FG |
5307 | |
5308 | for i in $(seq 1 $total_objs) ; do | |
5309 | objname=ROBJ${i} | |
5310 | ||
5311 | # Alternate corruption between osd.0 and osd.1 | |
5312 | local osd=$(expr $i % 2) | |
5313 | ||
5314 | rados -p $poolname mksnap snap1 | |
5315 | echo -n head_of_snapshot_data > $dir/change | |
5316 | ||
5317 | case $i in | |
5318 | 1) | |
5319 | rados --pool $poolname put $objname $dir/change | |
5320 | objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1 | |
5321 | ;; | |
5322 | ||
5323 | 2) | |
5324 | rados --pool $poolname put $objname $dir/change | |
5325 | objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1 | |
5326 | ;; | |
5327 | ||
5328 | esac | |
5329 | done | |
5330 | rm $dir/change | |
5331 | ||
5332 | pg_scrub $pg | |
5333 | ||
5334 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
5335 | # Check pg count | |
5336 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
5337 | # Check pgid | |
5338 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
5339 | ||
5340 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
5341 | ||
94b18763 | 5342 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
3a9019d9 FG |
5343 | { |
5344 | "epoch": 34, | |
5345 | "inconsistents": [ | |
5346 | { | |
5347 | "object": { | |
5348 | "name": "ROBJ1", | |
5349 | "nspace": "", | |
5350 | "locator": "", | |
5351 | "snap": "head", | |
5352 | "version": 8 | |
5353 | }, | |
5354 | "errors": [ | |
5355 | "snapset_inconsistency" | |
5356 | ], | |
5357 | "union_shard_errors": [], | |
94b18763 FG |
5358 | "selected_object_info": { |
5359 | "oid": { | |
5360 | "oid": "ROBJ1", | |
5361 | "key": "", | |
5362 | "snapid": -2, | |
5363 | "hash": 1454963827, | |
5364 | "max": 0, | |
5365 | "pool": 3, | |
5366 | "namespace": "" | |
5367 | }, | |
5368 | "version": "24'8", | |
5369 | "prior_version": "21'3", | |
5370 | "last_reqid": "client.4195.0:1", | |
5371 | "user_version": 8, | |
5372 | "size": 21, | |
5373 | "mtime": "2018-04-05 14:35:43.286117", | |
5374 | "local_mtime": "2018-04-05 14:35:43.288990", | |
5375 | "lost": 0, | |
5376 | "flags": [ | |
5377 | "dirty", | |
5378 | "omap", | |
5379 | "data_digest" | |
5380 | ], | |
5381 | "truncate_seq": 0, | |
5382 | "truncate_size": 0, | |
5383 | "data_digest": "0x53acb008", | |
5384 | "omap_digest": "0xffffffff", | |
5385 | "expected_object_size": 0, | |
5386 | "expected_write_size": 0, | |
5387 | "alloc_hint_flags": 0, | |
5388 | "manifest": { | |
5389 | "type": 0 | |
5390 | }, | |
5391 | "watchers": {} | |
5392 | }, | |
3a9019d9 FG |
5393 | "shards": [ |
5394 | { | |
5395 | "osd": 0, | |
5396 | "primary": false, | |
5397 | "errors": [], | |
5398 | "size": 21, | |
94b18763 FG |
5399 | "snapset": { |
5400 | "clones": [ | |
5401 | { | |
5402 | "overlap": "[]", | |
5403 | "size": 7, | |
5404 | "snap": 1, | |
5405 | "snaps": [ | |
5406 | 1 | |
5407 | ] | |
5408 | } | |
5409 | ], | |
5410 | "head_exists": 1, | |
5411 | "snap_context": { | |
5412 | "seq": 1, | |
5413 | "snaps": [ | |
5414 | 1 | |
5415 | ] | |
5416 | } | |
5417 | } | |
3a9019d9 FG |
5418 | }, |
5419 | { | |
5420 | "osd": 1, | |
5421 | "primary": true, | |
5422 | "errors": [], | |
5423 | "size": 21, | |
94b18763 FG |
5424 | "snapset": { |
5425 | "clones": [], | |
5426 | "head_exists": 0, | |
5427 | "snap_context": { | |
5428 | "seq": 0, | |
5429 | "snaps": [] | |
5430 | } | |
5431 | } | |
3a9019d9 FG |
5432 | } |
5433 | ] | |
5434 | }, | |
5435 | { | |
5436 | "object": { | |
5437 | "name": "ROBJ2", | |
5438 | "nspace": "", | |
5439 | "locator": "", | |
5440 | "snap": "head", | |
5441 | "version": 10 | |
5442 | }, | |
5443 | "errors": [ | |
5444 | "snapset_inconsistency" | |
5445 | ], | |
5446 | "union_shard_errors": [], | |
94b18763 FG |
5447 | "selected_object_info": { |
5448 | "oid": { | |
5449 | "oid": "ROBJ2", | |
5450 | "key": "", | |
5451 | "snapid": -2, | |
5452 | "hash": 2026323607, | |
5453 | "max": 0, | |
5454 | "pool": 3, | |
5455 | "namespace": "" | |
5456 | }, | |
5457 | "version": "28'10", | |
5458 | "prior_version": "23'6", | |
5459 | "last_reqid": "client.4223.0:1", | |
5460 | "user_version": 10, | |
5461 | "size": 21, | |
5462 | "mtime": "2018-04-05 14:35:48.326856", | |
5463 | "local_mtime": "2018-04-05 14:35:48.328097", | |
5464 | "lost": 0, | |
5465 | "flags": [ | |
5466 | "dirty", | |
5467 | "omap", | |
5468 | "data_digest" | |
5469 | ], | |
5470 | "truncate_seq": 0, | |
5471 | "truncate_size": 0, | |
5472 | "data_digest": "0x53acb008", | |
5473 | "omap_digest": "0xffffffff", | |
5474 | "expected_object_size": 0, | |
5475 | "expected_write_size": 0, | |
5476 | "alloc_hint_flags": 0, | |
5477 | "manifest": { | |
5478 | "type": 0 | |
5479 | }, | |
5480 | "watchers": {} | |
5481 | }, | |
3a9019d9 FG |
5482 | "shards": [ |
5483 | { | |
5484 | "osd": 0, | |
5485 | "primary": false, | |
5486 | "errors": [], | |
5487 | "size": 21, | |
94b18763 FG |
5488 | "snapset": { |
5489 | "clones": [], | |
5490 | "head_exists": 0, | |
5491 | "snap_context": { | |
5492 | "seq": 0, | |
5493 | "snaps": [] | |
5494 | } | |
5495 | } | |
3a9019d9 FG |
5496 | }, |
5497 | { | |
5498 | "osd": 1, | |
5499 | "primary": true, | |
5500 | "errors": [], | |
5501 | "size": 21, | |
94b18763 FG |
5502 | "snapset": { |
5503 | "clones": [ | |
5504 | { | |
5505 | "overlap": "[]", | |
5506 | "size": 7, | |
5507 | "snap": 1, | |
5508 | "snaps": [ | |
5509 | 1 | |
5510 | ] | |
5511 | } | |
5512 | ], | |
5513 | "head_exists": 1, | |
5514 | "snap_context": { | |
5515 | "seq": 1, | |
5516 | "snaps": [ | |
5517 | 1 | |
5518 | ] | |
5519 | } | |
5520 | } | |
3a9019d9 FG |
5521 | } |
5522 | ] | |
5523 | } | |
5524 | ] | |
5525 | } | |
5526 | EOF | |
5527 | ||
94b18763 | 5528 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
1adf2230 | 5529 | multidiff $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
3a9019d9 FG |
5530 | if test $getjson = "yes" |
5531 | then | |
5532 | jq '.' $dir/json > save6.json | |
5533 | fi | |
5534 | ||
94b18763 | 5535 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
3a9019d9 FG |
5536 | then |
5537 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
5538 | fi | |
5539 | ||
91327a77 AA |
5540 | ERRORS=0 |
5541 | declare -a err_strings | |
5542 | err_strings[0]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ1:head : snapset inconsistent" | |
5543 | err_strings[1]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 soid [0-9]*:.*:::ROBJ2:head : snapset inconsistent" | |
5544 | err_strings[2]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:1 : is an unexpected clone" | |
5545 | err_strings[3]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub : stat mismatch, got 3/4 objects, 1/2 clones, 3/4 dirty, 3/4 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 49/56 bytes, 0/0 hit_set_archive bytes." | |
5546 | err_strings[4]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 0 missing, 2 inconsistent objects" | |
5547 | err_strings[5]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 5 errors" | |
5548 | err_strings[6]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 [0-9]*:.*:::ROBJ1:head : snapset.head_exists=false, but head exists" | |
5549 | for err_string in "${err_strings[@]}" | |
5550 | do | |
5551 | if ! grep -q "$err_string" $dir/osd.${primary}.log | |
5552 | then | |
5553 | echo "Missing log message '$err_string'" | |
5554 | ERRORS=$(expr $ERRORS + 1) | |
5555 | fi | |
5556 | done | |
5557 | ||
5558 | if [ $ERRORS != "0" ]; | |
5559 | then | |
5560 | echo "TEST FAILED WITH $ERRORS ERRORS" | |
5561 | return 1 | |
5562 | fi | |
5563 | ||
3a9019d9 FG |
5564 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it |
5565 | teardown $dir || return 1 | |
5566 | } | |
7c673cae FG |
5567 | |
5568 | main osd-scrub-repair "$@" | |
5569 | ||
5570 | # Local Variables: | |
28e407b8 AA |
5571 | # compile-command: "cd build ; make -j4 && \ |
5572 | # ../qa/run-standalone.sh osd-scrub-repair.sh" | |
7c673cae | 5573 | # End: |