]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | #!/bin/bash -x |
2 | # | |
3 | # Copyright (C) 2014 Red Hat <contact@redhat.com> | |
4 | # | |
5 | # Author: Loic Dachary <loic@dachary.org> | |
6 | # | |
7 | # This program is free software; you can redistribute it and/or modify | |
8 | # it under the terms of the GNU Library Public License as published by | |
9 | # the Free Software Foundation; either version 2, or (at your option) | |
10 | # any later version. | |
11 | # | |
12 | # This program is distributed in the hope that it will be useful, | |
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | # GNU Library Public License for more details. | |
16 | # | |
c07f9fc5 | 17 | source $CEPH_ROOT/qa/standalone/ceph-helpers.sh |
7c673cae | 18 | |
224ce89b | 19 | if [ `uname` = FreeBSD ]; then |
c07f9fc5 | 20 | # erasure coding overwrites are only tested on Bluestore |
224ce89b WB |
21 | # erasure coding on filestore is unsafe |
22 | # http://docs.ceph.com/docs/master/rados/operations/erasure-code/#erasure-coding-with-overwrites | |
23 | use_ec_overwrite=false | |
24 | else | |
25 | use_ec_overwrite=true | |
26 | fi | |
27 | ||
7c673cae FG |
28 | # Test development and debugging |
29 | # Set to "yes" in order to ignore diff errors and save results to update test | |
30 | getjson="no" | |
31 | ||
94b18763 FG |
32 | # Filter out mtime and local_mtime dates, version, prior_version and last_reqid (client) from any object_info. |
33 | jqfilter='def walk(f): | |
34 | . as $in | |
35 | | if type == "object" then | |
36 | reduce keys[] as $key | |
37 | ( {}; . + { ($key): ($in[$key] | walk(f)) } ) | f | |
38 | elif type == "array" then map( walk(f) ) | f | |
39 | else f | |
40 | end; | |
41 | walk(if type == "object" then del(.mtime) else . end) | |
42 | | walk(if type == "object" then del(.local_mtime) else . end) | |
43 | | walk(if type == "object" then del(.last_reqid) else . end) | |
44 | | walk(if type == "object" then del(.version) else . end) | |
45 | | walk(if type == "object" then del(.prior_version) else . end) | |
46 | | walk(if type == "object" then del(.redirect_target) else . end) | |
47 | | walk(if type == "object" then del(.legacy_snaps) else . end)' | |
7c673cae | 48 | |
94b18763 | 49 | sortkeys='import json; import sys ; JSON=sys.stdin.read() ; ud = json.loads(JSON) ; print json.dumps(ud, sort_keys=True, indent=2)' |
7c673cae FG |
50 | |
51 | function run() { | |
52 | local dir=$1 | |
53 | shift | |
54 | ||
55 | export CEPH_MON="127.0.0.1:7107" # git grep '\<7107\>' : there must be only one | |
56 | export CEPH_ARGS | |
57 | CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none " | |
58 | CEPH_ARGS+="--mon-host=$CEPH_MON " | |
28e407b8 | 59 | CEPH_ARGS+="--osd-skip-data-digest=false " |
7c673cae FG |
60 | |
61 | local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')} | |
62 | for func in $funcs ; do | |
63 | $func $dir || return 1 | |
64 | done | |
65 | } | |
66 | ||
67 | function add_something() { | |
68 | local dir=$1 | |
69 | local poolname=$2 | |
70 | local obj=${3:-SOMETHING} | |
71 | local scrub=${4:-noscrub} | |
72 | ||
73 | if [ "$scrub" = "noscrub" ]; | |
74 | then | |
75 | ceph osd set noscrub || return 1 | |
76 | ceph osd set nodeep-scrub || return 1 | |
77 | else | |
78 | ceph osd unset noscrub || return 1 | |
79 | ceph osd unset nodeep-scrub || return 1 | |
80 | fi | |
81 | ||
82 | local payload=ABCDEF | |
83 | echo $payload > $dir/ORIGINAL | |
84 | rados --pool $poolname put $obj $dir/ORIGINAL || return 1 | |
85 | } | |
86 | ||
87 | # | |
88 | # Corrupt one copy of a replicated pool | |
89 | # | |
90 | function TEST_corrupt_and_repair_replicated() { | |
91 | local dir=$1 | |
92 | local poolname=rbd | |
93 | ||
94 | setup $dir || return 1 | |
95 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
96 | run_mgr $dir x || return 1 | |
97 | run_osd $dir 0 || return 1 | |
98 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 99 | create_rbd_pool || return 1 |
7c673cae FG |
100 | wait_for_clean || return 1 |
101 | ||
102 | add_something $dir $poolname || return 1 | |
103 | corrupt_and_repair_one $dir $poolname $(get_not_primary $poolname SOMETHING) || return 1 | |
104 | # Reproduces http://tracker.ceph.com/issues/8914 | |
105 | corrupt_and_repair_one $dir $poolname $(get_primary $poolname SOMETHING) || return 1 | |
106 | ||
107 | teardown $dir || return 1 | |
108 | } | |
109 | ||
110 | function corrupt_and_repair_two() { | |
111 | local dir=$1 | |
112 | local poolname=$2 | |
113 | local first=$3 | |
114 | local second=$4 | |
115 | ||
116 | # | |
117 | # 1) remove the corresponding file from the OSDs | |
118 | # | |
119 | pids="" | |
120 | run_in_background pids objectstore_tool $dir $first SOMETHING remove | |
121 | run_in_background pids objectstore_tool $dir $second SOMETHING remove | |
122 | wait_background pids | |
123 | return_code=$? | |
124 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
125 | ||
126 | # | |
127 | # 2) repair the PG | |
128 | # | |
129 | local pg=$(get_pg $poolname SOMETHING) | |
130 | repair $pg | |
131 | # | |
132 | # 3) The files must be back | |
133 | # | |
134 | pids="" | |
135 | run_in_background pids objectstore_tool $dir $first SOMETHING list-attrs | |
136 | run_in_background pids objectstore_tool $dir $second SOMETHING list-attrs | |
137 | wait_background pids | |
138 | return_code=$? | |
139 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
140 | ||
141 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
142 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
143 | } | |
144 | ||
145 | # | |
146 | # 1) add an object | |
147 | # 2) remove the corresponding file from a designated OSD | |
148 | # 3) repair the PG | |
149 | # 4) check that the file has been restored in the designated OSD | |
150 | # | |
151 | function corrupt_and_repair_one() { | |
152 | local dir=$1 | |
153 | local poolname=$2 | |
154 | local osd=$3 | |
155 | ||
156 | # | |
157 | # 1) remove the corresponding file from the OSD | |
158 | # | |
159 | objectstore_tool $dir $osd SOMETHING remove || return 1 | |
160 | # | |
161 | # 2) repair the PG | |
162 | # | |
163 | local pg=$(get_pg $poolname SOMETHING) | |
164 | repair $pg | |
165 | # | |
166 | # 3) The file must be back | |
167 | # | |
168 | objectstore_tool $dir $osd SOMETHING list-attrs || return 1 | |
169 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
170 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
171 | } | |
172 | ||
173 | function corrupt_and_repair_erasure_coded() { | |
174 | local dir=$1 | |
175 | local poolname=$2 | |
176 | ||
177 | add_something $dir $poolname || return 1 | |
178 | ||
179 | local primary=$(get_primary $poolname SOMETHING) | |
180 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
181 | local not_primary_first=${osds[0]} | |
182 | local not_primary_second=${osds[1]} | |
183 | ||
184 | # Reproduces http://tracker.ceph.com/issues/10017 | |
185 | corrupt_and_repair_one $dir $poolname $primary || return 1 | |
186 | # Reproduces http://tracker.ceph.com/issues/10409 | |
187 | corrupt_and_repair_one $dir $poolname $not_primary_first || return 1 | |
188 | corrupt_and_repair_two $dir $poolname $not_primary_first $not_primary_second || return 1 | |
189 | corrupt_and_repair_two $dir $poolname $primary $not_primary_first || return 1 | |
190 | ||
191 | } | |
192 | ||
193 | function create_ec_pool() { | |
194 | local pool_name=$1 | |
195 | local allow_overwrites=$2 | |
196 | ||
224ce89b | 197 | ceph osd erasure-code-profile set myprofile crush-failure-domain=osd $3 $4 $5 $6 $7 || return 1 |
7c673cae | 198 | |
b5b8bbf5 | 199 | create_pool "$poolname" 1 1 erasure myprofile || return 1 |
7c673cae FG |
200 | |
201 | if [ "$allow_overwrites" = "true" ]; then | |
202 | ceph osd pool set "$poolname" allow_ec_overwrites true || return 1 | |
203 | fi | |
204 | ||
205 | wait_for_clean || return 1 | |
206 | return 0 | |
207 | } | |
208 | ||
209 | function auto_repair_erasure_coded() { | |
210 | local dir=$1 | |
211 | local allow_overwrites=$2 | |
212 | local poolname=ecpool | |
213 | ||
214 | # Launch a cluster with 5 seconds scrub interval | |
215 | setup $dir || return 1 | |
216 | run_mon $dir a || return 1 | |
217 | run_mgr $dir x || return 1 | |
218 | local ceph_osd_args="--osd-scrub-auto-repair=true \ | |
219 | --osd-deep-scrub-interval=5 \ | |
220 | --osd-scrub-max-interval=5 \ | |
221 | --osd-scrub-min-interval=5 \ | |
222 | --osd-scrub-interval-randomize-ratio=0" | |
223 | for id in $(seq 0 2) ; do | |
224 | if [ "$allow_overwrites" = "true" ]; then | |
225 | run_osd_bluestore $dir $id $ceph_osd_args || return 1 | |
226 | else | |
227 | run_osd $dir $id $ceph_osd_args || return 1 | |
228 | fi | |
229 | done | |
c07f9fc5 | 230 | create_rbd_pool || return 1 |
7c673cae FG |
231 | wait_for_clean || return 1 |
232 | ||
233 | # Create an EC pool | |
234 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
235 | ||
236 | # Put an object | |
237 | local payload=ABCDEF | |
238 | echo $payload > $dir/ORIGINAL | |
239 | rados --pool $poolname put SOMETHING $dir/ORIGINAL || return 1 | |
240 | ||
241 | # Remove the object from one shard physically | |
242 | # Restarted osd get $ceph_osd_args passed | |
243 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING remove || return 1 | |
244 | # Wait for auto repair | |
245 | local pgid=$(get_pg $poolname SOMETHING) | |
246 | wait_for_scrub $pgid "$(get_last_scrub_stamp $pgid)" | |
247 | wait_for_clean || return 1 | |
248 | # Verify - the file should be back | |
249 | # Restarted osd get $ceph_osd_args passed | |
250 | objectstore_tool $dir $(get_not_primary $poolname SOMETHING) SOMETHING list-attrs || return 1 | |
251 | rados --pool $poolname get SOMETHING $dir/COPY || return 1 | |
252 | diff $dir/ORIGINAL $dir/COPY || return 1 | |
253 | ||
254 | # Tear down | |
255 | teardown $dir || return 1 | |
256 | } | |
257 | ||
258 | function TEST_auto_repair_erasure_coded_appends() { | |
259 | auto_repair_erasure_coded $1 false | |
260 | } | |
261 | ||
262 | function TEST_auto_repair_erasure_coded_overwrites() { | |
224ce89b WB |
263 | if [ "$use_ec_overwrite" = "true" ]; then |
264 | auto_repair_erasure_coded $1 true | |
265 | fi | |
7c673cae FG |
266 | } |
267 | ||
268 | function corrupt_and_repair_jerasure() { | |
269 | local dir=$1 | |
270 | local allow_overwrites=$2 | |
271 | local poolname=ecpool | |
272 | ||
273 | setup $dir || return 1 | |
274 | run_mon $dir a || return 1 | |
275 | run_mgr $dir x || return 1 | |
276 | for id in $(seq 0 3) ; do | |
277 | if [ "$allow_overwrites" = "true" ]; then | |
278 | run_osd_bluestore $dir $id || return 1 | |
279 | else | |
280 | run_osd $dir $id || return 1 | |
281 | fi | |
282 | done | |
c07f9fc5 | 283 | create_rbd_pool || return 1 |
7c673cae FG |
284 | wait_for_clean || return 1 |
285 | ||
286 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
287 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
288 | ||
289 | teardown $dir || return 1 | |
290 | } | |
291 | ||
292 | function TEST_corrupt_and_repair_jerasure_appends() { | |
293 | corrupt_and_repair_jerasure $1 | |
294 | } | |
295 | ||
296 | function TEST_corrupt_and_repair_jerasure_overwrites() { | |
224ce89b WB |
297 | if [ "$use_ec_overwrite" = "true" ]; then |
298 | corrupt_and_repair_jerasure $1 true | |
299 | fi | |
7c673cae FG |
300 | } |
301 | ||
302 | function corrupt_and_repair_lrc() { | |
303 | local dir=$1 | |
304 | local allow_overwrites=$2 | |
305 | local poolname=ecpool | |
306 | ||
307 | setup $dir || return 1 | |
308 | run_mon $dir a || return 1 | |
309 | run_mgr $dir x || return 1 | |
310 | for id in $(seq 0 9) ; do | |
311 | if [ "$allow_overwrites" = "true" ]; then | |
312 | run_osd_bluestore $dir $id || return 1 | |
313 | else | |
314 | run_osd $dir $id || return 1 | |
315 | fi | |
316 | done | |
c07f9fc5 | 317 | create_rbd_pool || return 1 |
7c673cae FG |
318 | wait_for_clean || return 1 |
319 | ||
320 | create_ec_pool $poolname $allow_overwrites k=4 m=2 l=3 plugin=lrc || return 1 | |
321 | corrupt_and_repair_erasure_coded $dir $poolname || return 1 | |
322 | ||
323 | teardown $dir || return 1 | |
324 | } | |
325 | ||
326 | function TEST_corrupt_and_repair_lrc_appends() { | |
327 | corrupt_and_repair_jerasure $1 | |
328 | } | |
329 | ||
330 | function TEST_corrupt_and_repair_lrc_overwrites() { | |
224ce89b WB |
331 | if [ "$use_ec_overwrite" = "true" ]; then |
332 | corrupt_and_repair_jerasure $1 true | |
333 | fi | |
7c673cae FG |
334 | } |
335 | ||
336 | function unfound_erasure_coded() { | |
337 | local dir=$1 | |
338 | local allow_overwrites=$2 | |
339 | local poolname=ecpool | |
340 | local payload=ABCDEF | |
341 | ||
342 | setup $dir || return 1 | |
343 | run_mon $dir a || return 1 | |
344 | run_mgr $dir x || return 1 | |
345 | for id in $(seq 0 3) ; do | |
346 | if [ "$allow_overwrites" = "true" ]; then | |
347 | run_osd_bluestore $dir $id || return 1 | |
348 | else | |
349 | run_osd $dir $id || return 1 | |
350 | fi | |
351 | done | |
c07f9fc5 | 352 | create_rbd_pool || return 1 |
7c673cae FG |
353 | wait_for_clean || return 1 |
354 | ||
355 | create_ec_pool $poolname $allow_overwrites k=2 m=2 || return 1 | |
356 | ||
357 | add_something $dir $poolname || return 1 | |
358 | ||
359 | local primary=$(get_primary $poolname SOMETHING) | |
360 | local -a osds=($(get_osds $poolname SOMETHING | sed -e "s/$primary//")) | |
361 | local not_primary_first=${osds[0]} | |
362 | local not_primary_second=${osds[1]} | |
363 | local not_primary_third=${osds[2]} | |
364 | ||
365 | # | |
366 | # 1) remove the corresponding file from the OSDs | |
367 | # | |
368 | pids="" | |
369 | run_in_background pids objectstore_tool $dir $not_primary_first SOMETHING remove | |
370 | run_in_background pids objectstore_tool $dir $not_primary_second SOMETHING remove | |
371 | run_in_background pids objectstore_tool $dir $not_primary_third SOMETHING remove | |
372 | wait_background pids | |
373 | return_code=$? | |
374 | if [ $return_code -ne 0 ]; then return $return_code; fi | |
375 | ||
376 | # | |
377 | # 2) repair the PG | |
378 | # | |
379 | local pg=$(get_pg $poolname SOMETHING) | |
380 | repair $pg | |
381 | # | |
382 | # 3) check pg state | |
383 | # | |
384 | # it may take a bit to appear due to mon/mgr asynchrony | |
385 | for f in `seq 1 60`; do | |
b5b8bbf5 | 386 | ceph -s | grep "1/1 objects unfound" && break |
7c673cae FG |
387 | sleep 1 |
388 | done | |
389 | ceph -s|grep "4 osds: 4 up, 4 in" || return 1 | |
b5b8bbf5 | 390 | ceph -s|grep "1/1 objects unfound" || return 1 |
7c673cae FG |
391 | |
392 | teardown $dir || return 1 | |
393 | } | |
394 | ||
395 | function TEST_unfound_erasure_coded_appends() { | |
396 | unfound_erasure_coded $1 | |
397 | } | |
398 | ||
399 | function TEST_unfound_erasure_coded_overwrites() { | |
224ce89b WB |
400 | if [ "$use_ec_overwrite" = "true" ]; then |
401 | unfound_erasure_coded $1 true | |
402 | fi | |
7c673cae FG |
403 | } |
404 | ||
405 | # | |
406 | # list_missing for EC pool | |
407 | # | |
408 | function list_missing_erasure_coded() { | |
409 | local dir=$1 | |
410 | local allow_overwrites=$2 | |
411 | local poolname=ecpool | |
412 | ||
413 | setup $dir || return 1 | |
414 | run_mon $dir a || return 1 | |
415 | run_mgr $dir x || return 1 | |
416 | for id in $(seq 0 2) ; do | |
417 | if [ "$allow_overwrites" = "true" ]; then | |
418 | run_osd_bluestore $dir $id || return 1 | |
419 | else | |
420 | run_osd $dir $id || return 1 | |
421 | fi | |
422 | done | |
c07f9fc5 | 423 | create_rbd_pool || return 1 |
7c673cae FG |
424 | wait_for_clean || return 1 |
425 | ||
426 | create_ec_pool $poolname $allow_overwrites k=2 m=1 || return 1 | |
427 | ||
428 | # Put an object and remove the two shards (including primary) | |
429 | add_something $dir $poolname MOBJ0 || return 1 | |
430 | local -a osds0=($(get_osds $poolname MOBJ0)) | |
431 | ||
432 | # Put another object and remove two shards (excluding primary) | |
433 | add_something $dir $poolname MOBJ1 || return 1 | |
434 | local -a osds1=($(get_osds $poolname MOBJ1)) | |
435 | ||
436 | # Stop all osd daemons | |
437 | for id in $(seq 0 2) ; do | |
438 | kill_daemons $dir TERM osd.$id >&2 < /dev/null || return 1 | |
439 | done | |
440 | ||
441 | id=${osds0[0]} | |
31f18b77 | 442 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
443 | MOBJ0 remove || return 1 |
444 | id=${osds0[1]} | |
31f18b77 | 445 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
446 | MOBJ0 remove || return 1 |
447 | ||
448 | id=${osds1[1]} | |
31f18b77 | 449 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
450 | MOBJ1 remove || return 1 |
451 | id=${osds1[2]} | |
31f18b77 | 452 | ceph-objectstore-tool --data-path $dir/$id \ |
7c673cae FG |
453 | MOBJ1 remove || return 1 |
454 | ||
455 | for id in $(seq 0 2) ; do | |
456 | activate_osd $dir $id >&2 || return 1 | |
457 | done | |
c07f9fc5 | 458 | create_rbd_pool || return 1 |
7c673cae FG |
459 | wait_for_clean || return 1 |
460 | ||
461 | # Get get - both objects should in the same PG | |
462 | local pg=$(get_pg $poolname MOBJ0) | |
463 | ||
464 | # Repair the PG, which triggers the recovering, | |
465 | # and should mark the object as unfound | |
466 | repair $pg | |
c07f9fc5 | 467 | |
7c673cae FG |
468 | for i in $(seq 0 120) ; do |
469 | [ $i -lt 60 ] || return 1 | |
470 | matches=$(ceph pg $pg list_missing | egrep "MOBJ0|MOBJ1" | wc -l) | |
471 | [ $matches -eq 2 ] && break | |
472 | done | |
473 | ||
474 | teardown $dir || return 1 | |
475 | } | |
476 | ||
477 | function TEST_list_missing_erasure_coded_appends() { | |
478 | list_missing_erasure_coded $1 false | |
479 | } | |
480 | ||
481 | function TEST_list_missing_erasure_coded_overwrites() { | |
224ce89b WB |
482 | if [ "$use_ec_overwrite" = "true" ]; then |
483 | list_missing_erasure_coded $1 true | |
484 | fi | |
7c673cae FG |
485 | } |
486 | ||
487 | # | |
488 | # Corrupt one copy of a replicated pool | |
489 | # | |
490 | function TEST_corrupt_scrub_replicated() { | |
491 | local dir=$1 | |
492 | local poolname=csr_pool | |
b5b8bbf5 | 493 | local total_objs=16 |
7c673cae FG |
494 | |
495 | setup $dir || return 1 | |
496 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
497 | run_mgr $dir x || return 1 | |
498 | run_osd $dir 0 || return 1 | |
499 | run_osd $dir 1 || return 1 | |
c07f9fc5 | 500 | create_rbd_pool || return 1 |
7c673cae FG |
501 | wait_for_clean || return 1 |
502 | ||
b5b8bbf5 FG |
503 | create_pool foo 1 || return 1 |
504 | create_pool $poolname 1 1 || return 1 | |
7c673cae FG |
505 | wait_for_clean || return 1 |
506 | ||
507 | for i in $(seq 1 $total_objs) ; do | |
508 | objname=ROBJ${i} | |
509 | add_something $dir $poolname $objname || return 1 | |
510 | ||
511 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
512 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
31f18b77 FG |
513 | done |
514 | ||
515 | local pg=$(get_pg $poolname ROBJ0) | |
516 | ||
517 | # Compute an old omap digest and save oi | |
c07f9fc5 | 518 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.0) \ |
31f18b77 | 519 | config set osd_deep_scrub_update_digest_min_age 0 |
c07f9fc5 | 520 | CEPH_ARGS='' ceph daemon $(get_asok_path osd.1) \ |
31f18b77 FG |
521 | config set osd_deep_scrub_update_digest_min_age 0 |
522 | pg_deep_scrub $pg | |
523 | ||
524 | for i in $(seq 1 $total_objs) ; do | |
525 | objname=ROBJ${i} | |
7c673cae FG |
526 | |
527 | # Alternate corruption between osd.0 and osd.1 | |
528 | local osd=$(expr $i % 2) | |
529 | ||
530 | case $i in | |
531 | 1) | |
532 | # Size (deep scrub data_digest too) | |
533 | local payload=UVWXYZZZ | |
534 | echo $payload > $dir/CORRUPT | |
535 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
536 | ;; | |
537 | ||
538 | 2) | |
539 | # digest (deep scrub only) | |
540 | local payload=UVWXYZ | |
541 | echo $payload > $dir/CORRUPT | |
542 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
543 | ;; | |
544 | ||
545 | 3) | |
546 | # missing | |
547 | objectstore_tool $dir $osd $objname remove || return 1 | |
548 | ;; | |
549 | ||
550 | 4) | |
551 | # Modify omap value (deep scrub only) | |
552 | objectstore_tool $dir $osd $objname set-omap key-$objname $dir/CORRUPT || return 1 | |
553 | ;; | |
554 | ||
555 | 5) | |
556 | # Delete omap key (deep scrub only) | |
557 | objectstore_tool $dir $osd $objname rm-omap key-$objname || return 1 | |
558 | ;; | |
559 | ||
560 | 6) | |
561 | # Add extra omap key (deep scrub only) | |
562 | echo extra > $dir/extra-val | |
563 | objectstore_tool $dir $osd $objname set-omap key2-$objname $dir/extra-val || return 1 | |
564 | rm $dir/extra-val | |
565 | ;; | |
566 | ||
567 | 7) | |
568 | # Modify omap header (deep scrub only) | |
569 | echo -n newheader > $dir/hdr | |
570 | objectstore_tool $dir $osd $objname set-omaphdr $dir/hdr || return 1 | |
571 | rm $dir/hdr | |
572 | ;; | |
573 | ||
574 | 8) | |
575 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
576 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
577 | ||
578 | # Break xattrs | |
579 | echo -n bad-val > $dir/bad-val | |
580 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
581 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
582 | echo -n val3-$objname > $dir/newval | |
583 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
584 | rm $dir/bad-val $dir/newval | |
585 | ;; | |
586 | ||
587 | 9) | |
588 | objectstore_tool $dir $osd $objname get-attr _ > $dir/robj9-oi | |
589 | echo -n D > $dir/change | |
590 | rados --pool $poolname put $objname $dir/change | |
591 | objectstore_tool $dir $osd $objname set-attr _ $dir/robj9-oi | |
592 | rm $dir/oi $dir/change | |
593 | ;; | |
594 | ||
595 | # ROBJ10 must be handled after digests are re-computed by a deep scrub below | |
596 | # ROBJ11 must be handled with config change before deep scrub | |
597 | # ROBJ12 must be handled with config change before scrubs | |
598 | # ROBJ13 must be handled before scrubs | |
599 | ||
600 | 14) | |
601 | echo -n bad-val > $dir/bad-val | |
602 | objectstore_tool $dir 0 $objname set-attr _ $dir/bad-val || return 1 | |
603 | objectstore_tool $dir 1 $objname rm-attr _ || return 1 | |
604 | rm $dir/bad-val | |
605 | ;; | |
606 | ||
607 | 15) | |
608 | objectstore_tool $dir $osd $objname rm-attr _ || return 1 | |
b5b8bbf5 FG |
609 | ;; |
610 | ||
611 | 16) | |
612 | objectstore_tool $dir 0 $objname rm-attr snapset || return 1 | |
613 | echo -n bad-val > $dir/bad-val | |
614 | objectstore_tool $dir 1 $objname set-attr snapset $dir/bad-val || return 1 | |
7c673cae FG |
615 | |
616 | esac | |
617 | done | |
618 | ||
619 | local pg=$(get_pg $poolname ROBJ0) | |
620 | ||
b5b8bbf5 FG |
621 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
622 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
623 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
624 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
625 | |
626 | pg_scrub $pg | |
627 | ||
628 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
629 | # Check pg count | |
630 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
631 | # Check pgid | |
632 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
633 | ||
634 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
635 | # Get epoch for repair-get requests | |
636 | epoch=$(jq .epoch $dir/json) | |
637 | ||
94b18763 | 638 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
639 | { |
640 | "inconsistents": [ | |
641 | { | |
642 | "shards": [ | |
643 | { | |
644 | "size": 7, | |
645 | "errors": [], | |
b5b8bbf5 FG |
646 | "osd": 0, |
647 | "primary": false | |
7c673cae FG |
648 | }, |
649 | { | |
94b18763 FG |
650 | "object_info": { |
651 | "oid": { | |
652 | "oid": "ROBJ1", | |
653 | "key": "", | |
654 | "snapid": -2, | |
655 | "hash": 1454963827, | |
656 | "max": 0, | |
657 | "pool": 3, | |
658 | "namespace": "" | |
659 | }, | |
660 | "version": "51'58", | |
661 | "prior_version": "21'3", | |
662 | "last_reqid": "osd.1.0:57", | |
663 | "user_version": 3, | |
664 | "size": 7, | |
665 | "mtime": "", | |
666 | "local_mtime": "", | |
667 | "lost": 0, | |
668 | "flags": [ | |
669 | "dirty", | |
670 | "omap", | |
671 | "data_digest", | |
672 | "omap_digest" | |
673 | ], | |
674 | "truncate_seq": 0, | |
675 | "truncate_size": 0, | |
676 | "data_digest": "0x2ddbf8f5", | |
677 | "omap_digest": "0xf5fba2c6", | |
678 | "expected_object_size": 0, | |
679 | "expected_write_size": 0, | |
680 | "alloc_hint_flags": 0, | |
681 | "manifest": { | |
682 | "type": 0 | |
683 | }, | |
684 | "watchers": {} | |
685 | }, | |
7c673cae FG |
686 | "size": 9, |
687 | "errors": [ | |
94b18763 FG |
688 | "size_mismatch_info", |
689 | "obj_size_info_mismatch" | |
7c673cae | 690 | ], |
b5b8bbf5 FG |
691 | "osd": 1, |
692 | "primary": true | |
7c673cae FG |
693 | } |
694 | ], | |
94b18763 FG |
695 | "selected_object_info": { |
696 | "oid": { | |
697 | "oid": "ROBJ1", | |
698 | "key": "", | |
699 | "snapid": -2, | |
700 | "hash": 1454963827, | |
701 | "max": 0, | |
702 | "pool": 3, | |
703 | "namespace": "" | |
704 | }, | |
705 | "version": "51'58", | |
706 | "prior_version": "21'3", | |
707 | "last_reqid": "osd.1.0:57", | |
708 | "user_version": 3, | |
709 | "size": 7, | |
710 | "mtime": "2018-04-05 14:33:19.804040", | |
711 | "local_mtime": "2018-04-05 14:33:19.804839", | |
712 | "lost": 0, | |
713 | "flags": [ | |
714 | "dirty", | |
715 | "omap", | |
716 | "data_digest", | |
717 | "omap_digest" | |
718 | ], | |
719 | "truncate_seq": 0, | |
720 | "truncate_size": 0, | |
721 | "data_digest": "0x2ddbf8f5", | |
722 | "omap_digest": "0xf5fba2c6", | |
723 | "expected_object_size": 0, | |
724 | "expected_write_size": 0, | |
725 | "alloc_hint_flags": 0, | |
726 | "manifest": { | |
727 | "type": 0 | |
728 | }, | |
729 | "watchers": {} | |
730 | }, | |
7c673cae | 731 | "union_shard_errors": [ |
94b18763 FG |
732 | "size_mismatch_info", |
733 | "obj_size_info_mismatch" | |
7c673cae FG |
734 | ], |
735 | "errors": [ | |
736 | "size_mismatch" | |
737 | ], | |
738 | "object": { | |
739 | "version": 3, | |
740 | "snap": "head", | |
741 | "locator": "", | |
742 | "nspace": "", | |
743 | "name": "ROBJ1" | |
744 | } | |
745 | }, | |
746 | { | |
747 | "shards": [ | |
748 | { | |
749 | "errors": [ | |
750 | "stat_error" | |
751 | ], | |
b5b8bbf5 FG |
752 | "osd": 0, |
753 | "primary": false | |
7c673cae FG |
754 | }, |
755 | { | |
756 | "size": 7, | |
757 | "errors": [], | |
b5b8bbf5 FG |
758 | "osd": 1, |
759 | "primary": true | |
7c673cae FG |
760 | } |
761 | ], | |
94b18763 FG |
762 | "selected_object_info": { |
763 | "oid": { | |
764 | "oid": "ROBJ12", | |
765 | "key": "", | |
766 | "snapid": -2, | |
767 | "hash": 3920199997, | |
768 | "max": 0, | |
769 | "pool": 3, | |
770 | "namespace": "" | |
771 | }, | |
772 | "version": "51'56", | |
773 | "prior_version": "43'36", | |
774 | "last_reqid": "osd.1.0:55", | |
775 | "user_version": 36, | |
776 | "size": 7, | |
777 | "mtime": "", | |
778 | "local_mtime": "", | |
779 | "lost": 0, | |
780 | "flags": [ | |
781 | "dirty", | |
782 | "omap", | |
783 | "data_digest", | |
784 | "omap_digest" | |
785 | ], | |
786 | "truncate_seq": 0, | |
787 | "truncate_size": 0, | |
788 | "data_digest": "0x2ddbf8f5", | |
789 | "omap_digest": "0x067f306a", | |
790 | "expected_object_size": 0, | |
791 | "expected_write_size": 0, | |
792 | "alloc_hint_flags": 0, | |
793 | "manifest": { | |
794 | "type": 0 | |
795 | }, | |
796 | "watchers": {} | |
797 | }, | |
7c673cae FG |
798 | "union_shard_errors": [ |
799 | "stat_error" | |
800 | ], | |
801 | "errors": [], | |
802 | "object": { | |
31f18b77 | 803 | "version": 36, |
7c673cae FG |
804 | "snap": "head", |
805 | "locator": "", | |
806 | "nspace": "", | |
807 | "name": "ROBJ12" | |
808 | } | |
809 | }, | |
810 | { | |
811 | "shards": [ | |
812 | { | |
813 | "errors": [ | |
814 | "stat_error" | |
815 | ], | |
b5b8bbf5 FG |
816 | "osd": 0, |
817 | "primary": false | |
7c673cae FG |
818 | }, |
819 | { | |
820 | "size": 7, | |
821 | "errors": [], | |
b5b8bbf5 FG |
822 | "osd": 1, |
823 | "primary": true | |
7c673cae FG |
824 | } |
825 | ], | |
94b18763 FG |
826 | "selected_object_info": { |
827 | "oid": { | |
828 | "oid": "ROBJ13", | |
829 | "key": "", | |
830 | "snapid": -2, | |
831 | "hash": 2682806379, | |
832 | "max": 0, | |
833 | "pool": 3, | |
834 | "namespace": "" | |
835 | }, | |
836 | "version": "51'59", | |
837 | "prior_version": "45'39", | |
838 | "last_reqid": "osd.1.0:58", | |
839 | "user_version": 39, | |
840 | "size": 7, | |
841 | "mtime": "", | |
842 | "local_mtime": "", | |
843 | "lost": 0, | |
844 | "flags": [ | |
845 | "dirty", | |
846 | "omap", | |
847 | "data_digest", | |
848 | "omap_digest" | |
849 | ], | |
850 | "truncate_seq": 0, | |
851 | "truncate_size": 0, | |
852 | "data_digest": "0x2ddbf8f5", | |
853 | "omap_digest": "0x6441854d", | |
854 | "expected_object_size": 0, | |
855 | "expected_write_size": 0, | |
856 | "alloc_hint_flags": 0, | |
857 | "manifest": { | |
858 | "type": 0 | |
859 | }, | |
860 | "watchers": {} | |
861 | }, | |
7c673cae FG |
862 | "union_shard_errors": [ |
863 | "stat_error" | |
864 | ], | |
865 | "errors": [], | |
866 | "object": { | |
31f18b77 | 867 | "version": 39, |
7c673cae FG |
868 | "snap": "head", |
869 | "locator": "", | |
870 | "nspace": "", | |
871 | "name": "ROBJ13" | |
872 | } | |
873 | }, | |
874 | { | |
875 | "shards": [ | |
876 | { | |
94b18763 | 877 | "object_info": "bad-val", |
7c673cae FG |
878 | "size": 7, |
879 | "errors": [ | |
94b18763 | 880 | "info_corrupted" |
7c673cae | 881 | ], |
b5b8bbf5 FG |
882 | "osd": 0, |
883 | "primary": false | |
7c673cae FG |
884 | }, |
885 | { | |
886 | "size": 7, | |
887 | "errors": [ | |
94b18763 | 888 | "info_missing" |
7c673cae | 889 | ], |
b5b8bbf5 FG |
890 | "osd": 1, |
891 | "primary": true | |
7c673cae FG |
892 | } |
893 | ], | |
894 | "union_shard_errors": [ | |
94b18763 FG |
895 | "info_missing", |
896 | "info_corrupted" | |
7c673cae FG |
897 | ], |
898 | "errors": [], | |
899 | "object": { | |
900 | "version": 0, | |
901 | "snap": "head", | |
902 | "locator": "", | |
903 | "nspace": "", | |
904 | "name": "ROBJ14" | |
905 | } | |
906 | }, | |
907 | { | |
908 | "shards": [ | |
909 | { | |
94b18763 FG |
910 | "object_info": { |
911 | "oid": { | |
912 | "oid": "ROBJ15", | |
913 | "key": "", | |
914 | "snapid": -2, | |
915 | "hash": 504996876, | |
916 | "max": 0, | |
917 | "pool": 3, | |
918 | "namespace": "" | |
7c673cae | 919 | }, |
94b18763 FG |
920 | "version": "51'49", |
921 | "prior_version": "49'45", | |
922 | "last_reqid": "osd.1.0:48", | |
923 | "user_version": 45, | |
924 | "size": 7, | |
925 | "mtime": "2018-04-05 14:33:29.498969", | |
926 | "local_mtime": "2018-04-05 14:33:29.499890", | |
927 | "lost": 0, | |
928 | "flags": [ | |
929 | "dirty", | |
930 | "omap", | |
931 | "data_digest", | |
932 | "omap_digest" | |
933 | ], | |
934 | "truncate_seq": 0, | |
935 | "truncate_size": 0, | |
936 | "data_digest": "0x2ddbf8f5", | |
937 | "omap_digest": "0x2d2a4d6e", | |
938 | "expected_object_size": 0, | |
939 | "expected_write_size": 0, | |
940 | "alloc_hint_flags": 0, | |
941 | "manifest": { | |
942 | "type": 0 | |
943 | }, | |
944 | "watchers": {} | |
945 | }, | |
7c673cae FG |
946 | "size": 7, |
947 | "errors": [], | |
b5b8bbf5 FG |
948 | "osd": 0, |
949 | "primary": false | |
7c673cae FG |
950 | }, |
951 | { | |
7c673cae FG |
952 | "size": 7, |
953 | "errors": [ | |
94b18763 | 954 | "info_missing" |
7c673cae | 955 | ], |
b5b8bbf5 FG |
956 | "osd": 1, |
957 | "primary": true | |
7c673cae FG |
958 | } |
959 | ], | |
94b18763 FG |
960 | "selected_object_info": { |
961 | "oid": { | |
962 | "oid": "ROBJ15", | |
963 | "key": "", | |
964 | "snapid": -2, | |
965 | "hash": 504996876, | |
966 | "max": 0, | |
967 | "pool": 3, | |
968 | "namespace": "" | |
969 | }, | |
970 | "version": "51'49", | |
971 | "prior_version": "49'45", | |
972 | "last_reqid": "osd.1.0:48", | |
973 | "user_version": 45, | |
974 | "size": 7, | |
975 | "mtime": "", | |
976 | "local_mtime": "", | |
977 | "lost": 0, | |
978 | "flags": [ | |
979 | "dirty", | |
980 | "omap", | |
981 | "data_digest", | |
982 | "omap_digest" | |
983 | ], | |
984 | "truncate_seq": 0, | |
985 | "truncate_size": 0, | |
986 | "data_digest": "0x2ddbf8f5", | |
987 | "omap_digest": "0x2d2a4d6e", | |
988 | "expected_object_size": 0, | |
989 | "expected_write_size": 0, | |
990 | "alloc_hint_flags": 0, | |
991 | "manifest": { | |
992 | "type": 0 | |
993 | }, | |
994 | "watchers": {} | |
995 | }, | |
7c673cae | 996 | "union_shard_errors": [ |
94b18763 | 997 | "info_missing" |
7c673cae | 998 | ], |
b5b8bbf5 | 999 | "errors": [], |
7c673cae | 1000 | "object": { |
31f18b77 | 1001 | "version": 45, |
7c673cae FG |
1002 | "snap": "head", |
1003 | "locator": "", | |
1004 | "nspace": "", | |
1005 | "name": "ROBJ15" | |
1006 | } | |
1007 | }, | |
b5b8bbf5 FG |
1008 | { |
1009 | "errors": [], | |
1010 | "object": { | |
1011 | "locator": "", | |
1012 | "name": "ROBJ16", | |
1013 | "nspace": "", | |
1014 | "snap": "head", | |
1015 | "version": 0 | |
1016 | }, | |
1017 | "shards": [ | |
1018 | { | |
b5b8bbf5 | 1019 | "errors": [ |
94b18763 | 1020 | "snapset_missing" |
b5b8bbf5 FG |
1021 | ], |
1022 | "osd": 0, | |
1023 | "primary": false, | |
1024 | "size": 7 | |
1025 | }, | |
1026 | { | |
b5b8bbf5 | 1027 | "errors": [ |
94b18763 | 1028 | "snapset_corrupted" |
b5b8bbf5 FG |
1029 | ], |
1030 | "osd": 1, | |
1031 | "primary": true, | |
94b18763 | 1032 | "snapset": "bad-val", |
b5b8bbf5 FG |
1033 | "size": 7 |
1034 | } | |
1035 | ], | |
1036 | "union_shard_errors": [ | |
94b18763 FG |
1037 | "snapset_missing", |
1038 | "snapset_corrupted" | |
b5b8bbf5 FG |
1039 | ] |
1040 | }, | |
7c673cae FG |
1041 | { |
1042 | "shards": [ | |
1043 | { | |
1044 | "size": 7, | |
1045 | "errors": [], | |
b5b8bbf5 FG |
1046 | "osd": 0, |
1047 | "primary": false | |
7c673cae FG |
1048 | }, |
1049 | { | |
1050 | "errors": [ | |
1051 | "missing" | |
1052 | ], | |
b5b8bbf5 FG |
1053 | "osd": 1, |
1054 | "primary": true | |
7c673cae FG |
1055 | } |
1056 | ], | |
94b18763 FG |
1057 | "selected_object_info": { |
1058 | "oid": { | |
1059 | "oid": "ROBJ3", | |
1060 | "key": "", | |
1061 | "snapid": -2, | |
1062 | "hash": 625845583, | |
1063 | "max": 0, | |
1064 | "pool": 3, | |
1065 | "namespace": "" | |
1066 | }, | |
1067 | "version": "51'61", | |
1068 | "prior_version": "25'9", | |
1069 | "last_reqid": "osd.1.0:60", | |
1070 | "user_version": 9, | |
1071 | "size": 7, | |
1072 | "mtime": "", | |
1073 | "local_mtime": "", | |
1074 | "lost": 0, | |
1075 | "flags": [ | |
1076 | "dirty", | |
1077 | "omap", | |
1078 | "data_digest", | |
1079 | "omap_digest" | |
1080 | ], | |
1081 | "truncate_seq": 0, | |
1082 | "truncate_size": 0, | |
1083 | "data_digest": "0x2ddbf8f5", | |
1084 | "omap_digest": "0x00b35dfd", | |
1085 | "expected_object_size": 0, | |
1086 | "expected_write_size": 0, | |
1087 | "alloc_hint_flags": 0, | |
1088 | "manifest": { | |
1089 | "type": 0 | |
1090 | }, | |
1091 | "watchers": {} | |
1092 | }, | |
7c673cae FG |
1093 | "union_shard_errors": [ |
1094 | "missing" | |
1095 | ], | |
1096 | "errors": [], | |
1097 | "object": { | |
1098 | "version": 9, | |
1099 | "snap": "head", | |
1100 | "locator": "", | |
1101 | "nspace": "", | |
1102 | "name": "ROBJ3" | |
1103 | } | |
1104 | }, | |
1105 | { | |
1106 | "shards": [ | |
1107 | { | |
1108 | "attrs": [ | |
7c673cae FG |
1109 | { |
1110 | "Base64": false, | |
1111 | "value": "bad-val", | |
94b18763 | 1112 | "name": "key1-ROBJ8" |
7c673cae FG |
1113 | }, |
1114 | { | |
1115 | "Base64": false, | |
b32b8144 | 1116 | "value": "val2-ROBJ8", |
94b18763 | 1117 | "name": "key2-ROBJ8" |
7c673cae FG |
1118 | } |
1119 | ], | |
1120 | "size": 7, | |
1121 | "errors": [], | |
b5b8bbf5 FG |
1122 | "osd": 0, |
1123 | "primary": false | |
7c673cae FG |
1124 | }, |
1125 | { | |
1126 | "attrs": [ | |
7c673cae FG |
1127 | { |
1128 | "Base64": false, | |
1129 | "value": "val1-ROBJ8", | |
94b18763 | 1130 | "name": "key1-ROBJ8" |
7c673cae FG |
1131 | }, |
1132 | { | |
1133 | "Base64": false, | |
b32b8144 | 1134 | "value": "val3-ROBJ8", |
94b18763 | 1135 | "name": "key3-ROBJ8" |
7c673cae FG |
1136 | } |
1137 | ], | |
1138 | "size": 7, | |
1139 | "errors": [], | |
b5b8bbf5 FG |
1140 | "osd": 1, |
1141 | "primary": true | |
7c673cae FG |
1142 | } |
1143 | ], | |
94b18763 FG |
1144 | "selected_object_info": { |
1145 | "oid": { | |
1146 | "oid": "ROBJ8", | |
1147 | "key": "", | |
1148 | "snapid": -2, | |
1149 | "hash": 2359695969, | |
1150 | "max": 0, | |
1151 | "pool": 3, | |
1152 | "namespace": "" | |
1153 | }, | |
1154 | "version": "79'66", | |
1155 | "prior_version": "79'65", | |
1156 | "last_reqid": "client.4554.0:1", | |
1157 | "user_version": 66, | |
1158 | "size": 7, | |
1159 | "mtime": "", | |
1160 | "local_mtime": "", | |
1161 | "lost": 0, | |
1162 | "flags": [ | |
1163 | "dirty", | |
1164 | "omap", | |
1165 | "data_digest", | |
1166 | "omap_digest" | |
1167 | ], | |
1168 | "truncate_seq": 0, | |
1169 | "truncate_size": 0, | |
1170 | "data_digest": "0x2ddbf8f5", | |
1171 | "omap_digest": "0xd6be81dc", | |
1172 | "expected_object_size": 0, | |
1173 | "expected_write_size": 0, | |
1174 | "alloc_hint_flags": 0, | |
1175 | "manifest": { | |
1176 | "type": 0 | |
1177 | }, | |
1178 | "watchers": {} | |
1179 | }, | |
7c673cae FG |
1180 | "union_shard_errors": [], |
1181 | "errors": [ | |
1182 | "attr_value_mismatch", | |
1183 | "attr_name_mismatch" | |
1184 | ], | |
1185 | "object": { | |
b5b8bbf5 | 1186 | "version": 66, |
7c673cae FG |
1187 | "snap": "head", |
1188 | "locator": "", | |
1189 | "nspace": "", | |
1190 | "name": "ROBJ8" | |
1191 | } | |
1192 | }, | |
1193 | { | |
1194 | "shards": [ | |
1195 | { | |
94b18763 FG |
1196 | "object_info": { |
1197 | "oid": { | |
1198 | "oid": "ROBJ9", | |
1199 | "key": "", | |
1200 | "snapid": -2, | |
1201 | "hash": 537189375, | |
1202 | "max": 0, | |
1203 | "pool": 3, | |
1204 | "namespace": "" | |
1205 | }, | |
1206 | "version": "95'67", | |
1207 | "prior_version": "51'64", | |
1208 | "last_reqid": "client.4649.0:1", | |
1209 | "user_version": 67, | |
1210 | "size": 1, | |
1211 | "mtime": "", | |
1212 | "local_mtime": "", | |
1213 | "lost": 0, | |
1214 | "flags": [ | |
1215 | "dirty", | |
1216 | "omap", | |
1217 | "data_digest", | |
1218 | "omap_digest" | |
1219 | ], | |
1220 | "truncate_seq": 0, | |
1221 | "truncate_size": 0, | |
1222 | "data_digest": "0x2b63260d", | |
1223 | "omap_digest": "0x2eecc539", | |
1224 | "expected_object_size": 0, | |
1225 | "expected_write_size": 0, | |
1226 | "alloc_hint_flags": 0, | |
1227 | "manifest": { | |
1228 | "type": 0 | |
1229 | }, | |
1230 | "watchers": {} | |
1231 | }, | |
7c673cae FG |
1232 | "size": 1, |
1233 | "errors": [], | |
b5b8bbf5 FG |
1234 | "osd": 0, |
1235 | "primary": false | |
7c673cae FG |
1236 | }, |
1237 | { | |
94b18763 FG |
1238 | "object_info": { |
1239 | "oid": { | |
1240 | "oid": "ROBJ9", | |
1241 | "key": "", | |
1242 | "snapid": -2, | |
1243 | "hash": 537189375, | |
1244 | "max": 0, | |
1245 | "pool": 3, | |
1246 | "namespace": "" | |
1247 | }, | |
1248 | "version": "51'64", | |
1249 | "prior_version": "37'27", | |
1250 | "last_reqid": "osd.1.0:63", | |
1251 | "user_version": 27, | |
1252 | "size": 7, | |
1253 | "mtime": "2018-04-05 14:33:25.352485", | |
1254 | "local_mtime": "2018-04-05 14:33:25.353746", | |
1255 | "lost": 0, | |
1256 | "flags": [ | |
1257 | "dirty", | |
1258 | "omap", | |
1259 | "data_digest", | |
1260 | "omap_digest" | |
1261 | ], | |
1262 | "truncate_seq": 0, | |
1263 | "truncate_size": 0, | |
1264 | "data_digest": "0x2ddbf8f5", | |
1265 | "omap_digest": "0x2eecc539", | |
1266 | "expected_object_size": 0, | |
1267 | "expected_write_size": 0, | |
1268 | "alloc_hint_flags": 0, | |
1269 | "manifest": { | |
1270 | "type": 0 | |
1271 | }, | |
1272 | "watchers": {} | |
1273 | }, | |
7c673cae | 1274 | "size": 1, |
b5b8bbf5 | 1275 | "errors": [ |
94b18763 | 1276 | "obj_size_info_mismatch" |
b5b8bbf5 FG |
1277 | ], |
1278 | "osd": 1, | |
1279 | "primary": true | |
7c673cae FG |
1280 | } |
1281 | ], | |
94b18763 FG |
1282 | "selected_object_info": { |
1283 | "oid": { | |
1284 | "oid": "ROBJ9", | |
1285 | "key": "", | |
1286 | "snapid": -2, | |
1287 | "hash": 537189375, | |
1288 | "max": 0, | |
1289 | "pool": 3, | |
1290 | "namespace": "" | |
1291 | }, | |
1292 | "version": "95'67", | |
1293 | "prior_version": "51'64", | |
1294 | "last_reqid": "client.4649.0:1", | |
1295 | "user_version": 67, | |
1296 | "size": 1, | |
1297 | "mtime": "", | |
1298 | "local_mtime": "", | |
1299 | "lost": 0, | |
1300 | "flags": [ | |
1301 | "dirty", | |
1302 | "omap", | |
1303 | "data_digest", | |
1304 | "omap_digest" | |
1305 | ], | |
1306 | "truncate_seq": 0, | |
1307 | "truncate_size": 0, | |
1308 | "data_digest": "0x2b63260d", | |
1309 | "omap_digest": "0x2eecc539", | |
1310 | "expected_object_size": 0, | |
1311 | "expected_write_size": 0, | |
1312 | "alloc_hint_flags": 0, | |
1313 | "manifest": { | |
1314 | "type": 0 | |
1315 | }, | |
1316 | "watchers": {} | |
1317 | }, | |
b5b8bbf5 | 1318 | "union_shard_errors": [ |
94b18763 | 1319 | "obj_size_info_mismatch" |
b5b8bbf5 | 1320 | ], |
7c673cae | 1321 | "errors": [ |
b5b8bbf5 | 1322 | "object_info_inconsistency" |
7c673cae FG |
1323 | ], |
1324 | "object": { | |
b5b8bbf5 | 1325 | "version": 67, |
7c673cae FG |
1326 | "snap": "head", |
1327 | "locator": "", | |
1328 | "nspace": "", | |
1329 | "name": "ROBJ9" | |
1330 | } | |
1331 | } | |
1332 | ], | |
1333 | "epoch": 0 | |
1334 | } | |
1335 | EOF | |
1336 | ||
94b18763 | 1337 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
31f18b77 | 1338 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
1339 | if test $getjson = "yes" |
1340 | then | |
1341 | jq '.' $dir/json > save1.json | |
1342 | fi | |
1343 | ||
94b18763 | 1344 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
1345 | then |
1346 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
1347 | fi | |
1348 | ||
7c673cae FG |
1349 | objname=ROBJ9 |
1350 | # Change data and size again because digest was recomputed | |
1351 | echo -n ZZZ > $dir/change | |
1352 | rados --pool $poolname put $objname $dir/change | |
1353 | # Set one to an even older value | |
1354 | objectstore_tool $dir 0 $objname set-attr _ $dir/robj9-oi | |
1355 | rm $dir/oi $dir/change | |
1356 | ||
1357 | objname=ROBJ10 | |
1358 | objectstore_tool $dir 1 $objname get-attr _ > $dir/oi | |
1359 | rados --pool $poolname setomapval $objname key2-$objname val2-$objname | |
1360 | objectstore_tool $dir 0 $objname set-attr _ $dir/oi | |
1361 | objectstore_tool $dir 1 $objname set-attr _ $dir/oi | |
1362 | rm $dir/oi | |
1363 | ||
b5b8bbf5 FG |
1364 | inject_eio rep data $poolname ROBJ11 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 |
1365 | inject_eio rep mdata $poolname ROBJ12 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1366 | inject_eio rep mdata $poolname ROBJ13 $dir 1 || return 1 # shard 1 of [1, 0], osd.0 | |
1367 | inject_eio rep data $poolname ROBJ13 $dir 0 || return 1 # shard 0 of [1, 0], osd.1 | |
7c673cae FG |
1368 | pg_deep_scrub $pg |
1369 | ||
1370 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
1371 | # Check pg count | |
1372 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
1373 | # Check pgid | |
1374 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
1375 | ||
1376 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
1377 | # Get epoch for repair-get requests | |
1378 | epoch=$(jq .epoch $dir/json) | |
1379 | ||
94b18763 | 1380 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
1381 | { |
1382 | "inconsistents": [ | |
1383 | { | |
1384 | "shards": [ | |
1385 | { | |
1386 | "data_digest": "0x2ddbf8f5", | |
1387 | "omap_digest": "0xf5fba2c6", | |
1388 | "size": 7, | |
1389 | "errors": [], | |
b5b8bbf5 FG |
1390 | "osd": 0, |
1391 | "primary": false | |
7c673cae FG |
1392 | }, |
1393 | { | |
94b18763 FG |
1394 | "object_info": { |
1395 | "oid": { | |
1396 | "oid": "ROBJ1", | |
1397 | "key": "", | |
1398 | "snapid": -2, | |
1399 | "hash": 1454963827, | |
1400 | "max": 0, | |
1401 | "pool": 3, | |
1402 | "namespace": "" | |
1403 | }, | |
1404 | "version": "51'58", | |
1405 | "prior_version": "21'3", | |
1406 | "last_reqid": "osd.1.0:57", | |
1407 | "user_version": 3, | |
1408 | "size": 7, | |
1409 | "mtime": "2018-04-05 14:33:19.804040", | |
1410 | "local_mtime": "2018-04-05 14:33:19.804839", | |
1411 | "lost": 0, | |
1412 | "flags": [ | |
1413 | "dirty", | |
1414 | "omap", | |
1415 | "data_digest", | |
1416 | "omap_digest" | |
1417 | ], | |
1418 | "truncate_seq": 0, | |
1419 | "truncate_size": 0, | |
1420 | "data_digest": "0x2ddbf8f5", | |
1421 | "omap_digest": "0xf5fba2c6", | |
1422 | "expected_object_size": 0, | |
1423 | "expected_write_size": 0, | |
1424 | "alloc_hint_flags": 0, | |
1425 | "manifest": { | |
1426 | "type": 0 | |
1427 | }, | |
1428 | "watchers": {} | |
1429 | }, | |
7c673cae FG |
1430 | "data_digest": "0x2d4a11c2", |
1431 | "omap_digest": "0xf5fba2c6", | |
1432 | "size": 9, | |
1433 | "errors": [ | |
94b18763 FG |
1434 | "data_digest_mismatch_info", |
1435 | "size_mismatch_info", | |
1436 | "obj_size_info_mismatch" | |
7c673cae | 1437 | ], |
b5b8bbf5 FG |
1438 | "osd": 1, |
1439 | "primary": true | |
7c673cae FG |
1440 | } |
1441 | ], | |
94b18763 FG |
1442 | "selected_object_info": { |
1443 | "oid": { | |
1444 | "oid": "ROBJ1", | |
1445 | "key": "", | |
1446 | "snapid": -2, | |
1447 | "hash": 1454963827, | |
1448 | "max": 0, | |
1449 | "pool": 3, | |
1450 | "namespace": "" | |
1451 | }, | |
1452 | "version": "51'58", | |
1453 | "prior_version": "21'3", | |
1454 | "last_reqid": "osd.1.0:57", | |
1455 | "user_version": 3, | |
1456 | "size": 7, | |
1457 | "mtime": "2018-04-05 14:33:19.804040", | |
1458 | "local_mtime": "2018-04-05 14:33:19.804839", | |
1459 | "lost": 0, | |
1460 | "flags": [ | |
1461 | "dirty", | |
1462 | "omap", | |
1463 | "data_digest", | |
1464 | "omap_digest" | |
1465 | ], | |
1466 | "truncate_seq": 0, | |
1467 | "truncate_size": 0, | |
1468 | "data_digest": "0x2ddbf8f5", | |
1469 | "omap_digest": "0xf5fba2c6", | |
1470 | "expected_object_size": 0, | |
1471 | "expected_write_size": 0, | |
1472 | "alloc_hint_flags": 0, | |
1473 | "manifest": { | |
1474 | "type": 0 | |
1475 | }, | |
1476 | "watchers": {} | |
1477 | }, | |
7c673cae | 1478 | "union_shard_errors": [ |
94b18763 FG |
1479 | "data_digest_mismatch_info", |
1480 | "size_mismatch_info", | |
1481 | "obj_size_info_mismatch" | |
7c673cae FG |
1482 | ], |
1483 | "errors": [ | |
1484 | "data_digest_mismatch", | |
1485 | "size_mismatch" | |
1486 | ], | |
1487 | "object": { | |
1488 | "version": 3, | |
1489 | "snap": "head", | |
1490 | "locator": "", | |
1491 | "nspace": "", | |
1492 | "name": "ROBJ1" | |
1493 | } | |
1494 | }, | |
1495 | { | |
1496 | "shards": [ | |
1497 | { | |
1498 | "data_digest": "0x2ddbf8f5", | |
1499 | "omap_digest": "0xa8dd5adc", | |
1500 | "size": 7, | |
1501 | "errors": [ | |
94b18763 | 1502 | "omap_digest_mismatch_info" |
7c673cae | 1503 | ], |
b5b8bbf5 FG |
1504 | "osd": 0, |
1505 | "primary": false | |
7c673cae FG |
1506 | }, |
1507 | { | |
1508 | "data_digest": "0x2ddbf8f5", | |
1509 | "omap_digest": "0xa8dd5adc", | |
1510 | "size": 7, | |
1511 | "errors": [ | |
94b18763 | 1512 | "omap_digest_mismatch_info" |
7c673cae | 1513 | ], |
b5b8bbf5 FG |
1514 | "osd": 1, |
1515 | "primary": true | |
7c673cae FG |
1516 | } |
1517 | ], | |
94b18763 FG |
1518 | "selected_object_info": { |
1519 | "alloc_hint_flags": 0, | |
1520 | "data_digest": "0x2ddbf8f5", | |
1521 | "expected_object_size": 0, | |
1522 | "expected_write_size": 0, | |
1523 | "flags": [ | |
1524 | "dirty", | |
1525 | "omap", | |
1526 | "data_digest", | |
1527 | "omap_digest" | |
1528 | ], | |
1529 | "last_reqid": "osd.0.0:50", | |
1530 | "local_mtime": "2018-04-05 14:33:26.762368", | |
1531 | "lost": 0, | |
1532 | "manifest": { | |
1533 | "type": 0 | |
1534 | }, | |
1535 | "mtime": "2018-04-05 14:33:26.762368", | |
1536 | "oid": { | |
1537 | "hash": 3174666125, | |
1538 | "key": "", | |
1539 | "max": 0, | |
1540 | "namespace": "", | |
1541 | "oid": "ROBJ10", | |
1542 | "pool": 3, | |
1543 | "snapid": -2 | |
1544 | }, | |
1545 | "omap_digest": "0xc2025a24", | |
1546 | "prior_version": "41'33", | |
1547 | "size": 7, | |
1548 | "truncate_seq": 0, | |
1549 | "truncate_size": 0, | |
1550 | "user_version": 30, | |
1551 | "version": "47'51", | |
1552 | "watchers": {} | |
1553 | }, | |
7c673cae | 1554 | "union_shard_errors": [ |
94b18763 | 1555 | "omap_digest_mismatch_info" |
7c673cae FG |
1556 | ], |
1557 | "errors": [], | |
1558 | "object": { | |
31f18b77 | 1559 | "version": 30, |
7c673cae FG |
1560 | "snap": "head", |
1561 | "locator": "", | |
1562 | "nspace": "", | |
1563 | "name": "ROBJ10" | |
1564 | } | |
1565 | }, | |
1566 | { | |
1567 | "shards": [ | |
1568 | { | |
1569 | "data_digest": "0x2ddbf8f5", | |
1570 | "omap_digest": "0xa03cef03", | |
1571 | "size": 7, | |
1572 | "errors": [], | |
b5b8bbf5 FG |
1573 | "osd": 0, |
1574 | "primary": false | |
7c673cae FG |
1575 | }, |
1576 | { | |
1577 | "size": 7, | |
1578 | "errors": [ | |
1579 | "read_error" | |
1580 | ], | |
b5b8bbf5 FG |
1581 | "osd": 1, |
1582 | "primary": true | |
7c673cae FG |
1583 | } |
1584 | ], | |
94b18763 FG |
1585 | "selected_object_info": { |
1586 | "oid": { | |
1587 | "oid": "ROBJ11", | |
1588 | "key": "", | |
1589 | "snapid": -2, | |
1590 | "hash": 1828574689, | |
1591 | "max": 0, | |
1592 | "pool": 3, | |
1593 | "namespace": "" | |
1594 | }, | |
1595 | "version": "51'52", | |
1596 | "prior_version": "41'33", | |
1597 | "last_reqid": "osd.1.0:51", | |
1598 | "user_version": 33, | |
1599 | "size": 7, | |
1600 | "mtime": "2018-04-05 14:33:26.761286", | |
1601 | "local_mtime": "2018-04-05 14:33:26.762368", | |
1602 | "lost": 0, | |
1603 | "flags": [ | |
1604 | "dirty", | |
1605 | "omap", | |
1606 | "data_digest", | |
1607 | "omap_digest" | |
1608 | ], | |
1609 | "truncate_seq": 0, | |
1610 | "truncate_size": 0, | |
1611 | "data_digest": "0x2ddbf8f5", | |
1612 | "omap_digest": "0xa03cef03", | |
1613 | "expected_object_size": 0, | |
1614 | "expected_write_size": 0, | |
1615 | "alloc_hint_flags": 0, | |
1616 | "manifest": { | |
1617 | "type": 0 | |
1618 | }, | |
1619 | "watchers": {} | |
1620 | }, | |
7c673cae FG |
1621 | "union_shard_errors": [ |
1622 | "read_error" | |
1623 | ], | |
1624 | "errors": [], | |
1625 | "object": { | |
31f18b77 | 1626 | "version": 33, |
7c673cae FG |
1627 | "snap": "head", |
1628 | "locator": "", | |
1629 | "nspace": "", | |
1630 | "name": "ROBJ11" | |
1631 | } | |
1632 | }, | |
1633 | { | |
1634 | "shards": [ | |
1635 | { | |
1636 | "errors": [ | |
1637 | "stat_error" | |
1638 | ], | |
b5b8bbf5 FG |
1639 | "osd": 0, |
1640 | "primary": false | |
7c673cae FG |
1641 | }, |
1642 | { | |
1643 | "data_digest": "0x2ddbf8f5", | |
1644 | "omap_digest": "0x067f306a", | |
1645 | "size": 7, | |
1646 | "errors": [], | |
b5b8bbf5 FG |
1647 | "osd": 1, |
1648 | "primary": true | |
7c673cae FG |
1649 | } |
1650 | ], | |
94b18763 FG |
1651 | "selected_object_info": { |
1652 | "oid": { | |
1653 | "oid": "ROBJ12", | |
1654 | "key": "", | |
1655 | "snapid": -2, | |
1656 | "hash": 3920199997, | |
1657 | "max": 0, | |
1658 | "pool": 3, | |
1659 | "namespace": "" | |
1660 | }, | |
1661 | "version": "51'56", | |
1662 | "prior_version": "43'36", | |
1663 | "last_reqid": "osd.1.0:55", | |
1664 | "user_version": 36, | |
1665 | "size": 7, | |
1666 | "mtime": "2018-04-05 14:33:27.460958", | |
1667 | "local_mtime": "2018-04-05 14:33:27.462109", | |
1668 | "lost": 0, | |
1669 | "flags": [ | |
1670 | "dirty", | |
1671 | "omap", | |
1672 | "data_digest", | |
1673 | "omap_digest" | |
1674 | ], | |
1675 | "truncate_seq": 0, | |
1676 | "truncate_size": 0, | |
1677 | "data_digest": "0x2ddbf8f5", | |
1678 | "omap_digest": "0x067f306a", | |
1679 | "expected_object_size": 0, | |
1680 | "expected_write_size": 0, | |
1681 | "alloc_hint_flags": 0, | |
1682 | "manifest": { | |
1683 | "type": 0 | |
1684 | }, | |
1685 | "watchers": {} | |
1686 | }, | |
7c673cae FG |
1687 | "union_shard_errors": [ |
1688 | "stat_error" | |
1689 | ], | |
1690 | "errors": [], | |
1691 | "object": { | |
31f18b77 | 1692 | "version": 36, |
7c673cae FG |
1693 | "snap": "head", |
1694 | "locator": "", | |
1695 | "nspace": "", | |
1696 | "name": "ROBJ12" | |
1697 | } | |
1698 | }, | |
1699 | { | |
1700 | "shards": [ | |
1701 | { | |
1702 | "errors": [ | |
1703 | "stat_error" | |
1704 | ], | |
b5b8bbf5 FG |
1705 | "osd": 0, |
1706 | "primary": false | |
7c673cae FG |
1707 | }, |
1708 | { | |
1709 | "size": 7, | |
1710 | "errors": [ | |
1711 | "read_error" | |
1712 | ], | |
b5b8bbf5 FG |
1713 | "osd": 1, |
1714 | "primary": true | |
7c673cae FG |
1715 | } |
1716 | ], | |
1717 | "union_shard_errors": [ | |
1718 | "stat_error", | |
1719 | "read_error" | |
1720 | ], | |
1721 | "errors": [], | |
1722 | "object": { | |
1723 | "version": 0, | |
1724 | "snap": "head", | |
1725 | "locator": "", | |
1726 | "nspace": "", | |
1727 | "name": "ROBJ13" | |
1728 | } | |
1729 | }, | |
1730 | { | |
1731 | "shards": [ | |
1732 | { | |
94b18763 | 1733 | "object_info": "bad-val", |
7c673cae FG |
1734 | "data_digest": "0x2ddbf8f5", |
1735 | "omap_digest": "0x4f14f849", | |
1736 | "size": 7, | |
1737 | "errors": [ | |
94b18763 | 1738 | "info_corrupted" |
7c673cae | 1739 | ], |
b5b8bbf5 FG |
1740 | "osd": 0, |
1741 | "primary": false | |
7c673cae FG |
1742 | }, |
1743 | { | |
1744 | "data_digest": "0x2ddbf8f5", | |
1745 | "omap_digest": "0x4f14f849", | |
1746 | "size": 7, | |
1747 | "errors": [ | |
94b18763 | 1748 | "info_missing" |
7c673cae | 1749 | ], |
b5b8bbf5 FG |
1750 | "osd": 1, |
1751 | "primary": true | |
7c673cae FG |
1752 | } |
1753 | ], | |
1754 | "union_shard_errors": [ | |
94b18763 FG |
1755 | "info_missing", |
1756 | "info_corrupted" | |
7c673cae FG |
1757 | ], |
1758 | "errors": [], | |
1759 | "object": { | |
1760 | "version": 0, | |
1761 | "snap": "head", | |
1762 | "locator": "", | |
1763 | "nspace": "", | |
1764 | "name": "ROBJ14" | |
1765 | } | |
1766 | }, | |
1767 | { | |
1768 | "shards": [ | |
1769 | { | |
94b18763 FG |
1770 | "object_info": { |
1771 | "oid": { | |
1772 | "oid": "ROBJ15", | |
1773 | "key": "", | |
1774 | "snapid": -2, | |
1775 | "hash": 504996876, | |
1776 | "max": 0, | |
1777 | "pool": 3, | |
1778 | "namespace": "" | |
7c673cae | 1779 | }, |
94b18763 FG |
1780 | "version": "51'49", |
1781 | "prior_version": "49'45", | |
1782 | "last_reqid": "osd.1.0:48", | |
1783 | "user_version": 45, | |
1784 | "size": 7, | |
1785 | "mtime": "2018-04-05 14:33:29.498969", | |
1786 | "local_mtime": "2018-04-05 14:33:29.499890", | |
1787 | "lost": 0, | |
1788 | "flags": [ | |
1789 | "dirty", | |
1790 | "omap", | |
1791 | "data_digest", | |
1792 | "omap_digest" | |
1793 | ], | |
1794 | "truncate_seq": 0, | |
1795 | "truncate_size": 0, | |
1796 | "data_digest": "0x2ddbf8f5", | |
1797 | "omap_digest": "0x2d2a4d6e", | |
1798 | "expected_object_size": 0, | |
1799 | "expected_write_size": 0, | |
1800 | "alloc_hint_flags": 0, | |
1801 | "manifest": { | |
1802 | "type": 0 | |
1803 | }, | |
1804 | "watchers": {} | |
1805 | }, | |
7c673cae FG |
1806 | "data_digest": "0x2ddbf8f5", |
1807 | "omap_digest": "0x2d2a4d6e", | |
1808 | "size": 7, | |
1809 | "errors": [], | |
b5b8bbf5 FG |
1810 | "osd": 0, |
1811 | "primary": false | |
7c673cae FG |
1812 | }, |
1813 | { | |
7c673cae FG |
1814 | "data_digest": "0x2ddbf8f5", |
1815 | "omap_digest": "0x2d2a4d6e", | |
1816 | "size": 7, | |
1817 | "errors": [ | |
94b18763 | 1818 | "info_missing" |
7c673cae | 1819 | ], |
b5b8bbf5 FG |
1820 | "osd": 1, |
1821 | "primary": true | |
7c673cae FG |
1822 | } |
1823 | ], | |
94b18763 FG |
1824 | "selected_object_info": { |
1825 | "oid": { | |
1826 | "oid": "ROBJ15", | |
1827 | "key": "", | |
1828 | "snapid": -2, | |
1829 | "hash": 504996876, | |
1830 | "max": 0, | |
1831 | "pool": 3, | |
1832 | "namespace": "" | |
1833 | }, | |
1834 | "version": "51'49", | |
1835 | "prior_version": "49'45", | |
1836 | "last_reqid": "osd.1.0:48", | |
1837 | "user_version": 45, | |
1838 | "size": 7, | |
1839 | "mtime": "2018-04-05 14:33:29.498969", | |
1840 | "local_mtime": "2018-04-05 14:33:29.499890", | |
1841 | "lost": 0, | |
1842 | "flags": [ | |
1843 | "dirty", | |
1844 | "omap", | |
1845 | "data_digest", | |
1846 | "omap_digest" | |
1847 | ], | |
1848 | "truncate_seq": 0, | |
1849 | "truncate_size": 0, | |
1850 | "data_digest": "0x2ddbf8f5", | |
1851 | "omap_digest": "0x2d2a4d6e", | |
1852 | "expected_object_size": 0, | |
1853 | "expected_write_size": 0, | |
1854 | "alloc_hint_flags": 0, | |
1855 | "manifest": { | |
1856 | "type": 0 | |
1857 | }, | |
1858 | "watchers": {} | |
1859 | }, | |
7c673cae | 1860 | "union_shard_errors": [ |
94b18763 | 1861 | "info_missing" |
7c673cae | 1862 | ], |
b5b8bbf5 | 1863 | "errors": [], |
7c673cae | 1864 | "object": { |
31f18b77 | 1865 | "version": 45, |
7c673cae FG |
1866 | "snap": "head", |
1867 | "locator": "", | |
1868 | "nspace": "", | |
1869 | "name": "ROBJ15" | |
1870 | } | |
1871 | }, | |
b5b8bbf5 FG |
1872 | { |
1873 | "errors": [], | |
1874 | "object": { | |
1875 | "locator": "", | |
1876 | "name": "ROBJ16", | |
1877 | "nspace": "", | |
1878 | "snap": "head", | |
1879 | "version": 0 | |
1880 | }, | |
1881 | "shards": [ | |
1882 | { | |
b5b8bbf5 FG |
1883 | "data_digest": "0x2ddbf8f5", |
1884 | "errors": [ | |
94b18763 | 1885 | "snapset_missing" |
b5b8bbf5 FG |
1886 | ], |
1887 | "omap_digest": "0x8b699207", | |
1888 | "osd": 0, | |
1889 | "primary": false, | |
1890 | "size": 7 | |
1891 | }, | |
1892 | { | |
94b18763 | 1893 | "snapset": "bad-val", |
b5b8bbf5 FG |
1894 | "data_digest": "0x2ddbf8f5", |
1895 | "errors": [ | |
94b18763 | 1896 | "snapset_corrupted" |
b5b8bbf5 FG |
1897 | ], |
1898 | "omap_digest": "0x8b699207", | |
1899 | "osd": 1, | |
1900 | "primary": true, | |
1901 | "size": 7 | |
1902 | } | |
1903 | ], | |
1904 | "union_shard_errors": [ | |
94b18763 FG |
1905 | "snapset_missing", |
1906 | "snapset_corrupted" | |
b5b8bbf5 FG |
1907 | ] |
1908 | }, | |
7c673cae FG |
1909 | { |
1910 | "shards": [ | |
1911 | { | |
1912 | "data_digest": "0x578a4830", | |
1913 | "omap_digest": "0xf8e11918", | |
1914 | "size": 7, | |
1915 | "errors": [ | |
94b18763 | 1916 | "data_digest_mismatch_info" |
7c673cae | 1917 | ], |
b5b8bbf5 FG |
1918 | "osd": 0, |
1919 | "primary": false | |
7c673cae FG |
1920 | }, |
1921 | { | |
1922 | "data_digest": "0x2ddbf8f5", | |
1923 | "omap_digest": "0xf8e11918", | |
1924 | "size": 7, | |
1925 | "errors": [], | |
b5b8bbf5 FG |
1926 | "osd": 1, |
1927 | "primary": true | |
7c673cae FG |
1928 | } |
1929 | ], | |
94b18763 FG |
1930 | "selected_object_info": { |
1931 | "oid": { | |
1932 | "oid": "ROBJ2", | |
1933 | "key": "", | |
1934 | "snapid": -2, | |
1935 | "hash": 2026323607, | |
1936 | "max": 0, | |
1937 | "pool": 3, | |
1938 | "namespace": "" | |
1939 | }, | |
1940 | "version": "51'60", | |
1941 | "prior_version": "23'6", | |
1942 | "last_reqid": "osd.1.0:59", | |
1943 | "user_version": 6, | |
1944 | "size": 7, | |
1945 | "mtime": "2018-04-05 14:33:20.498756", | |
1946 | "local_mtime": "2018-04-05 14:33:20.499704", | |
1947 | "lost": 0, | |
1948 | "flags": [ | |
1949 | "dirty", | |
1950 | "omap", | |
1951 | "data_digest", | |
1952 | "omap_digest" | |
1953 | ], | |
1954 | "truncate_seq": 0, | |
1955 | "truncate_size": 0, | |
1956 | "data_digest": "0x2ddbf8f5", | |
1957 | "omap_digest": "0xf8e11918", | |
1958 | "expected_object_size": 0, | |
1959 | "expected_write_size": 0, | |
1960 | "alloc_hint_flags": 0, | |
1961 | "manifest": { | |
1962 | "type": 0 | |
1963 | }, | |
1964 | "watchers": {} | |
1965 | }, | |
7c673cae | 1966 | "union_shard_errors": [ |
94b18763 | 1967 | "data_digest_mismatch_info" |
7c673cae FG |
1968 | ], |
1969 | "errors": [ | |
1970 | "data_digest_mismatch" | |
1971 | ], | |
1972 | "object": { | |
1973 | "version": 6, | |
1974 | "snap": "head", | |
1975 | "locator": "", | |
1976 | "nspace": "", | |
1977 | "name": "ROBJ2" | |
1978 | } | |
1979 | }, | |
1980 | { | |
1981 | "shards": [ | |
1982 | { | |
1983 | "data_digest": "0x2ddbf8f5", | |
1984 | "omap_digest": "0x00b35dfd", | |
1985 | "size": 7, | |
1986 | "errors": [], | |
b5b8bbf5 FG |
1987 | "osd": 0, |
1988 | "primary": false | |
7c673cae FG |
1989 | }, |
1990 | { | |
1991 | "errors": [ | |
1992 | "missing" | |
1993 | ], | |
b5b8bbf5 FG |
1994 | "osd": 1, |
1995 | "primary": true | |
7c673cae FG |
1996 | } |
1997 | ], | |
94b18763 FG |
1998 | "selected_object_info": { |
1999 | "oid": { | |
2000 | "oid": "ROBJ3", | |
2001 | "key": "", | |
2002 | "snapid": -2, | |
2003 | "hash": 625845583, | |
2004 | "max": 0, | |
2005 | "pool": 3, | |
2006 | "namespace": "" | |
2007 | }, | |
2008 | "version": "51'61", | |
2009 | "prior_version": "25'9", | |
2010 | "last_reqid": "osd.1.0:60", | |
2011 | "user_version": 9, | |
2012 | "size": 7, | |
2013 | "mtime": "2018-04-05 14:33:21.189382", | |
2014 | "local_mtime": "2018-04-05 14:33:21.190446", | |
2015 | "lost": 0, | |
2016 | "flags": [ | |
2017 | "dirty", | |
2018 | "omap", | |
2019 | "data_digest", | |
2020 | "omap_digest" | |
2021 | ], | |
2022 | "truncate_seq": 0, | |
2023 | "truncate_size": 0, | |
2024 | "data_digest": "0x2ddbf8f5", | |
2025 | "omap_digest": "0x00b35dfd", | |
2026 | "expected_object_size": 0, | |
2027 | "expected_write_size": 0, | |
2028 | "alloc_hint_flags": 0, | |
2029 | "manifest": { | |
2030 | "type": 0 | |
2031 | }, | |
2032 | "watchers": {} | |
2033 | }, | |
7c673cae FG |
2034 | "union_shard_errors": [ |
2035 | "missing" | |
2036 | ], | |
2037 | "errors": [], | |
2038 | "object": { | |
2039 | "version": 9, | |
2040 | "snap": "head", | |
2041 | "locator": "", | |
2042 | "nspace": "", | |
2043 | "name": "ROBJ3" | |
2044 | } | |
2045 | }, | |
2046 | { | |
2047 | "shards": [ | |
2048 | { | |
2049 | "data_digest": "0x2ddbf8f5", | |
2050 | "omap_digest": "0xd7178dfe", | |
2051 | "size": 7, | |
31f18b77 | 2052 | "errors": [ |
94b18763 | 2053 | "omap_digest_mismatch_info" |
31f18b77 | 2054 | ], |
b5b8bbf5 FG |
2055 | "osd": 0, |
2056 | "primary": false | |
7c673cae FG |
2057 | }, |
2058 | { | |
2059 | "data_digest": "0x2ddbf8f5", | |
2060 | "omap_digest": "0xe2d46ea4", | |
2061 | "size": 7, | |
31f18b77 | 2062 | "errors": [], |
b5b8bbf5 FG |
2063 | "osd": 1, |
2064 | "primary": true | |
7c673cae FG |
2065 | } |
2066 | ], | |
94b18763 FG |
2067 | "selected_object_info": { |
2068 | "oid": { | |
2069 | "oid": "ROBJ4", | |
2070 | "key": "", | |
2071 | "snapid": -2, | |
2072 | "hash": 2360875311, | |
2073 | "max": 0, | |
2074 | "pool": 3, | |
2075 | "namespace": "" | |
2076 | }, | |
2077 | "version": "51'62", | |
2078 | "prior_version": "27'12", | |
2079 | "last_reqid": "osd.1.0:61", | |
2080 | "user_version": 12, | |
2081 | "size": 7, | |
2082 | "mtime": "2018-04-05 14:33:21.862313", | |
2083 | "local_mtime": "2018-04-05 14:33:21.863261", | |
2084 | "lost": 0, | |
2085 | "flags": [ | |
2086 | "dirty", | |
2087 | "omap", | |
2088 | "data_digest", | |
2089 | "omap_digest" | |
2090 | ], | |
2091 | "truncate_seq": 0, | |
2092 | "truncate_size": 0, | |
2093 | "data_digest": "0x2ddbf8f5", | |
2094 | "omap_digest": "0xe2d46ea4", | |
2095 | "expected_object_size": 0, | |
2096 | "expected_write_size": 0, | |
2097 | "alloc_hint_flags": 0, | |
2098 | "manifest": { | |
2099 | "type": 0 | |
2100 | }, | |
2101 | "watchers": {} | |
2102 | }, | |
7c673cae | 2103 | "union_shard_errors": [ |
94b18763 | 2104 | "omap_digest_mismatch_info" |
7c673cae FG |
2105 | ], |
2106 | "errors": [ | |
2107 | "omap_digest_mismatch" | |
2108 | ], | |
2109 | "object": { | |
2110 | "version": 12, | |
2111 | "snap": "head", | |
2112 | "locator": "", | |
2113 | "nspace": "", | |
2114 | "name": "ROBJ4" | |
2115 | } | |
2116 | }, | |
2117 | { | |
2118 | "shards": [ | |
2119 | { | |
2120 | "data_digest": "0x2ddbf8f5", | |
2121 | "omap_digest": "0x1a862a41", | |
2122 | "size": 7, | |
2123 | "errors": [], | |
b5b8bbf5 FG |
2124 | "osd": 0, |
2125 | "primary": false | |
7c673cae FG |
2126 | }, |
2127 | { | |
2128 | "data_digest": "0x2ddbf8f5", | |
2129 | "omap_digest": "0x06cac8f6", | |
2130 | "size": 7, | |
2131 | "errors": [ | |
94b18763 | 2132 | "omap_digest_mismatch_info" |
7c673cae | 2133 | ], |
b5b8bbf5 FG |
2134 | "osd": 1, |
2135 | "primary": true | |
7c673cae FG |
2136 | } |
2137 | ], | |
94b18763 FG |
2138 | "selected_object_info": { |
2139 | "oid": { | |
2140 | "oid": "ROBJ5", | |
2141 | "key": "", | |
2142 | "snapid": -2, | |
2143 | "hash": 2334915887, | |
2144 | "max": 0, | |
2145 | "pool": 3, | |
2146 | "namespace": "" | |
2147 | }, | |
2148 | "version": "51'63", | |
2149 | "prior_version": "29'15", | |
2150 | "last_reqid": "osd.1.0:62", | |
2151 | "user_version": 15, | |
2152 | "size": 7, | |
2153 | "mtime": "2018-04-05 14:33:22.589300", | |
2154 | "local_mtime": "2018-04-05 14:33:22.590376", | |
2155 | "lost": 0, | |
2156 | "flags": [ | |
2157 | "dirty", | |
2158 | "omap", | |
2159 | "data_digest", | |
2160 | "omap_digest" | |
2161 | ], | |
2162 | "truncate_seq": 0, | |
2163 | "truncate_size": 0, | |
2164 | "data_digest": "0x2ddbf8f5", | |
2165 | "omap_digest": "0x1a862a41", | |
2166 | "expected_object_size": 0, | |
2167 | "expected_write_size": 0, | |
2168 | "alloc_hint_flags": 0, | |
2169 | "manifest": { | |
2170 | "type": 0 | |
2171 | }, | |
2172 | "watchers": {} | |
2173 | }, | |
7c673cae | 2174 | "union_shard_errors": [ |
94b18763 | 2175 | "omap_digest_mismatch_info" |
7c673cae FG |
2176 | ], |
2177 | "errors": [ | |
2178 | "omap_digest_mismatch" | |
2179 | ], | |
2180 | "object": { | |
2181 | "version": 15, | |
2182 | "snap": "head", | |
2183 | "locator": "", | |
2184 | "nspace": "", | |
2185 | "name": "ROBJ5" | |
2186 | } | |
2187 | }, | |
2188 | { | |
2189 | "shards": [ | |
2190 | { | |
2191 | "data_digest": "0x2ddbf8f5", | |
2192 | "omap_digest": "0x689ee887", | |
2193 | "size": 7, | |
31f18b77 | 2194 | "errors": [ |
94b18763 | 2195 | "omap_digest_mismatch_info" |
31f18b77 | 2196 | ], |
b5b8bbf5 FG |
2197 | "osd": 0, |
2198 | "primary": false | |
7c673cae FG |
2199 | }, |
2200 | { | |
2201 | "data_digest": "0x2ddbf8f5", | |
2202 | "omap_digest": "0x179c919f", | |
2203 | "size": 7, | |
31f18b77 | 2204 | "errors": [], |
b5b8bbf5 FG |
2205 | "osd": 1, |
2206 | "primary": true | |
7c673cae FG |
2207 | } |
2208 | ], | |
94b18763 FG |
2209 | "selected_object_info": { |
2210 | "oid": { | |
2211 | "oid": "ROBJ6", | |
2212 | "key": "", | |
2213 | "snapid": -2, | |
2214 | "hash": 390610085, | |
2215 | "max": 0, | |
2216 | "pool": 3, | |
2217 | "namespace": "" | |
2218 | }, | |
2219 | "version": "51'54", | |
2220 | "prior_version": "31'18", | |
2221 | "last_reqid": "osd.1.0:53", | |
2222 | "user_version": 18, | |
2223 | "size": 7, | |
2224 | "mtime": "2018-04-05 14:33:23.289188", | |
2225 | "local_mtime": "2018-04-05 14:33:23.290130", | |
2226 | "lost": 0, | |
2227 | "flags": [ | |
2228 | "dirty", | |
2229 | "omap", | |
2230 | "data_digest", | |
2231 | "omap_digest" | |
2232 | ], | |
2233 | "truncate_seq": 0, | |
2234 | "truncate_size": 0, | |
2235 | "data_digest": "0x2ddbf8f5", | |
2236 | "omap_digest": "0x179c919f", | |
2237 | "expected_object_size": 0, | |
2238 | "expected_write_size": 0, | |
2239 | "alloc_hint_flags": 0, | |
2240 | "manifest": { | |
2241 | "type": 0 | |
2242 | }, | |
2243 | "watchers": {} | |
2244 | }, | |
7c673cae | 2245 | "union_shard_errors": [ |
94b18763 | 2246 | "omap_digest_mismatch_info" |
7c673cae FG |
2247 | ], |
2248 | "errors": [ | |
2249 | "omap_digest_mismatch" | |
2250 | ], | |
2251 | "object": { | |
2252 | "version": 18, | |
2253 | "snap": "head", | |
2254 | "locator": "", | |
2255 | "nspace": "", | |
2256 | "name": "ROBJ6" | |
2257 | } | |
2258 | }, | |
2259 | { | |
2260 | "shards": [ | |
2261 | { | |
2262 | "data_digest": "0x2ddbf8f5", | |
2263 | "omap_digest": "0xefced57a", | |
2264 | "size": 7, | |
2265 | "errors": [], | |
b5b8bbf5 FG |
2266 | "osd": 0, |
2267 | "primary": false | |
7c673cae FG |
2268 | }, |
2269 | { | |
2270 | "data_digest": "0x2ddbf8f5", | |
2271 | "omap_digest": "0x6a73cc07", | |
2272 | "size": 7, | |
2273 | "errors": [ | |
94b18763 | 2274 | "omap_digest_mismatch_info" |
7c673cae | 2275 | ], |
b5b8bbf5 FG |
2276 | "osd": 1, |
2277 | "primary": true | |
7c673cae FG |
2278 | } |
2279 | ], | |
94b18763 FG |
2280 | "selected_object_info": { |
2281 | "oid": { | |
2282 | "oid": "ROBJ7", | |
2283 | "key": "", | |
2284 | "snapid": -2, | |
2285 | "hash": 3529485009, | |
2286 | "max": 0, | |
2287 | "pool": 3, | |
2288 | "namespace": "" | |
2289 | }, | |
2290 | "version": "51'53", | |
2291 | "prior_version": "33'21", | |
2292 | "last_reqid": "osd.1.0:52", | |
2293 | "user_version": 21, | |
2294 | "size": 7, | |
2295 | "mtime": "2018-04-05 14:33:23.979658", | |
2296 | "local_mtime": "2018-04-05 14:33:23.980731", | |
2297 | "lost": 0, | |
2298 | "flags": [ | |
2299 | "dirty", | |
2300 | "omap", | |
2301 | "data_digest", | |
2302 | "omap_digest" | |
2303 | ], | |
2304 | "truncate_seq": 0, | |
2305 | "truncate_size": 0, | |
2306 | "data_digest": "0x2ddbf8f5", | |
2307 | "omap_digest": "0xefced57a", | |
2308 | "expected_object_size": 0, | |
2309 | "expected_write_size": 0, | |
2310 | "alloc_hint_flags": 0, | |
2311 | "manifest": { | |
2312 | "type": 0 | |
2313 | }, | |
2314 | "watchers": {} | |
2315 | }, | |
7c673cae | 2316 | "union_shard_errors": [ |
94b18763 | 2317 | "omap_digest_mismatch_info" |
7c673cae FG |
2318 | ], |
2319 | "errors": [ | |
2320 | "omap_digest_mismatch" | |
2321 | ], | |
2322 | "object": { | |
2323 | "version": 21, | |
2324 | "snap": "head", | |
2325 | "locator": "", | |
2326 | "nspace": "", | |
2327 | "name": "ROBJ7" | |
2328 | } | |
2329 | }, | |
2330 | { | |
2331 | "shards": [ | |
2332 | { | |
2333 | "attrs": [ | |
7c673cae FG |
2334 | { |
2335 | "Base64": false, | |
2336 | "value": "bad-val", | |
94b18763 | 2337 | "name": "key1-ROBJ8" |
7c673cae FG |
2338 | }, |
2339 | { | |
2340 | "Base64": false, | |
b32b8144 | 2341 | "value": "val2-ROBJ8", |
94b18763 | 2342 | "name": "key2-ROBJ8" |
7c673cae FG |
2343 | } |
2344 | ], | |
2345 | "data_digest": "0x2ddbf8f5", | |
2346 | "omap_digest": "0xd6be81dc", | |
2347 | "size": 7, | |
2348 | "errors": [], | |
b5b8bbf5 FG |
2349 | "osd": 0, |
2350 | "primary": false | |
7c673cae FG |
2351 | }, |
2352 | { | |
2353 | "attrs": [ | |
7c673cae FG |
2354 | { |
2355 | "Base64": false, | |
2356 | "value": "val1-ROBJ8", | |
94b18763 | 2357 | "name": "key1-ROBJ8" |
7c673cae FG |
2358 | }, |
2359 | { | |
2360 | "Base64": false, | |
b32b8144 | 2361 | "value": "val3-ROBJ8", |
94b18763 | 2362 | "name": "key3-ROBJ8" |
7c673cae FG |
2363 | } |
2364 | ], | |
2365 | "data_digest": "0x2ddbf8f5", | |
2366 | "omap_digest": "0xd6be81dc", | |
2367 | "size": 7, | |
2368 | "errors": [], | |
b5b8bbf5 FG |
2369 | "osd": 1, |
2370 | "primary": true | |
7c673cae FG |
2371 | } |
2372 | ], | |
94b18763 FG |
2373 | "selected_object_info": { |
2374 | "oid": { | |
2375 | "oid": "ROBJ8", | |
2376 | "key": "", | |
2377 | "snapid": -2, | |
2378 | "hash": 2359695969, | |
2379 | "max": 0, | |
2380 | "pool": 3, | |
2381 | "namespace": "" | |
2382 | }, | |
2383 | "version": "79'66", | |
2384 | "prior_version": "79'65", | |
2385 | "last_reqid": "client.4554.0:1", | |
2386 | "user_version": 66, | |
2387 | "size": 7, | |
2388 | "mtime": "2018-04-05 14:34:05.598688", | |
2389 | "local_mtime": "2018-04-05 14:34:05.599698", | |
2390 | "lost": 0, | |
2391 | "flags": [ | |
2392 | "dirty", | |
2393 | "omap", | |
2394 | "data_digest", | |
2395 | "omap_digest" | |
2396 | ], | |
2397 | "truncate_seq": 0, | |
2398 | "truncate_size": 0, | |
2399 | "data_digest": "0x2ddbf8f5", | |
2400 | "omap_digest": "0xd6be81dc", | |
2401 | "expected_object_size": 0, | |
2402 | "expected_write_size": 0, | |
2403 | "alloc_hint_flags": 0, | |
2404 | "manifest": { | |
2405 | "type": 0 | |
2406 | }, | |
2407 | "watchers": {} | |
2408 | }, | |
7c673cae FG |
2409 | "union_shard_errors": [], |
2410 | "errors": [ | |
2411 | "attr_value_mismatch", | |
2412 | "attr_name_mismatch" | |
2413 | ], | |
2414 | "object": { | |
b5b8bbf5 | 2415 | "version": 66, |
7c673cae FG |
2416 | "snap": "head", |
2417 | "locator": "", | |
2418 | "nspace": "", | |
2419 | "name": "ROBJ8" | |
2420 | } | |
2421 | }, | |
2422 | { | |
2423 | "shards": [ | |
2424 | { | |
94b18763 FG |
2425 | "object_info": { |
2426 | "oid": { | |
2427 | "oid": "ROBJ9", | |
2428 | "key": "", | |
2429 | "snapid": -2, | |
2430 | "hash": 537189375, | |
2431 | "max": 0, | |
2432 | "pool": 3, | |
2433 | "namespace": "" | |
2434 | }, | |
2435 | "version": "51'64", | |
2436 | "prior_version": "37'27", | |
2437 | "last_reqid": "osd.1.0:63", | |
2438 | "user_version": 27, | |
2439 | "size": 7, | |
2440 | "mtime": "2018-04-05 14:33:25.352485", | |
2441 | "local_mtime": "2018-04-05 14:33:25.353746", | |
2442 | "lost": 0, | |
2443 | "flags": [ | |
2444 | "dirty", | |
2445 | "omap", | |
2446 | "data_digest", | |
2447 | "omap_digest" | |
2448 | ], | |
2449 | "truncate_seq": 0, | |
2450 | "truncate_size": 0, | |
2451 | "data_digest": "0x2ddbf8f5", | |
2452 | "omap_digest": "0x2eecc539", | |
2453 | "expected_object_size": 0, | |
2454 | "expected_write_size": 0, | |
2455 | "alloc_hint_flags": 0, | |
2456 | "manifest": { | |
2457 | "type": 0 | |
2458 | }, | |
2459 | "watchers": {} | |
2460 | }, | |
7c673cae FG |
2461 | "data_digest": "0x1f26fb26", |
2462 | "omap_digest": "0x2eecc539", | |
2463 | "size": 3, | |
b5b8bbf5 | 2464 | "errors": [ |
94b18763 | 2465 | "obj_size_info_mismatch" |
b5b8bbf5 FG |
2466 | ], |
2467 | "osd": 0, | |
2468 | "primary": false | |
7c673cae FG |
2469 | }, |
2470 | { | |
94b18763 FG |
2471 | "object_info": { |
2472 | "oid": { | |
2473 | "oid": "ROBJ9", | |
2474 | "key": "", | |
2475 | "snapid": -2, | |
2476 | "hash": 537189375, | |
2477 | "max": 0, | |
2478 | "pool": 3, | |
2479 | "namespace": "" | |
2480 | }, | |
2481 | "version": "119'68", | |
2482 | "prior_version": "51'64", | |
2483 | "last_reqid": "client.4834.0:1", | |
2484 | "user_version": 68, | |
2485 | "size": 3, | |
2486 | "mtime": "2018-04-05 14:35:01.500659", | |
2487 | "local_mtime": "2018-04-05 14:35:01.502117", | |
2488 | "lost": 0, | |
2489 | "flags": [ | |
2490 | "dirty", | |
2491 | "omap", | |
2492 | "data_digest", | |
2493 | "omap_digest" | |
2494 | ], | |
2495 | "truncate_seq": 0, | |
2496 | "truncate_size": 0, | |
2497 | "data_digest": "0x1f26fb26", | |
2498 | "omap_digest": "0x2eecc539", | |
2499 | "expected_object_size": 0, | |
2500 | "expected_write_size": 0, | |
2501 | "alloc_hint_flags": 0, | |
2502 | "manifest": { | |
2503 | "type": 0 | |
2504 | }, | |
2505 | "watchers": {} | |
2506 | }, | |
7c673cae FG |
2507 | "data_digest": "0x1f26fb26", |
2508 | "omap_digest": "0x2eecc539", | |
2509 | "size": 3, | |
2510 | "errors": [], | |
b5b8bbf5 FG |
2511 | "osd": 1, |
2512 | "primary": true | |
7c673cae FG |
2513 | } |
2514 | ], | |
94b18763 FG |
2515 | "selected_object_info": { |
2516 | "oid": { | |
2517 | "oid": "ROBJ9", | |
2518 | "key": "", | |
2519 | "snapid": -2, | |
2520 | "hash": 537189375, | |
2521 | "max": 0, | |
2522 | "pool": 3, | |
2523 | "namespace": "" | |
2524 | }, | |
2525 | "version": "119'68", | |
2526 | "prior_version": "51'64", | |
2527 | "last_reqid": "client.4834.0:1", | |
2528 | "user_version": 68, | |
2529 | "size": 3, | |
2530 | "mtime": "2018-04-05 14:35:01.500659", | |
2531 | "local_mtime": "2018-04-05 14:35:01.502117", | |
2532 | "lost": 0, | |
2533 | "flags": [ | |
2534 | "dirty", | |
2535 | "omap", | |
2536 | "data_digest", | |
2537 | "omap_digest" | |
2538 | ], | |
2539 | "truncate_seq": 0, | |
2540 | "truncate_size": 0, | |
2541 | "data_digest": "0x1f26fb26", | |
2542 | "omap_digest": "0x2eecc539", | |
2543 | "expected_object_size": 0, | |
2544 | "expected_write_size": 0, | |
2545 | "alloc_hint_flags": 0, | |
2546 | "manifest": { | |
2547 | "type": 0 | |
2548 | }, | |
2549 | "watchers": {} | |
2550 | }, | |
b5b8bbf5 | 2551 | "union_shard_errors": [ |
94b18763 | 2552 | "obj_size_info_mismatch" |
b5b8bbf5 | 2553 | ], |
7c673cae | 2554 | "errors": [ |
b5b8bbf5 | 2555 | "object_info_inconsistency" |
7c673cae FG |
2556 | ], |
2557 | "object": { | |
b5b8bbf5 | 2558 | "version": 68, |
7c673cae FG |
2559 | "snap": "head", |
2560 | "locator": "", | |
2561 | "nspace": "", | |
2562 | "name": "ROBJ9" | |
2563 | } | |
2564 | } | |
2565 | ], | |
2566 | "epoch": 0 | |
2567 | } | |
2568 | EOF | |
2569 | ||
94b18763 | 2570 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
31f18b77 | 2571 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
2572 | if test $getjson = "yes" |
2573 | then | |
2574 | jq '.' $dir/json > save2.json | |
2575 | fi | |
2576 | ||
94b18763 | 2577 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
2578 | then |
2579 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
2580 | fi | |
2581 | ||
2582 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
2583 | teardown $dir || return 1 | |
2584 | } | |
2585 | ||
2586 | ||
2587 | # | |
2588 | # Test scrub errors for an erasure coded pool | |
2589 | # | |
2590 | function corrupt_scrub_erasure() { | |
2591 | local dir=$1 | |
2592 | local allow_overwrites=$2 | |
2593 | local poolname=ecpool | |
94b18763 | 2594 | local total_objs=7 |
7c673cae FG |
2595 | |
2596 | setup $dir || return 1 | |
2597 | run_mon $dir a || return 1 | |
2598 | run_mgr $dir x || return 1 | |
2599 | for id in $(seq 0 2) ; do | |
2600 | if [ "$allow_overwrites" = "true" ]; then | |
2601 | run_osd_bluestore $dir $id || return 1 | |
2602 | else | |
2603 | run_osd $dir $id || return 1 | |
2604 | fi | |
2605 | done | |
c07f9fc5 | 2606 | create_rbd_pool || return 1 |
b5b8bbf5 | 2607 | create_pool foo 1 |
7c673cae FG |
2608 | |
2609 | create_ec_pool $poolname $allow_overwrites k=2 m=1 stripe_unit=2K --force || return 1 | |
c07f9fc5 | 2610 | wait_for_clean || return 1 |
7c673cae FG |
2611 | |
2612 | for i in $(seq 1 $total_objs) ; do | |
2613 | objname=EOBJ${i} | |
2614 | add_something $dir $poolname $objname || return 1 | |
2615 | ||
2616 | local osd=$(expr $i % 2) | |
2617 | ||
2618 | case $i in | |
2619 | 1) | |
2620 | # Size (deep scrub data_digest too) | |
2621 | local payload=UVWXYZZZ | |
2622 | echo $payload > $dir/CORRUPT | |
2623 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
2624 | ;; | |
2625 | ||
2626 | 2) | |
2627 | # Corrupt EC shard | |
2628 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=1 | |
2629 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
2630 | ;; | |
2631 | ||
2632 | 3) | |
2633 | # missing | |
2634 | objectstore_tool $dir $osd $objname remove || return 1 | |
2635 | ;; | |
2636 | ||
2637 | 4) | |
2638 | rados --pool $poolname setxattr $objname key1-$objname val1-$objname || return 1 | |
2639 | rados --pool $poolname setxattr $objname key2-$objname val2-$objname || return 1 | |
2640 | ||
2641 | # Break xattrs | |
2642 | echo -n bad-val > $dir/bad-val | |
2643 | objectstore_tool $dir $osd $objname set-attr _key1-$objname $dir/bad-val || return 1 | |
2644 | objectstore_tool $dir $osd $objname rm-attr _key2-$objname || return 1 | |
2645 | echo -n val3-$objname > $dir/newval | |
2646 | objectstore_tool $dir $osd $objname set-attr _key3-$objname $dir/newval || return 1 | |
2647 | rm $dir/bad-val $dir/newval | |
2648 | ;; | |
2649 | ||
2650 | 5) | |
2651 | # Corrupt EC shard | |
2652 | dd if=/dev/urandom of=$dir/CORRUPT bs=2048 count=2 | |
2653 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
2654 | ;; | |
2655 | ||
94b18763 FG |
2656 | 6) |
2657 | objectstore_tool $dir 0 $objname rm-attr hinfo_key || return 1 | |
2658 | echo -n bad-val > $dir/bad-val | |
2659 | objectstore_tool $dir 1 $objname set-attr hinfo_key $dir/bad-val || return 1 | |
2660 | ;; | |
2661 | ||
2662 | 7) | |
2663 | local payload=MAKETHISDIFFERENTFROMOTHEROBJECTS | |
2664 | echo $payload > $dir/DIFFERENT | |
2665 | rados --pool $poolname put $objname $dir/DIFFERENT || return 1 | |
2666 | ||
2667 | # Get hinfo_key from EOBJ1 | |
2668 | objectstore_tool $dir 0 EOBJ1 get-attr hinfo_key > $dir/hinfo | |
2669 | objectstore_tool $dir 0 $objname set-attr hinfo_key $dir/hinfo || return 1 | |
2670 | rm -f $dir/hinfo | |
2671 | ;; | |
2672 | ||
7c673cae FG |
2673 | esac |
2674 | done | |
2675 | ||
2676 | local pg=$(get_pg $poolname EOBJ0) | |
2677 | ||
2678 | pg_scrub $pg | |
2679 | ||
2680 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
2681 | # Check pg count | |
2682 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
2683 | # Check pgid | |
2684 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
2685 | ||
2686 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
2687 | # Get epoch for repair-get requests | |
2688 | epoch=$(jq .epoch $dir/json) | |
2689 | ||
94b18763 | 2690 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
2691 | { |
2692 | "inconsistents": [ | |
2693 | { | |
2694 | "shards": [ | |
2695 | { | |
2696 | "size": 2048, | |
2697 | "errors": [], | |
2698 | "shard": 2, | |
b5b8bbf5 FG |
2699 | "osd": 0, |
2700 | "primary": false | |
7c673cae FG |
2701 | }, |
2702 | { | |
94b18763 FG |
2703 | "object_info": { |
2704 | "oid": { | |
2705 | "oid": "EOBJ1", | |
2706 | "key": "", | |
2707 | "snapid": -2, | |
2708 | "hash": 560836233, | |
2709 | "max": 0, | |
2710 | "pool": 3, | |
2711 | "namespace": "" | |
2712 | }, | |
2713 | "version": "27'1", | |
2714 | "prior_version": "0'0", | |
2715 | "last_reqid": "client.4184.0:1", | |
2716 | "user_version": 1, | |
2717 | "size": 7, | |
2718 | "mtime": "", | |
2719 | "local_mtime": "", | |
2720 | "lost": 0, | |
2721 | "flags": [ | |
2722 | "dirty", | |
28e407b8 | 2723 | "data_digest" |
94b18763 FG |
2724 | ], |
2725 | "truncate_seq": 0, | |
2726 | "truncate_size": 0, | |
2727 | "data_digest": "0x2ddbf8f5", | |
2728 | "omap_digest": "0xffffffff", | |
2729 | "expected_object_size": 0, | |
2730 | "expected_write_size": 0, | |
2731 | "alloc_hint_flags": 0, | |
2732 | "manifest": { | |
2733 | "type": 0 | |
2734 | }, | |
2735 | "watchers": {} | |
2736 | }, | |
7c673cae | 2737 | "size": 9, |
224ce89b | 2738 | "shard": 0, |
7c673cae | 2739 | "errors": [ |
94b18763 FG |
2740 | "size_mismatch_info", |
2741 | "obj_size_info_mismatch" | |
7c673cae | 2742 | ], |
b5b8bbf5 FG |
2743 | "osd": 1, |
2744 | "primary": true | |
7c673cae FG |
2745 | }, |
2746 | { | |
2747 | "size": 2048, | |
224ce89b | 2748 | "shard": 1, |
7c673cae | 2749 | "errors": [], |
b5b8bbf5 FG |
2750 | "osd": 2, |
2751 | "primary": false | |
7c673cae FG |
2752 | } |
2753 | ], | |
94b18763 FG |
2754 | "selected_object_info": { |
2755 | "oid": { | |
2756 | "oid": "EOBJ1", | |
2757 | "key": "", | |
2758 | "snapid": -2, | |
2759 | "hash": 560836233, | |
2760 | "max": 0, | |
2761 | "pool": 3, | |
2762 | "namespace": "" | |
2763 | }, | |
2764 | "version": "27'1", | |
2765 | "prior_version": "0'0", | |
2766 | "last_reqid": "client.4184.0:1", | |
2767 | "user_version": 1, | |
2768 | "size": 7, | |
2769 | "mtime": "", | |
2770 | "local_mtime": "", | |
2771 | "lost": 0, | |
2772 | "flags": [ | |
2773 | "dirty", | |
28e407b8 | 2774 | "data_digest" |
94b18763 FG |
2775 | ], |
2776 | "truncate_seq": 0, | |
2777 | "truncate_size": 0, | |
2778 | "data_digest": "0x2ddbf8f5", | |
2779 | "omap_digest": "0xffffffff", | |
2780 | "expected_object_size": 0, | |
2781 | "expected_write_size": 0, | |
2782 | "alloc_hint_flags": 0, | |
2783 | "manifest": { | |
2784 | "type": 0 | |
2785 | }, | |
2786 | "watchers": {} | |
2787 | }, | |
7c673cae | 2788 | "union_shard_errors": [ |
94b18763 FG |
2789 | "size_mismatch_info", |
2790 | "obj_size_info_mismatch" | |
7c673cae FG |
2791 | ], |
2792 | "errors": [ | |
2793 | "size_mismatch" | |
2794 | ], | |
2795 | "object": { | |
2796 | "version": 1, | |
2797 | "snap": "head", | |
2798 | "locator": "", | |
2799 | "nspace": "", | |
2800 | "name": "EOBJ1" | |
2801 | } | |
2802 | }, | |
2803 | { | |
2804 | "shards": [ | |
2805 | { | |
2806 | "size": 2048, | |
2807 | "errors": [], | |
2808 | "shard": 2, | |
b5b8bbf5 FG |
2809 | "osd": 0, |
2810 | "primary": false | |
7c673cae FG |
2811 | }, |
2812 | { | |
224ce89b | 2813 | "shard": 0, |
7c673cae FG |
2814 | "errors": [ |
2815 | "missing" | |
2816 | ], | |
b5b8bbf5 FG |
2817 | "osd": 1, |
2818 | "primary": true | |
7c673cae FG |
2819 | }, |
2820 | { | |
2821 | "size": 2048, | |
224ce89b | 2822 | "shard": 1, |
7c673cae | 2823 | "errors": [], |
b5b8bbf5 FG |
2824 | "osd": 2, |
2825 | "primary": false | |
7c673cae FG |
2826 | } |
2827 | ], | |
94b18763 FG |
2828 | "selected_object_info": { |
2829 | "oid": { | |
2830 | "oid": "EOBJ3", | |
2831 | "key": "", | |
2832 | "snapid": -2, | |
2833 | "hash": 3125668237, | |
2834 | "max": 0, | |
2835 | "pool": 3, | |
2836 | "namespace": "" | |
2837 | }, | |
2838 | "version": "39'3", | |
2839 | "prior_version": "0'0", | |
2840 | "last_reqid": "client.4252.0:1", | |
2841 | "user_version": 3, | |
2842 | "size": 7, | |
2843 | "mtime": "", | |
2844 | "local_mtime": "", | |
2845 | "lost": 0, | |
2846 | "flags": [ | |
2847 | "dirty", | |
28e407b8 | 2848 | "data_digest" |
94b18763 FG |
2849 | ], |
2850 | "truncate_seq": 0, | |
2851 | "truncate_size": 0, | |
2852 | "data_digest": "0x2ddbf8f5", | |
2853 | "omap_digest": "0xffffffff", | |
2854 | "expected_object_size": 0, | |
2855 | "expected_write_size": 0, | |
2856 | "alloc_hint_flags": 0, | |
2857 | "manifest": { | |
2858 | "type": 0 | |
2859 | }, | |
2860 | "watchers": {} | |
2861 | }, | |
7c673cae FG |
2862 | "union_shard_errors": [ |
2863 | "missing" | |
2864 | ], | |
2865 | "errors": [], | |
2866 | "object": { | |
2867 | "version": 3, | |
2868 | "snap": "head", | |
2869 | "locator": "", | |
2870 | "nspace": "", | |
2871 | "name": "EOBJ3" | |
2872 | } | |
2873 | }, | |
2874 | { | |
2875 | "shards": [ | |
2876 | { | |
2877 | "attrs": [ | |
7c673cae FG |
2878 | { |
2879 | "Base64": false, | |
2880 | "value": "bad-val", | |
94b18763 | 2881 | "name": "key1-EOBJ4" |
7c673cae FG |
2882 | }, |
2883 | { | |
2884 | "Base64": false, | |
b32b8144 | 2885 | "value": "val2-EOBJ4", |
94b18763 | 2886 | "name": "key2-EOBJ4" |
7c673cae FG |
2887 | } |
2888 | ], | |
2889 | "size": 2048, | |
2890 | "errors": [], | |
2891 | "shard": 2, | |
b5b8bbf5 FG |
2892 | "osd": 0, |
2893 | "primary": false | |
7c673cae FG |
2894 | }, |
2895 | { | |
224ce89b | 2896 | "osd": 1, |
b5b8bbf5 | 2897 | "primary": true, |
224ce89b WB |
2898 | "shard": 0, |
2899 | "errors": [], | |
2900 | "size": 2048, | |
7c673cae | 2901 | "attrs": [ |
7c673cae FG |
2902 | { |
2903 | "Base64": false, | |
2904 | "value": "val1-EOBJ4", | |
94b18763 | 2905 | "name": "key1-EOBJ4" |
7c673cae FG |
2906 | }, |
2907 | { | |
2908 | "Base64": false, | |
2909 | "value": "val2-EOBJ4", | |
94b18763 | 2910 | "name": "key2-EOBJ4" |
7c673cae | 2911 | } |
224ce89b | 2912 | ] |
7c673cae FG |
2913 | }, |
2914 | { | |
224ce89b | 2915 | "osd": 2, |
b5b8bbf5 | 2916 | "primary": false, |
224ce89b WB |
2917 | "shard": 1, |
2918 | "errors": [], | |
2919 | "size": 2048, | |
7c673cae | 2920 | "attrs": [ |
7c673cae FG |
2921 | { |
2922 | "Base64": false, | |
2923 | "value": "val1-EOBJ4", | |
94b18763 | 2924 | "name": "key1-EOBJ4" |
7c673cae FG |
2925 | }, |
2926 | { | |
2927 | "Base64": false, | |
b32b8144 | 2928 | "value": "val3-EOBJ4", |
94b18763 | 2929 | "name": "key3-EOBJ4" |
7c673cae | 2930 | } |
224ce89b | 2931 | ] |
7c673cae FG |
2932 | } |
2933 | ], | |
94b18763 FG |
2934 | "selected_object_info": { |
2935 | "oid": { | |
2936 | "oid": "EOBJ4", | |
2937 | "key": "", | |
2938 | "snapid": -2, | |
2939 | "hash": 1618759290, | |
2940 | "max": 0, | |
2941 | "pool": 3, | |
2942 | "namespace": "" | |
2943 | }, | |
2944 | "version": "45'6", | |
2945 | "prior_version": "45'5", | |
2946 | "last_reqid": "client.4294.0:1", | |
2947 | "user_version": 6, | |
2948 | "size": 7, | |
2949 | "mtime": "", | |
2950 | "local_mtime": "", | |
2951 | "lost": 0, | |
2952 | "flags": [ | |
2953 | "dirty", | |
28e407b8 | 2954 | "data_digest" |
94b18763 FG |
2955 | ], |
2956 | "truncate_seq": 0, | |
2957 | "truncate_size": 0, | |
2958 | "data_digest": "0x2ddbf8f5", | |
2959 | "omap_digest": "0xffffffff", | |
2960 | "expected_object_size": 0, | |
2961 | "expected_write_size": 0, | |
2962 | "alloc_hint_flags": 0, | |
2963 | "manifest": { | |
2964 | "type": 0 | |
2965 | }, | |
2966 | "watchers": {} | |
2967 | }, | |
7c673cae FG |
2968 | "union_shard_errors": [], |
2969 | "errors": [ | |
2970 | "attr_value_mismatch", | |
2971 | "attr_name_mismatch" | |
2972 | ], | |
2973 | "object": { | |
2974 | "version": 6, | |
2975 | "snap": "head", | |
2976 | "locator": "", | |
2977 | "nspace": "", | |
2978 | "name": "EOBJ4" | |
2979 | } | |
2980 | }, | |
2981 | { | |
2982 | "shards": [ | |
2983 | { | |
2984 | "size": 2048, | |
2985 | "errors": [], | |
2986 | "shard": 2, | |
b5b8bbf5 FG |
2987 | "osd": 0, |
2988 | "primary": false | |
7c673cae FG |
2989 | }, |
2990 | { | |
94b18763 FG |
2991 | "object_info": { |
2992 | "oid": { | |
2993 | "oid": "EOBJ5", | |
2994 | "key": "", | |
2995 | "snapid": -2, | |
2996 | "hash": 2918945441, | |
2997 | "max": 0, | |
2998 | "pool": 3, | |
2999 | "namespace": "" | |
3000 | }, | |
3001 | "version": "59'7", | |
3002 | "prior_version": "0'0", | |
3003 | "last_reqid": "client.4382.0:1", | |
3004 | "user_version": 7, | |
3005 | "size": 7, | |
3006 | "mtime": "", | |
3007 | "local_mtime": "", | |
3008 | "lost": 0, | |
3009 | "flags": [ | |
3010 | "dirty", | |
28e407b8 | 3011 | "data_digest" |
94b18763 FG |
3012 | ], |
3013 | "truncate_seq": 0, | |
3014 | "truncate_size": 0, | |
3015 | "data_digest": "0x2ddbf8f5", | |
3016 | "omap_digest": "0xffffffff", | |
3017 | "expected_object_size": 0, | |
3018 | "expected_write_size": 0, | |
3019 | "alloc_hint_flags": 0, | |
3020 | "manifest": { | |
3021 | "type": 0 | |
3022 | }, | |
3023 | "watchers": {} | |
3024 | }, | |
7c673cae | 3025 | "size": 4096, |
224ce89b | 3026 | "shard": 0, |
7c673cae | 3027 | "errors": [ |
94b18763 FG |
3028 | "size_mismatch_info", |
3029 | "obj_size_info_mismatch" | |
7c673cae | 3030 | ], |
b5b8bbf5 FG |
3031 | "osd": 1, |
3032 | "primary": true | |
7c673cae FG |
3033 | }, |
3034 | { | |
3035 | "size": 2048, | |
224ce89b | 3036 | "shard": 1, |
7c673cae | 3037 | "errors": [], |
b5b8bbf5 FG |
3038 | "osd": 2, |
3039 | "primary": false | |
7c673cae FG |
3040 | } |
3041 | ], | |
94b18763 FG |
3042 | "selected_object_info": { |
3043 | "oid": { | |
3044 | "oid": "EOBJ5", | |
3045 | "key": "", | |
3046 | "snapid": -2, | |
3047 | "hash": 2918945441, | |
3048 | "max": 0, | |
3049 | "pool": 3, | |
3050 | "namespace": "" | |
3051 | }, | |
3052 | "version": "59'7", | |
3053 | "prior_version": "0'0", | |
3054 | "last_reqid": "client.4382.0:1", | |
3055 | "user_version": 7, | |
3056 | "size": 7, | |
3057 | "mtime": "", | |
3058 | "local_mtime": "", | |
3059 | "lost": 0, | |
3060 | "flags": [ | |
3061 | "dirty", | |
28e407b8 | 3062 | "data_digest" |
94b18763 FG |
3063 | ], |
3064 | "truncate_seq": 0, | |
3065 | "truncate_size": 0, | |
3066 | "data_digest": "0x2ddbf8f5", | |
3067 | "omap_digest": "0xffffffff", | |
3068 | "expected_object_size": 0, | |
3069 | "expected_write_size": 0, | |
3070 | "alloc_hint_flags": 0, | |
3071 | "manifest": { | |
3072 | "type": 0 | |
3073 | }, | |
3074 | "watchers": {} | |
3075 | }, | |
7c673cae | 3076 | "union_shard_errors": [ |
94b18763 FG |
3077 | "size_mismatch_info", |
3078 | "obj_size_info_mismatch" | |
7c673cae FG |
3079 | ], |
3080 | "errors": [ | |
3081 | "size_mismatch" | |
3082 | ], | |
3083 | "object": { | |
3084 | "version": 7, | |
3085 | "snap": "head", | |
3086 | "locator": "", | |
3087 | "nspace": "", | |
3088 | "name": "EOBJ5" | |
3089 | } | |
94b18763 FG |
3090 | }, |
3091 | { | |
3092 | "errors": [], | |
3093 | "object": { | |
3094 | "locator": "", | |
3095 | "name": "EOBJ6", | |
3096 | "nspace": "", | |
3097 | "snap": "head", | |
3098 | "version": 8 | |
3099 | }, | |
3100 | "selected_object_info": { | |
3101 | "oid": { | |
3102 | "oid": "EOBJ6", | |
3103 | "key": "", | |
3104 | "snapid": -2, | |
3105 | "hash": 3050890866, | |
3106 | "max": 0, | |
3107 | "pool": 3, | |
3108 | "namespace": "" | |
3109 | }, | |
3110 | "version": "65'8", | |
3111 | "prior_version": "0'0", | |
3112 | "last_reqid": "client.4418.0:1", | |
3113 | "user_version": 8, | |
3114 | "size": 7, | |
3115 | "mtime": "", | |
3116 | "local_mtime": "", | |
3117 | "lost": 0, | |
3118 | "flags": [ | |
3119 | "dirty", | |
28e407b8 | 3120 | "data_digest" |
94b18763 FG |
3121 | ], |
3122 | "truncate_seq": 0, | |
3123 | "truncate_size": 0, | |
3124 | "data_digest": "0x2ddbf8f5", | |
3125 | "omap_digest": "0xffffffff", | |
3126 | "expected_object_size": 0, | |
3127 | "expected_write_size": 0, | |
3128 | "alloc_hint_flags": 0, | |
3129 | "manifest": { | |
3130 | "type": 0 | |
3131 | }, | |
3132 | "watchers": {} | |
3133 | }, | |
3134 | "shards": [ | |
3135 | { | |
3136 | "errors": [ | |
3137 | "hinfo_missing" | |
3138 | ], | |
3139 | "osd": 0, | |
3140 | "primary": false, | |
3141 | "shard": 2, | |
3142 | "size": 2048 | |
3143 | }, | |
3144 | { | |
3145 | "errors": [ | |
3146 | "hinfo_corrupted" | |
3147 | ], | |
3148 | "osd": 1, | |
3149 | "primary": true, | |
3150 | "shard": 0, | |
3151 | "hashinfo": "bad-val", | |
3152 | "size": 2048 | |
3153 | }, | |
3154 | { | |
3155 | "errors": [], | |
3156 | "osd": 2, | |
3157 | "primary": false, | |
3158 | "shard": 1, | |
3159 | "size": 2048, | |
3160 | "hashinfo": { | |
3161 | "cumulative_shard_hashes": [ | |
3162 | { | |
3163 | "hash": 80717615, | |
3164 | "shard": 0 | |
3165 | }, | |
3166 | { | |
3167 | "hash": 1534491824, | |
3168 | "shard": 1 | |
3169 | }, | |
3170 | { | |
3171 | "hash": 80717615, | |
3172 | "shard": 2 | |
3173 | } | |
3174 | ], | |
3175 | "total_chunk_size": 2048 | |
3176 | } | |
3177 | } | |
3178 | ], | |
3179 | "union_shard_errors": [ | |
3180 | "hinfo_missing", | |
3181 | "hinfo_corrupted" | |
3182 | ] | |
3183 | }, | |
3184 | { | |
3185 | "errors": [ | |
3186 | "hinfo_inconsistency" | |
3187 | ], | |
3188 | "object": { | |
3189 | "locator": "", | |
3190 | "name": "EOBJ7", | |
3191 | "nspace": "", | |
3192 | "snap": "head", | |
3193 | "version": 10 | |
3194 | }, | |
3195 | "selected_object_info": { | |
3196 | "oid": { | |
3197 | "oid": "EOBJ7", | |
3198 | "key": "", | |
3199 | "snapid": -2, | |
3200 | "hash": 3258066308, | |
3201 | "max": 0, | |
3202 | "pool": 3, | |
3203 | "namespace": "" | |
3204 | }, | |
3205 | "version": "75'10", | |
3206 | "prior_version": "75'9", | |
3207 | "last_reqid": "client.4482.0:1", | |
3208 | "user_version": 10, | |
3209 | "size": 34, | |
3210 | "mtime": "", | |
3211 | "local_mtime": "", | |
3212 | "lost": 0, | |
3213 | "flags": [ | |
3214 | "dirty", | |
28e407b8 | 3215 | "data_digest" |
94b18763 FG |
3216 | ], |
3217 | "truncate_seq": 0, | |
3218 | "truncate_size": 0, | |
3219 | "data_digest": "0x136e4e27", | |
3220 | "omap_digest": "0xffffffff", | |
3221 | "expected_object_size": 0, | |
3222 | "expected_write_size": 0, | |
3223 | "alloc_hint_flags": 0, | |
3224 | "manifest": { | |
3225 | "type": 0 | |
3226 | }, | |
3227 | "watchers": {} | |
3228 | }, | |
3229 | "shards": [ | |
3230 | { | |
3231 | "hashinfo": { | |
3232 | "cumulative_shard_hashes": [ | |
3233 | { | |
3234 | "hash": 80717615, | |
3235 | "shard": 0 | |
3236 | }, | |
3237 | { | |
3238 | "hash": 1534491824, | |
3239 | "shard": 1 | |
3240 | }, | |
3241 | { | |
3242 | "hash": 80717615, | |
3243 | "shard": 2 | |
3244 | } | |
3245 | ], | |
3246 | "total_chunk_size": 2048 | |
3247 | }, | |
3248 | "errors": [], | |
3249 | "osd": 0, | |
3250 | "primary": false, | |
3251 | "shard": 2, | |
3252 | "size": 2048 | |
3253 | }, | |
3254 | { | |
3255 | "hashinfo": { | |
3256 | "cumulative_shard_hashes": [ | |
3257 | { | |
3258 | "hash": 1534350760, | |
3259 | "shard": 0 | |
3260 | }, | |
3261 | { | |
3262 | "hash": 1534491824, | |
3263 | "shard": 1 | |
3264 | }, | |
3265 | { | |
3266 | "hash": 1534350760, | |
3267 | "shard": 2 | |
3268 | } | |
3269 | ], | |
3270 | "total_chunk_size": 2048 | |
3271 | }, | |
3272 | "errors": [], | |
3273 | "osd": 1, | |
3274 | "primary": true, | |
3275 | "shard": 0, | |
3276 | "size": 2048 | |
3277 | }, | |
3278 | { | |
3279 | "hashinfo": { | |
3280 | "cumulative_shard_hashes": [ | |
3281 | { | |
3282 | "hash": 1534350760, | |
3283 | "shard": 0 | |
3284 | }, | |
3285 | { | |
3286 | "hash": 1534491824, | |
3287 | "shard": 1 | |
3288 | }, | |
3289 | { | |
3290 | "hash": 1534350760, | |
3291 | "shard": 2 | |
3292 | } | |
3293 | ], | |
3294 | "total_chunk_size": 2048 | |
3295 | }, | |
3296 | "errors": [], | |
3297 | "osd": 2, | |
3298 | "primary": false, | |
3299 | "shard": 1, | |
3300 | "size": 2048 | |
3301 | } | |
3302 | ], | |
3303 | "union_shard_errors": [] | |
3304 | } | |
3305 | ], | |
3306 | "epoch": 0 | |
3307 | } | |
3308 | EOF | |
3309 | ||
3310 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson | |
3311 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 | |
3312 | if test $getjson = "yes" | |
3313 | then | |
3314 | jq '.' $dir/json > save3.json | |
3315 | fi | |
3316 | ||
3317 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; | |
3318 | then | |
3319 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
3320 | fi | |
3321 | ||
3322 | pg_deep_scrub $pg | |
3323 | ||
7c673cae FG |
3324 | rados list-inconsistent-pg $poolname > $dir/json || return 1 |
3325 | # Check pg count | |
3326 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
3327 | # Check pgid | |
3328 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
3329 | ||
3330 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
3331 | # Get epoch for repair-get requests | |
3332 | epoch=$(jq .epoch $dir/json) | |
3333 | ||
31f18b77 FG |
3334 | if [ "$allow_overwrites" = "true" ] |
3335 | then | |
94b18763 | 3336 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
31f18b77 FG |
3337 | { |
3338 | "inconsistents": [ | |
3339 | { | |
3340 | "shards": [ | |
3341 | { | |
3342 | "data_digest": "0x00000000", | |
3343 | "omap_digest": "0xffffffff", | |
3344 | "size": 2048, | |
3345 | "errors": [], | |
3346 | "shard": 2, | |
b5b8bbf5 FG |
3347 | "osd": 0, |
3348 | "primary": false | |
31f18b77 FG |
3349 | }, |
3350 | { | |
94b18763 FG |
3351 | "object_info": { |
3352 | "oid": { | |
3353 | "oid": "EOBJ1", | |
3354 | "key": "", | |
3355 | "snapid": -2, | |
3356 | "hash": 560836233, | |
3357 | "max": 0, | |
3358 | "pool": 3, | |
3359 | "namespace": "" | |
3360 | }, | |
3361 | "version": "27'1", | |
3362 | "prior_version": "0'0", | |
3363 | "last_reqid": "client.4184.0:1", | |
3364 | "user_version": 1, | |
3365 | "size": 7, | |
3366 | "mtime": "2018-04-05 14:31:33.837147", | |
3367 | "local_mtime": "2018-04-05 14:31:33.840763", | |
3368 | "lost": 0, | |
3369 | "flags": [ | |
3370 | "dirty", | |
28e407b8 | 3371 | "data_digest" |
94b18763 FG |
3372 | ], |
3373 | "truncate_seq": 0, | |
3374 | "truncate_size": 0, | |
3375 | "data_digest": "0x2ddbf8f5", | |
3376 | "omap_digest": "0xffffffff", | |
3377 | "expected_object_size": 0, | |
3378 | "expected_write_size": 0, | |
3379 | "alloc_hint_flags": 0, | |
3380 | "manifest": { | |
3381 | "type": 0 | |
3382 | }, | |
3383 | "watchers": {} | |
3384 | }, | |
31f18b77 | 3385 | "size": 9, |
224ce89b | 3386 | "shard": 0, |
31f18b77 FG |
3387 | "errors": [ |
3388 | "read_error", | |
94b18763 FG |
3389 | "size_mismatch_info", |
3390 | "obj_size_info_mismatch" | |
31f18b77 | 3391 | ], |
b5b8bbf5 FG |
3392 | "osd": 1, |
3393 | "primary": true | |
31f18b77 FG |
3394 | }, |
3395 | { | |
3396 | "data_digest": "0x00000000", | |
3397 | "omap_digest": "0xffffffff", | |
3398 | "size": 2048, | |
224ce89b | 3399 | "shard": 1, |
31f18b77 | 3400 | "errors": [], |
b5b8bbf5 FG |
3401 | "osd": 2, |
3402 | "primary": false | |
31f18b77 FG |
3403 | } |
3404 | ], | |
94b18763 FG |
3405 | "selected_object_info": { |
3406 | "oid": { | |
3407 | "oid": "EOBJ1", | |
3408 | "key": "", | |
3409 | "snapid": -2, | |
3410 | "hash": 560836233, | |
3411 | "max": 0, | |
3412 | "pool": 3, | |
3413 | "namespace": "" | |
3414 | }, | |
3415 | "version": "27'1", | |
3416 | "prior_version": "0'0", | |
3417 | "last_reqid": "client.4184.0:1", | |
3418 | "user_version": 1, | |
3419 | "size": 7, | |
3420 | "mtime": "2018-04-05 14:31:33.837147", | |
3421 | "local_mtime": "2018-04-05 14:31:33.840763", | |
3422 | "lost": 0, | |
3423 | "flags": [ | |
3424 | "dirty", | |
28e407b8 | 3425 | "data_digest" |
94b18763 FG |
3426 | ], |
3427 | "truncate_seq": 0, | |
3428 | "truncate_size": 0, | |
3429 | "data_digest": "0x2ddbf8f5", | |
3430 | "omap_digest": "0xffffffff", | |
3431 | "expected_object_size": 0, | |
3432 | "expected_write_size": 0, | |
3433 | "alloc_hint_flags": 0, | |
3434 | "manifest": { | |
3435 | "type": 0 | |
3436 | }, | |
3437 | "watchers": {} | |
3438 | }, | |
31f18b77 FG |
3439 | "union_shard_errors": [ |
3440 | "read_error", | |
94b18763 FG |
3441 | "size_mismatch_info", |
3442 | "obj_size_info_mismatch" | |
31f18b77 FG |
3443 | ], |
3444 | "errors": [ | |
3445 | "size_mismatch" | |
3446 | ], | |
3447 | "object": { | |
3448 | "version": 1, | |
3449 | "snap": "head", | |
3450 | "locator": "", | |
3451 | "nspace": "", | |
3452 | "name": "EOBJ1" | |
3453 | } | |
3454 | }, | |
3455 | { | |
3456 | "shards": [ | |
3457 | { | |
3458 | "data_digest": "0x00000000", | |
3459 | "omap_digest": "0xffffffff", | |
3460 | "size": 2048, | |
3461 | "errors": [], | |
3462 | "shard": 2, | |
b5b8bbf5 FG |
3463 | "osd": 0, |
3464 | "primary": false | |
31f18b77 FG |
3465 | }, |
3466 | { | |
224ce89b | 3467 | "shard": 0, |
31f18b77 FG |
3468 | "errors": [ |
3469 | "missing" | |
3470 | ], | |
b5b8bbf5 FG |
3471 | "osd": 1, |
3472 | "primary": true | |
31f18b77 FG |
3473 | }, |
3474 | { | |
3475 | "data_digest": "0x00000000", | |
3476 | "omap_digest": "0xffffffff", | |
3477 | "size": 2048, | |
224ce89b | 3478 | "shard": 1, |
31f18b77 | 3479 | "errors": [], |
b5b8bbf5 FG |
3480 | "osd": 2, |
3481 | "primary": false | |
31f18b77 FG |
3482 | } |
3483 | ], | |
94b18763 FG |
3484 | "selected_object_info": { |
3485 | "oid": { | |
3486 | "oid": "EOBJ3", | |
3487 | "key": "", | |
3488 | "snapid": -2, | |
3489 | "hash": 3125668237, | |
3490 | "max": 0, | |
3491 | "pool": 3, | |
3492 | "namespace": "" | |
3493 | }, | |
3494 | "version": "39'3", | |
3495 | "prior_version": "0'0", | |
3496 | "last_reqid": "client.4252.0:1", | |
3497 | "user_version": 3, | |
3498 | "size": 7, | |
3499 | "mtime": "2018-04-05 14:31:46.841145", | |
3500 | "local_mtime": "2018-04-05 14:31:46.844996", | |
3501 | "lost": 0, | |
3502 | "flags": [ | |
3503 | "dirty", | |
28e407b8 | 3504 | "data_digest" |
94b18763 FG |
3505 | ], |
3506 | "truncate_seq": 0, | |
3507 | "truncate_size": 0, | |
3508 | "data_digest": "0x2ddbf8f5", | |
3509 | "omap_digest": "0xffffffff", | |
3510 | "expected_object_size": 0, | |
3511 | "expected_write_size": 0, | |
3512 | "alloc_hint_flags": 0, | |
3513 | "manifest": { | |
3514 | "type": 0 | |
3515 | }, | |
3516 | "watchers": {} | |
3517 | }, | |
31f18b77 FG |
3518 | "union_shard_errors": [ |
3519 | "missing" | |
3520 | ], | |
3521 | "errors": [], | |
3522 | "object": { | |
3523 | "version": 3, | |
3524 | "snap": "head", | |
3525 | "locator": "", | |
3526 | "nspace": "", | |
3527 | "name": "EOBJ3" | |
3528 | } | |
3529 | }, | |
3530 | { | |
3531 | "shards": [ | |
3532 | { | |
3533 | "attrs": [ | |
31f18b77 FG |
3534 | { |
3535 | "Base64": false, | |
3536 | "value": "bad-val", | |
94b18763 | 3537 | "name": "key1-EOBJ4" |
31f18b77 FG |
3538 | }, |
3539 | { | |
3540 | "Base64": false, | |
b32b8144 | 3541 | "value": "val2-EOBJ4", |
94b18763 | 3542 | "name": "key2-EOBJ4" |
31f18b77 FG |
3543 | } |
3544 | ], | |
3545 | "data_digest": "0x00000000", | |
3546 | "omap_digest": "0xffffffff", | |
3547 | "size": 2048, | |
3548 | "errors": [], | |
3549 | "shard": 2, | |
b5b8bbf5 FG |
3550 | "osd": 0, |
3551 | "primary": false | |
31f18b77 FG |
3552 | }, |
3553 | { | |
3554 | "attrs": [ | |
31f18b77 FG |
3555 | { |
3556 | "Base64": false, | |
3557 | "value": "val1-EOBJ4", | |
94b18763 | 3558 | "name": "key1-EOBJ4" |
31f18b77 FG |
3559 | }, |
3560 | { | |
3561 | "Base64": false, | |
3562 | "value": "val2-EOBJ4", | |
94b18763 | 3563 | "name": "key2-EOBJ4" |
31f18b77 FG |
3564 | } |
3565 | ], | |
3566 | "data_digest": "0x00000000", | |
3567 | "omap_digest": "0xffffffff", | |
3568 | "size": 2048, | |
3569 | "errors": [], | |
224ce89b | 3570 | "shard": 0, |
b5b8bbf5 FG |
3571 | "osd": 1, |
3572 | "primary": true | |
31f18b77 FG |
3573 | }, |
3574 | { | |
3575 | "attrs": [ | |
31f18b77 FG |
3576 | { |
3577 | "Base64": false, | |
3578 | "value": "val1-EOBJ4", | |
94b18763 | 3579 | "name": "key1-EOBJ4" |
31f18b77 FG |
3580 | }, |
3581 | { | |
3582 | "Base64": false, | |
b32b8144 | 3583 | "value": "val3-EOBJ4", |
94b18763 | 3584 | "name": "key3-EOBJ4" |
31f18b77 FG |
3585 | } |
3586 | ], | |
3587 | "data_digest": "0x00000000", | |
3588 | "omap_digest": "0xffffffff", | |
3589 | "size": 2048, | |
3590 | "errors": [], | |
224ce89b | 3591 | "shard": 1, |
b5b8bbf5 FG |
3592 | "osd": 2, |
3593 | "primary": false | |
31f18b77 FG |
3594 | } |
3595 | ], | |
94b18763 FG |
3596 | "selected_object_info": { |
3597 | "oid": { | |
3598 | "oid": "EOBJ4", | |
3599 | "key": "", | |
3600 | "snapid": -2, | |
3601 | "hash": 1618759290, | |
3602 | "max": 0, | |
3603 | "pool": 3, | |
3604 | "namespace": "" | |
3605 | }, | |
3606 | "version": "45'6", | |
3607 | "prior_version": "45'5", | |
3608 | "last_reqid": "client.4294.0:1", | |
3609 | "user_version": 6, | |
3610 | "size": 7, | |
3611 | "mtime": "2018-04-05 14:31:54.663622", | |
3612 | "local_mtime": "2018-04-05 14:31:54.664527", | |
3613 | "lost": 0, | |
3614 | "flags": [ | |
3615 | "dirty", | |
28e407b8 | 3616 | "data_digest" |
94b18763 FG |
3617 | ], |
3618 | "truncate_seq": 0, | |
3619 | "truncate_size": 0, | |
3620 | "data_digest": "0x2ddbf8f5", | |
3621 | "omap_digest": "0xffffffff", | |
3622 | "expected_object_size": 0, | |
3623 | "expected_write_size": 0, | |
3624 | "alloc_hint_flags": 0, | |
3625 | "manifest": { | |
3626 | "type": 0 | |
3627 | }, | |
3628 | "watchers": {} | |
3629 | }, | |
31f18b77 FG |
3630 | "union_shard_errors": [], |
3631 | "errors": [ | |
3632 | "attr_value_mismatch", | |
3633 | "attr_name_mismatch" | |
3634 | ], | |
3635 | "object": { | |
3636 | "version": 6, | |
3637 | "snap": "head", | |
3638 | "locator": "", | |
3639 | "nspace": "", | |
3640 | "name": "EOBJ4" | |
3641 | } | |
3642 | }, | |
3643 | { | |
3644 | "shards": [ | |
3645 | { | |
3646 | "data_digest": "0x00000000", | |
3647 | "omap_digest": "0xffffffff", | |
3648 | "size": 2048, | |
3649 | "errors": [], | |
3650 | "shard": 2, | |
b5b8bbf5 FG |
3651 | "osd": 0, |
3652 | "primary": false | |
31f18b77 FG |
3653 | }, |
3654 | { | |
3655 | "data_digest": "0x00000000", | |
3656 | "omap_digest": "0xffffffff", | |
94b18763 FG |
3657 | "object_info": { |
3658 | "oid": { | |
3659 | "oid": "EOBJ5", | |
3660 | "key": "", | |
3661 | "snapid": -2, | |
3662 | "hash": 2918945441, | |
3663 | "max": 0, | |
3664 | "pool": 3, | |
3665 | "namespace": "" | |
3666 | }, | |
3667 | "version": "59'7", | |
3668 | "prior_version": "0'0", | |
3669 | "last_reqid": "client.4382.0:1", | |
3670 | "user_version": 7, | |
3671 | "size": 7, | |
3672 | "mtime": "2018-04-05 14:32:12.929161", | |
3673 | "local_mtime": "2018-04-05 14:32:12.934707", | |
3674 | "lost": 0, | |
3675 | "flags": [ | |
3676 | "dirty", | |
28e407b8 | 3677 | "data_digest" |
94b18763 FG |
3678 | ], |
3679 | "truncate_seq": 0, | |
3680 | "truncate_size": 0, | |
3681 | "data_digest": "0x2ddbf8f5", | |
3682 | "omap_digest": "0xffffffff", | |
3683 | "expected_object_size": 0, | |
3684 | "expected_write_size": 0, | |
3685 | "alloc_hint_flags": 0, | |
3686 | "manifest": { | |
3687 | "type": 0 | |
3688 | }, | |
3689 | "watchers": {} | |
3690 | }, | |
31f18b77 FG |
3691 | "size": 4096, |
3692 | "errors": [ | |
94b18763 FG |
3693 | "size_mismatch_info", |
3694 | "obj_size_info_mismatch" | |
31f18b77 | 3695 | ], |
224ce89b | 3696 | "shard": 0, |
b5b8bbf5 FG |
3697 | "osd": 1, |
3698 | "primary": true | |
31f18b77 FG |
3699 | }, |
3700 | { | |
3701 | "data_digest": "0x00000000", | |
3702 | "omap_digest": "0xffffffff", | |
3703 | "size": 2048, | |
3704 | "errors": [], | |
224ce89b | 3705 | "shard": 1, |
b5b8bbf5 FG |
3706 | "osd": 2, |
3707 | "primary": false | |
31f18b77 FG |
3708 | } |
3709 | ], | |
94b18763 FG |
3710 | "selected_object_info": { |
3711 | "oid": { | |
3712 | "oid": "EOBJ5", | |
3713 | "key": "", | |
3714 | "snapid": -2, | |
3715 | "hash": 2918945441, | |
3716 | "max": 0, | |
3717 | "pool": 3, | |
3718 | "namespace": "" | |
3719 | }, | |
3720 | "version": "59'7", | |
3721 | "prior_version": "0'0", | |
3722 | "last_reqid": "client.4382.0:1", | |
3723 | "user_version": 7, | |
3724 | "size": 7, | |
3725 | "mtime": "2018-04-05 14:32:12.929161", | |
3726 | "local_mtime": "2018-04-05 14:32:12.934707", | |
3727 | "lost": 0, | |
3728 | "flags": [ | |
3729 | "dirty", | |
28e407b8 | 3730 | "data_digest" |
94b18763 FG |
3731 | ], |
3732 | "truncate_seq": 0, | |
3733 | "truncate_size": 0, | |
3734 | "data_digest": "0x2ddbf8f5", | |
3735 | "omap_digest": "0xffffffff", | |
3736 | "expected_object_size": 0, | |
3737 | "expected_write_size": 0, | |
3738 | "alloc_hint_flags": 0, | |
3739 | "manifest": { | |
3740 | "type": 0 | |
3741 | }, | |
3742 | "watchers": {} | |
3743 | }, | |
31f18b77 | 3744 | "union_shard_errors": [ |
94b18763 FG |
3745 | "size_mismatch_info", |
3746 | "obj_size_info_mismatch" | |
31f18b77 FG |
3747 | ], |
3748 | "errors": [ | |
3749 | "size_mismatch" | |
3750 | ], | |
3751 | "object": { | |
3752 | "version": 7, | |
3753 | "snap": "head", | |
3754 | "locator": "", | |
3755 | "nspace": "", | |
3756 | "name": "EOBJ5" | |
3757 | } | |
94b18763 FG |
3758 | }, |
3759 | { | |
3760 | "object": { | |
3761 | "name": "EOBJ6", | |
3762 | "nspace": "", | |
3763 | "locator": "", | |
3764 | "snap": "head", | |
3765 | "version": 8 | |
3766 | }, | |
3767 | "errors": [], | |
3768 | "union_shard_errors": [ | |
3769 | "read_error", | |
3770 | "hinfo_missing", | |
3771 | "hinfo_corrupted" | |
3772 | ], | |
3773 | "selected_object_info": { | |
3774 | "oid": { | |
3775 | "oid": "EOBJ6", | |
3776 | "key": "", | |
3777 | "snapid": -2, | |
3778 | "hash": 3050890866, | |
3779 | "max": 0, | |
3780 | "pool": 3, | |
3781 | "namespace": "" | |
3782 | }, | |
3783 | "version": "65'8", | |
3784 | "prior_version": "0'0", | |
3785 | "last_reqid": "client.4418.0:1", | |
3786 | "user_version": 8, | |
3787 | "size": 7, | |
3788 | "mtime": "2018-04-05 14:32:20.634116", | |
3789 | "local_mtime": "2018-04-05 14:32:20.637999", | |
3790 | "lost": 0, | |
3791 | "flags": [ | |
3792 | "dirty", | |
28e407b8 | 3793 | "data_digest" |
94b18763 FG |
3794 | ], |
3795 | "truncate_seq": 0, | |
3796 | "truncate_size": 0, | |
3797 | "data_digest": "0x2ddbf8f5", | |
3798 | "omap_digest": "0xffffffff", | |
3799 | "expected_object_size": 0, | |
3800 | "expected_write_size": 0, | |
3801 | "alloc_hint_flags": 0, | |
3802 | "manifest": { | |
3803 | "type": 0 | |
3804 | }, | |
3805 | "watchers": {} | |
3806 | }, | |
3807 | "shards": [ | |
3808 | { | |
3809 | "osd": 0, | |
3810 | "primary": false, | |
3811 | "shard": 2, | |
3812 | "errors": [ | |
3813 | "read_error", | |
3814 | "hinfo_missing" | |
3815 | ], | |
3816 | "size": 2048 | |
3817 | }, | |
3818 | { | |
3819 | "osd": 1, | |
3820 | "primary": true, | |
3821 | "shard": 0, | |
3822 | "errors": [ | |
3823 | "read_error", | |
3824 | "hinfo_corrupted" | |
3825 | ], | |
3826 | "size": 2048, | |
3827 | "hashinfo": "bad-val" | |
3828 | }, | |
3829 | { | |
3830 | "osd": 2, | |
3831 | "primary": false, | |
3832 | "shard": 1, | |
3833 | "errors": [], | |
3834 | "size": 2048, | |
3835 | "omap_digest": "0xffffffff", | |
3836 | "data_digest": "0x00000000", | |
3837 | "hashinfo": { | |
3838 | "cumulative_shard_hashes": [ | |
3839 | { | |
3840 | "hash": 80717615, | |
3841 | "shard": 0 | |
3842 | }, | |
3843 | { | |
3844 | "hash": 1534491824, | |
3845 | "shard": 1 | |
3846 | }, | |
3847 | { | |
3848 | "hash": 80717615, | |
3849 | "shard": 2 | |
3850 | } | |
3851 | ], | |
3852 | "total_chunk_size": 2048 | |
3853 | } | |
3854 | } | |
3855 | ] | |
3856 | }, | |
3857 | { | |
3858 | "object": { | |
3859 | "name": "EOBJ7", | |
3860 | "nspace": "", | |
3861 | "locator": "", | |
3862 | "snap": "head", | |
3863 | "version": 10 | |
3864 | }, | |
3865 | "errors": [ | |
3866 | "hinfo_inconsistency" | |
3867 | ], | |
3868 | "union_shard_errors": [], | |
3869 | "selected_object_info": { | |
3870 | "oid": { | |
3871 | "oid": "EOBJ7", | |
3872 | "key": "", | |
3873 | "snapid": -2, | |
3874 | "hash": 3258066308, | |
3875 | "max": 0, | |
3876 | "pool": 3, | |
3877 | "namespace": "" | |
3878 | }, | |
3879 | "version": "75'10", | |
3880 | "prior_version": "75'9", | |
3881 | "last_reqid": "client.4482.0:1", | |
3882 | "user_version": 10, | |
3883 | "size": 34, | |
3884 | "mtime": "2018-04-05 14:32:33.058782", | |
3885 | "local_mtime": "2018-04-05 14:32:33.059679", | |
3886 | "lost": 0, | |
3887 | "flags": [ | |
3888 | "dirty", | |
28e407b8 | 3889 | "data_digest" |
94b18763 FG |
3890 | ], |
3891 | "truncate_seq": 0, | |
3892 | "truncate_size": 0, | |
3893 | "data_digest": "0x136e4e27", | |
3894 | "omap_digest": "0xffffffff", | |
3895 | "expected_object_size": 0, | |
3896 | "expected_write_size": 0, | |
3897 | "alloc_hint_flags": 0, | |
3898 | "manifest": { | |
3899 | "type": 0 | |
3900 | }, | |
3901 | "watchers": {} | |
3902 | }, | |
3903 | "shards": [ | |
3904 | { | |
3905 | "osd": 0, | |
3906 | "primary": false, | |
3907 | "shard": 2, | |
3908 | "errors": [], | |
3909 | "size": 2048, | |
3910 | "omap_digest": "0xffffffff", | |
3911 | "data_digest": "0x00000000", | |
3912 | "hashinfo": { | |
3913 | "cumulative_shard_hashes": [ | |
3914 | { | |
3915 | "hash": 80717615, | |
3916 | "shard": 0 | |
3917 | }, | |
3918 | { | |
3919 | "hash": 1534491824, | |
3920 | "shard": 1 | |
3921 | }, | |
3922 | { | |
3923 | "hash": 80717615, | |
3924 | "shard": 2 | |
3925 | } | |
3926 | ], | |
3927 | "total_chunk_size": 2048 | |
3928 | } | |
3929 | }, | |
3930 | { | |
3931 | "osd": 1, | |
3932 | "primary": true, | |
3933 | "shard": 0, | |
3934 | "errors": [], | |
3935 | "size": 2048, | |
3936 | "omap_digest": "0xffffffff", | |
3937 | "data_digest": "0x00000000", | |
3938 | "hashinfo": { | |
3939 | "cumulative_shard_hashes": [ | |
3940 | { | |
3941 | "hash": 1534350760, | |
3942 | "shard": 0 | |
3943 | }, | |
3944 | { | |
3945 | "hash": 1534491824, | |
3946 | "shard": 1 | |
3947 | }, | |
3948 | { | |
3949 | "hash": 1534350760, | |
3950 | "shard": 2 | |
3951 | } | |
3952 | ], | |
3953 | "total_chunk_size": 2048 | |
3954 | } | |
3955 | }, | |
3956 | { | |
3957 | "osd": 2, | |
3958 | "primary": false, | |
3959 | "shard": 1, | |
3960 | "errors": [], | |
3961 | "size": 2048, | |
3962 | "omap_digest": "0xffffffff", | |
3963 | "data_digest": "0x00000000", | |
3964 | "hashinfo": { | |
3965 | "cumulative_shard_hashes": [ | |
3966 | { | |
3967 | "hash": 1534350760, | |
3968 | "shard": 0 | |
3969 | }, | |
3970 | { | |
3971 | "hash": 1534491824, | |
3972 | "shard": 1 | |
3973 | }, | |
3974 | { | |
3975 | "hash": 1534350760, | |
3976 | "shard": 2 | |
3977 | } | |
3978 | ], | |
3979 | "total_chunk_size": 2048 | |
3980 | } | |
3981 | } | |
3982 | ] | |
31f18b77 FG |
3983 | } |
3984 | ], | |
3985 | "epoch": 0 | |
3986 | } | |
3987 | EOF | |
3988 | ||
3989 | else | |
3990 | ||
94b18763 | 3991 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
7c673cae FG |
3992 | { |
3993 | "inconsistents": [ | |
3994 | { | |
3995 | "shards": [ | |
3996 | { | |
3997 | "data_digest": "0x04cfa72f", | |
3998 | "omap_digest": "0xffffffff", | |
3999 | "size": 2048, | |
4000 | "errors": [], | |
4001 | "shard": 2, | |
b5b8bbf5 FG |
4002 | "osd": 0, |
4003 | "primary": false | |
7c673cae FG |
4004 | }, |
4005 | { | |
94b18763 FG |
4006 | "object_info": { |
4007 | "oid": { | |
4008 | "oid": "EOBJ1", | |
4009 | "key": "", | |
4010 | "snapid": -2, | |
4011 | "hash": 560836233, | |
4012 | "max": 0, | |
4013 | "pool": 3, | |
4014 | "namespace": "" | |
4015 | }, | |
4016 | "version": "27'1", | |
4017 | "prior_version": "0'0", | |
4018 | "last_reqid": "client.4192.0:1", | |
4019 | "user_version": 1, | |
4020 | "size": 7, | |
4021 | "mtime": "2018-04-05 14:30:10.688009", | |
4022 | "local_mtime": "2018-04-05 14:30:10.691774", | |
4023 | "lost": 0, | |
4024 | "flags": [ | |
4025 | "dirty", | |
28e407b8 | 4026 | "data_digest" |
94b18763 FG |
4027 | ], |
4028 | "truncate_seq": 0, | |
4029 | "truncate_size": 0, | |
4030 | "data_digest": "0x2ddbf8f5", | |
4031 | "omap_digest": "0xffffffff", | |
4032 | "expected_object_size": 0, | |
4033 | "expected_write_size": 0, | |
4034 | "alloc_hint_flags": 0, | |
4035 | "manifest": { | |
4036 | "type": 0 | |
4037 | }, | |
4038 | "watchers": {} | |
4039 | }, | |
7c673cae | 4040 | "size": 9, |
224ce89b | 4041 | "shard": 0, |
7c673cae FG |
4042 | "errors": [ |
4043 | "read_error", | |
94b18763 FG |
4044 | "size_mismatch_info", |
4045 | "obj_size_info_mismatch" | |
7c673cae | 4046 | ], |
b5b8bbf5 FG |
4047 | "osd": 1, |
4048 | "primary": true | |
7c673cae FG |
4049 | }, |
4050 | { | |
4051 | "data_digest": "0x04cfa72f", | |
4052 | "omap_digest": "0xffffffff", | |
4053 | "size": 2048, | |
224ce89b | 4054 | "shard": 1, |
7c673cae | 4055 | "errors": [], |
b5b8bbf5 FG |
4056 | "osd": 2, |
4057 | "primary": false | |
7c673cae FG |
4058 | } |
4059 | ], | |
94b18763 FG |
4060 | "selected_object_info": { |
4061 | "oid": { | |
4062 | "oid": "EOBJ1", | |
4063 | "key": "", | |
4064 | "snapid": -2, | |
4065 | "hash": 560836233, | |
4066 | "max": 0, | |
4067 | "pool": 3, | |
4068 | "namespace": "" | |
4069 | }, | |
4070 | "version": "27'1", | |
4071 | "prior_version": "0'0", | |
4072 | "last_reqid": "client.4192.0:1", | |
4073 | "user_version": 1, | |
4074 | "size": 7, | |
4075 | "mtime": "2018-04-05 14:30:10.688009", | |
4076 | "local_mtime": "2018-04-05 14:30:10.691774", | |
4077 | "lost": 0, | |
4078 | "flags": [ | |
4079 | "dirty", | |
28e407b8 | 4080 | "data_digest" |
94b18763 FG |
4081 | ], |
4082 | "truncate_seq": 0, | |
4083 | "truncate_size": 0, | |
4084 | "data_digest": "0x2ddbf8f5", | |
4085 | "omap_digest": "0xffffffff", | |
4086 | "expected_object_size": 0, | |
4087 | "expected_write_size": 0, | |
4088 | "alloc_hint_flags": 0, | |
4089 | "manifest": { | |
4090 | "type": 0 | |
4091 | }, | |
4092 | "watchers": {} | |
4093 | }, | |
7c673cae FG |
4094 | "union_shard_errors": [ |
4095 | "read_error", | |
94b18763 FG |
4096 | "size_mismatch_info", |
4097 | "obj_size_info_mismatch" | |
7c673cae FG |
4098 | ], |
4099 | "errors": [ | |
4100 | "size_mismatch" | |
4101 | ], | |
4102 | "object": { | |
4103 | "version": 1, | |
4104 | "snap": "head", | |
4105 | "locator": "", | |
4106 | "nspace": "", | |
4107 | "name": "EOBJ1" | |
4108 | } | |
4109 | }, | |
4110 | { | |
4111 | "shards": [ | |
4112 | { | |
4113 | "size": 2048, | |
4114 | "errors": [ | |
4115 | "ec_hash_error" | |
4116 | ], | |
4117 | "shard": 2, | |
b5b8bbf5 FG |
4118 | "osd": 0, |
4119 | "primary": false | |
7c673cae FG |
4120 | }, |
4121 | { | |
4122 | "data_digest": "0x04cfa72f", | |
4123 | "omap_digest": "0xffffffff", | |
4124 | "size": 2048, | |
4125 | "errors": [], | |
224ce89b | 4126 | "shard": 0, |
b5b8bbf5 FG |
4127 | "osd": 1, |
4128 | "primary": true | |
7c673cae FG |
4129 | }, |
4130 | { | |
4131 | "data_digest": "0x04cfa72f", | |
4132 | "omap_digest": "0xffffffff", | |
4133 | "size": 2048, | |
4134 | "errors": [], | |
224ce89b | 4135 | "shard": 1, |
b5b8bbf5 FG |
4136 | "osd": 2, |
4137 | "primary": false | |
7c673cae FG |
4138 | } |
4139 | ], | |
94b18763 FG |
4140 | "selected_object_info": { |
4141 | "oid": { | |
4142 | "oid": "EOBJ2", | |
4143 | "key": "", | |
4144 | "snapid": -2, | |
4145 | "hash": 562812377, | |
4146 | "max": 0, | |
4147 | "pool": 3, | |
4148 | "namespace": "" | |
4149 | }, | |
4150 | "version": "33'2", | |
4151 | "prior_version": "0'0", | |
4152 | "last_reqid": "client.4224.0:1", | |
4153 | "user_version": 2, | |
4154 | "size": 7, | |
4155 | "mtime": "2018-04-05 14:30:14.152945", | |
4156 | "local_mtime": "2018-04-05 14:30:14.154014", | |
4157 | "lost": 0, | |
4158 | "flags": [ | |
4159 | "dirty", | |
28e407b8 | 4160 | "data_digest" |
94b18763 FG |
4161 | ], |
4162 | "truncate_seq": 0, | |
4163 | "truncate_size": 0, | |
4164 | "data_digest": "0x2ddbf8f5", | |
4165 | "omap_digest": "0xffffffff", | |
4166 | "expected_object_size": 0, | |
4167 | "expected_write_size": 0, | |
4168 | "alloc_hint_flags": 0, | |
4169 | "manifest": { | |
4170 | "type": 0 | |
4171 | }, | |
4172 | "watchers": {} | |
4173 | }, | |
7c673cae FG |
4174 | "union_shard_errors": [ |
4175 | "ec_hash_error" | |
4176 | ], | |
4177 | "errors": [], | |
4178 | "object": { | |
4179 | "version": 2, | |
4180 | "snap": "head", | |
4181 | "locator": "", | |
4182 | "nspace": "", | |
4183 | "name": "EOBJ2" | |
4184 | } | |
4185 | }, | |
4186 | { | |
4187 | "shards": [ | |
4188 | { | |
4189 | "data_digest": "0x04cfa72f", | |
4190 | "omap_digest": "0xffffffff", | |
4191 | "size": 2048, | |
4192 | "errors": [], | |
4193 | "shard": 2, | |
b5b8bbf5 FG |
4194 | "osd": 0, |
4195 | "primary": false | |
7c673cae FG |
4196 | }, |
4197 | { | |
224ce89b | 4198 | "osd": 1, |
b5b8bbf5 | 4199 | "primary": true, |
224ce89b | 4200 | "shard": 0, |
7c673cae FG |
4201 | "errors": [ |
4202 | "missing" | |
224ce89b | 4203 | ] |
7c673cae FG |
4204 | }, |
4205 | { | |
4206 | "data_digest": "0x04cfa72f", | |
4207 | "omap_digest": "0xffffffff", | |
4208 | "size": 2048, | |
224ce89b | 4209 | "shard": 1, |
7c673cae | 4210 | "errors": [], |
b5b8bbf5 FG |
4211 | "osd": 2, |
4212 | "primary": false | |
7c673cae FG |
4213 | } |
4214 | ], | |
94b18763 FG |
4215 | "selected_object_info": { |
4216 | "oid": { | |
4217 | "oid": "EOBJ3", | |
4218 | "key": "", | |
4219 | "snapid": -2, | |
4220 | "hash": 3125668237, | |
4221 | "max": 0, | |
4222 | "pool": 3, | |
4223 | "namespace": "" | |
4224 | }, | |
4225 | "version": "39'3", | |
4226 | "prior_version": "0'0", | |
4227 | "last_reqid": "client.4258.0:1", | |
4228 | "user_version": 3, | |
4229 | "size": 7, | |
4230 | "mtime": "2018-04-05 14:30:18.875544", | |
4231 | "local_mtime": "2018-04-05 14:30:18.880153", | |
4232 | "lost": 0, | |
4233 | "flags": [ | |
4234 | "dirty", | |
28e407b8 | 4235 | "data_digest" |
94b18763 FG |
4236 | ], |
4237 | "truncate_seq": 0, | |
4238 | "truncate_size": 0, | |
4239 | "data_digest": "0x2ddbf8f5", | |
4240 | "omap_digest": "0xffffffff", | |
4241 | "expected_object_size": 0, | |
4242 | "expected_write_size": 0, | |
4243 | "alloc_hint_flags": 0, | |
4244 | "manifest": { | |
4245 | "type": 0 | |
4246 | }, | |
4247 | "watchers": {} | |
4248 | }, | |
7c673cae FG |
4249 | "union_shard_errors": [ |
4250 | "missing" | |
4251 | ], | |
4252 | "errors": [], | |
4253 | "object": { | |
4254 | "version": 3, | |
4255 | "snap": "head", | |
4256 | "locator": "", | |
4257 | "nspace": "", | |
4258 | "name": "EOBJ3" | |
4259 | } | |
4260 | }, | |
4261 | { | |
4262 | "shards": [ | |
4263 | { | |
4264 | "attrs": [ | |
7c673cae FG |
4265 | { |
4266 | "Base64": false, | |
4267 | "value": "bad-val", | |
94b18763 | 4268 | "name": "key1-EOBJ4" |
7c673cae FG |
4269 | }, |
4270 | { | |
4271 | "Base64": false, | |
b32b8144 | 4272 | "value": "val2-EOBJ4", |
94b18763 | 4273 | "name": "key2-EOBJ4" |
7c673cae FG |
4274 | } |
4275 | ], | |
4276 | "data_digest": "0x04cfa72f", | |
4277 | "omap_digest": "0xffffffff", | |
4278 | "size": 2048, | |
4279 | "errors": [], | |
4280 | "shard": 2, | |
b5b8bbf5 FG |
4281 | "osd": 0, |
4282 | "primary": false | |
7c673cae FG |
4283 | }, |
4284 | { | |
224ce89b | 4285 | "osd": 1, |
b5b8bbf5 | 4286 | "primary": true, |
224ce89b WB |
4287 | "shard": 0, |
4288 | "errors": [], | |
4289 | "size": 2048, | |
4290 | "omap_digest": "0xffffffff", | |
4291 | "data_digest": "0x04cfa72f", | |
7c673cae | 4292 | "attrs": [ |
7c673cae FG |
4293 | { |
4294 | "Base64": false, | |
4295 | "value": "val1-EOBJ4", | |
94b18763 | 4296 | "name": "key1-EOBJ4" |
7c673cae FG |
4297 | }, |
4298 | { | |
4299 | "Base64": false, | |
4300 | "value": "val2-EOBJ4", | |
94b18763 | 4301 | "name": "key2-EOBJ4" |
7c673cae | 4302 | } |
224ce89b | 4303 | ] |
7c673cae FG |
4304 | }, |
4305 | { | |
224ce89b | 4306 | "osd": 2, |
b5b8bbf5 | 4307 | "primary": false, |
224ce89b WB |
4308 | "shard": 1, |
4309 | "errors": [], | |
4310 | "size": 2048, | |
4311 | "omap_digest": "0xffffffff", | |
4312 | "data_digest": "0x04cfa72f", | |
7c673cae | 4313 | "attrs": [ |
7c673cae FG |
4314 | { |
4315 | "Base64": false, | |
4316 | "value": "val1-EOBJ4", | |
94b18763 | 4317 | "name": "key1-EOBJ4" |
7c673cae FG |
4318 | }, |
4319 | { | |
4320 | "Base64": false, | |
b32b8144 | 4321 | "value": "val3-EOBJ4", |
94b18763 | 4322 | "name": "key3-EOBJ4" |
7c673cae | 4323 | } |
224ce89b | 4324 | ] |
7c673cae FG |
4325 | } |
4326 | ], | |
94b18763 FG |
4327 | "selected_object_info": { |
4328 | "oid": { | |
4329 | "oid": "EOBJ4", | |
4330 | "key": "", | |
4331 | "snapid": -2, | |
4332 | "hash": 1618759290, | |
4333 | "max": 0, | |
4334 | "pool": 3, | |
4335 | "namespace": "" | |
4336 | }, | |
4337 | "version": "45'6", | |
4338 | "prior_version": "45'5", | |
4339 | "last_reqid": "client.4296.0:1", | |
4340 | "user_version": 6, | |
4341 | "size": 7, | |
4342 | "mtime": "2018-04-05 14:30:22.271983", | |
4343 | "local_mtime": "2018-04-05 14:30:22.272840", | |
4344 | "lost": 0, | |
4345 | "flags": [ | |
4346 | "dirty", | |
28e407b8 | 4347 | "data_digest" |
94b18763 FG |
4348 | ], |
4349 | "truncate_seq": 0, | |
4350 | "truncate_size": 0, | |
4351 | "data_digest": "0x2ddbf8f5", | |
4352 | "omap_digest": "0xffffffff", | |
4353 | "expected_object_size": 0, | |
4354 | "expected_write_size": 0, | |
4355 | "alloc_hint_flags": 0, | |
4356 | "manifest": { | |
4357 | "type": 0 | |
4358 | }, | |
4359 | "watchers": {} | |
4360 | }, | |
7c673cae FG |
4361 | "union_shard_errors": [], |
4362 | "errors": [ | |
4363 | "attr_value_mismatch", | |
4364 | "attr_name_mismatch" | |
4365 | ], | |
4366 | "object": { | |
4367 | "version": 6, | |
4368 | "snap": "head", | |
4369 | "locator": "", | |
4370 | "nspace": "", | |
4371 | "name": "EOBJ4" | |
4372 | } | |
4373 | }, | |
4374 | { | |
4375 | "shards": [ | |
4376 | { | |
4377 | "data_digest": "0x04cfa72f", | |
4378 | "omap_digest": "0xffffffff", | |
4379 | "size": 2048, | |
4380 | "errors": [], | |
4381 | "shard": 2, | |
b5b8bbf5 FG |
4382 | "osd": 0, |
4383 | "primary": false | |
7c673cae FG |
4384 | }, |
4385 | { | |
94b18763 FG |
4386 | "object_info": { |
4387 | "oid": { | |
4388 | "oid": "EOBJ5", | |
4389 | "key": "", | |
4390 | "snapid": -2, | |
4391 | "hash": 2918945441, | |
4392 | "max": 0, | |
4393 | "pool": 3, | |
4394 | "namespace": "" | |
4395 | }, | |
4396 | "version": "59'7", | |
4397 | "prior_version": "0'0", | |
4398 | "last_reqid": "client.4384.0:1", | |
4399 | "user_version": 7, | |
4400 | "size": 7, | |
4401 | "mtime": "2018-04-05 14:30:35.162395", | |
4402 | "local_mtime": "2018-04-05 14:30:35.166390", | |
4403 | "lost": 0, | |
4404 | "flags": [ | |
4405 | "dirty", | |
28e407b8 | 4406 | "data_digest" |
94b18763 FG |
4407 | ], |
4408 | "truncate_seq": 0, | |
4409 | "truncate_size": 0, | |
4410 | "data_digest": "0x2ddbf8f5", | |
4411 | "omap_digest": "0xffffffff", | |
4412 | "expected_object_size": 0, | |
4413 | "expected_write_size": 0, | |
4414 | "alloc_hint_flags": 0, | |
4415 | "manifest": { | |
4416 | "type": 0 | |
4417 | }, | |
4418 | "watchers": {} | |
4419 | }, | |
7c673cae | 4420 | "size": 4096, |
224ce89b | 4421 | "shard": 0, |
7c673cae | 4422 | "errors": [ |
94b18763 | 4423 | "size_mismatch_info", |
b5b8bbf5 | 4424 | "ec_size_error", |
94b18763 | 4425 | "obj_size_info_mismatch" |
7c673cae | 4426 | ], |
b5b8bbf5 FG |
4427 | "osd": 1, |
4428 | "primary": true | |
7c673cae FG |
4429 | }, |
4430 | { | |
4431 | "data_digest": "0x04cfa72f", | |
4432 | "omap_digest": "0xffffffff", | |
4433 | "size": 2048, | |
224ce89b | 4434 | "shard": 1, |
7c673cae | 4435 | "errors": [], |
b5b8bbf5 FG |
4436 | "osd": 2, |
4437 | "primary": false | |
7c673cae FG |
4438 | } |
4439 | ], | |
94b18763 FG |
4440 | "selected_object_info": { |
4441 | "oid": { | |
4442 | "oid": "EOBJ5", | |
4443 | "key": "", | |
4444 | "snapid": -2, | |
4445 | "hash": 2918945441, | |
4446 | "max": 0, | |
4447 | "pool": 3, | |
4448 | "namespace": "" | |
4449 | }, | |
4450 | "version": "59'7", | |
4451 | "prior_version": "0'0", | |
4452 | "last_reqid": "client.4384.0:1", | |
4453 | "user_version": 7, | |
4454 | "size": 7, | |
4455 | "mtime": "2018-04-05 14:30:35.162395", | |
4456 | "local_mtime": "2018-04-05 14:30:35.166390", | |
4457 | "lost": 0, | |
4458 | "flags": [ | |
4459 | "dirty", | |
28e407b8 | 4460 | "data_digest" |
94b18763 FG |
4461 | ], |
4462 | "truncate_seq": 0, | |
4463 | "truncate_size": 0, | |
4464 | "data_digest": "0x2ddbf8f5", | |
4465 | "omap_digest": "0xffffffff", | |
4466 | "expected_object_size": 0, | |
4467 | "expected_write_size": 0, | |
4468 | "alloc_hint_flags": 0, | |
4469 | "manifest": { | |
4470 | "type": 0 | |
4471 | }, | |
4472 | "watchers": {} | |
4473 | }, | |
7c673cae | 4474 | "union_shard_errors": [ |
94b18763 | 4475 | "size_mismatch_info", |
b5b8bbf5 | 4476 | "ec_size_error", |
94b18763 | 4477 | "obj_size_info_mismatch" |
7c673cae FG |
4478 | ], |
4479 | "errors": [ | |
4480 | "size_mismatch" | |
4481 | ], | |
4482 | "object": { | |
4483 | "version": 7, | |
4484 | "snap": "head", | |
4485 | "locator": "", | |
4486 | "nspace": "", | |
4487 | "name": "EOBJ5" | |
4488 | } | |
94b18763 FG |
4489 | }, |
4490 | { | |
4491 | "object": { | |
4492 | "name": "EOBJ6", | |
4493 | "nspace": "", | |
4494 | "locator": "", | |
4495 | "snap": "head", | |
4496 | "version": 8 | |
4497 | }, | |
4498 | "errors": [], | |
4499 | "union_shard_errors": [ | |
4500 | "read_error", | |
4501 | "hinfo_missing", | |
4502 | "hinfo_corrupted" | |
4503 | ], | |
4504 | "selected_object_info": { | |
4505 | "oid": { | |
4506 | "oid": "EOBJ6", | |
4507 | "key": "", | |
4508 | "snapid": -2, | |
4509 | "hash": 3050890866, | |
4510 | "max": 0, | |
4511 | "pool": 3, | |
4512 | "namespace": "" | |
4513 | }, | |
4514 | "version": "65'8", | |
4515 | "prior_version": "0'0", | |
4516 | "last_reqid": "client.4420.0:1", | |
4517 | "user_version": 8, | |
4518 | "size": 7, | |
4519 | "mtime": "2018-04-05 14:30:40.914673", | |
4520 | "local_mtime": "2018-04-05 14:30:40.917705", | |
4521 | "lost": 0, | |
4522 | "flags": [ | |
4523 | "dirty", | |
28e407b8 | 4524 | "data_digest" |
94b18763 FG |
4525 | ], |
4526 | "truncate_seq": 0, | |
4527 | "truncate_size": 0, | |
4528 | "data_digest": "0x2ddbf8f5", | |
4529 | "omap_digest": "0xffffffff", | |
4530 | "expected_object_size": 0, | |
4531 | "expected_write_size": 0, | |
4532 | "alloc_hint_flags": 0, | |
4533 | "manifest": { | |
4534 | "type": 0 | |
4535 | }, | |
4536 | "watchers": {} | |
4537 | }, | |
4538 | "shards": [ | |
4539 | { | |
4540 | "osd": 0, | |
4541 | "primary": false, | |
4542 | "shard": 2, | |
4543 | "errors": [ | |
4544 | "read_error", | |
4545 | "hinfo_missing" | |
4546 | ], | |
4547 | "size": 2048 | |
4548 | }, | |
4549 | { | |
4550 | "osd": 1, | |
4551 | "primary": true, | |
4552 | "shard": 0, | |
4553 | "errors": [ | |
4554 | "read_error", | |
4555 | "hinfo_corrupted" | |
4556 | ], | |
4557 | "size": 2048, | |
4558 | "hashinfo": "bad-val" | |
4559 | }, | |
4560 | { | |
4561 | "osd": 2, | |
4562 | "primary": false, | |
4563 | "shard": 1, | |
4564 | "errors": [], | |
4565 | "size": 2048, | |
4566 | "omap_digest": "0xffffffff", | |
4567 | "data_digest": "0x04cfa72f", | |
4568 | "hashinfo": { | |
4569 | "cumulative_shard_hashes": [ | |
4570 | { | |
4571 | "hash": 80717615, | |
4572 | "shard": 0 | |
4573 | }, | |
4574 | { | |
4575 | "hash": 1534491824, | |
4576 | "shard": 1 | |
4577 | }, | |
4578 | { | |
4579 | "hash": 80717615, | |
4580 | "shard": 2 | |
4581 | } | |
4582 | ], | |
4583 | "total_chunk_size": 2048 | |
4584 | } | |
4585 | } | |
4586 | ] | |
4587 | }, | |
4588 | { | |
4589 | "object": { | |
4590 | "name": "EOBJ7", | |
4591 | "nspace": "", | |
4592 | "locator": "", | |
4593 | "snap": "head", | |
4594 | "version": 10 | |
4595 | }, | |
4596 | "errors": [ | |
4597 | "hinfo_inconsistency" | |
4598 | ], | |
4599 | "union_shard_errors": [ | |
4600 | "ec_hash_error" | |
4601 | ], | |
4602 | "selected_object_info": { | |
4603 | "oid": { | |
4604 | "oid": "EOBJ7", | |
4605 | "key": "", | |
4606 | "snapid": -2, | |
4607 | "hash": 3258066308, | |
4608 | "max": 0, | |
4609 | "pool": 3, | |
4610 | "namespace": "" | |
4611 | }, | |
4612 | "version": "75'10", | |
4613 | "prior_version": "75'9", | |
4614 | "last_reqid": "client.4486.0:1", | |
4615 | "user_version": 10, | |
4616 | "size": 34, | |
4617 | "mtime": "2018-04-05 14:30:50.995009", | |
4618 | "local_mtime": "2018-04-05 14:30:50.996112", | |
4619 | "lost": 0, | |
4620 | "flags": [ | |
4621 | "dirty", | |
28e407b8 | 4622 | "data_digest" |
94b18763 FG |
4623 | ], |
4624 | "truncate_seq": 0, | |
4625 | "truncate_size": 0, | |
4626 | "data_digest": "0x136e4e27", | |
4627 | "omap_digest": "0xffffffff", | |
4628 | "expected_object_size": 0, | |
4629 | "expected_write_size": 0, | |
4630 | "alloc_hint_flags": 0, | |
4631 | "manifest": { | |
4632 | "type": 0 | |
4633 | }, | |
4634 | "watchers": {} | |
4635 | }, | |
4636 | "shards": [ | |
4637 | { | |
4638 | "osd": 0, | |
4639 | "primary": false, | |
4640 | "shard": 2, | |
4641 | "errors": [ | |
4642 | "ec_hash_error" | |
4643 | ], | |
4644 | "size": 2048, | |
4645 | "hashinfo": { | |
4646 | "cumulative_shard_hashes": [ | |
4647 | { | |
4648 | "hash": 80717615, | |
4649 | "shard": 0 | |
4650 | }, | |
4651 | { | |
4652 | "hash": 1534491824, | |
4653 | "shard": 1 | |
4654 | }, | |
4655 | { | |
4656 | "hash": 80717615, | |
4657 | "shard": 2 | |
4658 | } | |
4659 | ], | |
4660 | "total_chunk_size": 2048 | |
4661 | } | |
4662 | }, | |
4663 | { | |
4664 | "osd": 1, | |
4665 | "primary": true, | |
4666 | "shard": 0, | |
4667 | "errors": [], | |
4668 | "size": 2048, | |
4669 | "omap_digest": "0xffffffff", | |
4670 | "data_digest": "0x5b7455a8", | |
4671 | "hashinfo": { | |
4672 | "cumulative_shard_hashes": [ | |
4673 | { | |
4674 | "hash": 1534350760, | |
4675 | "shard": 0 | |
4676 | }, | |
4677 | { | |
4678 | "hash": 1534491824, | |
4679 | "shard": 1 | |
4680 | }, | |
4681 | { | |
4682 | "hash": 1534350760, | |
4683 | "shard": 2 | |
4684 | } | |
4685 | ], | |
4686 | "total_chunk_size": 2048 | |
4687 | } | |
4688 | }, | |
4689 | { | |
4690 | "osd": 2, | |
4691 | "primary": false, | |
4692 | "shard": 1, | |
4693 | "errors": [], | |
4694 | "size": 2048, | |
4695 | "omap_digest": "0xffffffff", | |
4696 | "data_digest": "0x5b7455a8", | |
4697 | "hashinfo": { | |
4698 | "cumulative_shard_hashes": [ | |
4699 | { | |
4700 | "hash": 1534350760, | |
4701 | "shard": 0 | |
4702 | }, | |
4703 | { | |
4704 | "hash": 1534491824, | |
4705 | "shard": 1 | |
4706 | }, | |
4707 | { | |
4708 | "hash": 1534350760, | |
4709 | "shard": 2 | |
4710 | } | |
4711 | ], | |
4712 | "total_chunk_size": 2048 | |
4713 | } | |
4714 | } | |
4715 | ] | |
7c673cae FG |
4716 | } |
4717 | ], | |
4718 | "epoch": 0 | |
4719 | } | |
4720 | EOF | |
4721 | ||
7c673cae | 4722 | fi |
31f18b77 | 4723 | |
94b18763 | 4724 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
224ce89b | 4725 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
7c673cae FG |
4726 | if test $getjson = "yes" |
4727 | then | |
31f18b77 FG |
4728 | if [ "$allow_overwrites" = "true" ] |
4729 | then | |
4730 | num=4 | |
4731 | else | |
4732 | num=5 | |
4733 | fi | |
4734 | jq '.' $dir/json > save${num}.json | |
7c673cae FG |
4735 | fi |
4736 | ||
94b18763 | 4737 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
7c673cae FG |
4738 | then |
4739 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
4740 | fi | |
4741 | ||
4742 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
4743 | teardown $dir || return 1 | |
4744 | } | |
4745 | ||
4746 | function TEST_corrupt_scrub_erasure_appends() { | |
4747 | corrupt_scrub_erasure $1 false | |
4748 | } | |
4749 | ||
4750 | function TEST_corrupt_scrub_erasure_overwrites() { | |
224ce89b WB |
4751 | if [ "$use_ec_overwrite" = "true" ]; then |
4752 | corrupt_scrub_erasure $1 true | |
4753 | fi | |
7c673cae FG |
4754 | } |
4755 | ||
4756 | # | |
4757 | # Test to make sure that a periodic scrub won't cause deep-scrub info to be lost | |
4758 | # | |
4759 | function TEST_periodic_scrub_replicated() { | |
4760 | local dir=$1 | |
4761 | local poolname=psr_pool | |
4762 | local objname=POBJ | |
4763 | ||
4764 | setup $dir || return 1 | |
4765 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
4766 | run_mgr $dir x || return 1 | |
b5b8bbf5 FG |
4767 | local ceph_osd_args="--osd-scrub-interval-randomize-ratio=0 --osd-deep-scrub-randomize-ratio=0 " |
4768 | ceph_osd_args+="--osd_scrub_backoff_ratio=0" | |
7c673cae FG |
4769 | run_osd $dir 0 $ceph_osd_args || return 1 |
4770 | run_osd $dir 1 $ceph_osd_args || return 1 | |
c07f9fc5 | 4771 | create_rbd_pool || return 1 |
7c673cae FG |
4772 | wait_for_clean || return 1 |
4773 | ||
b5b8bbf5 | 4774 | create_pool $poolname 1 1 || return 1 |
7c673cae FG |
4775 | wait_for_clean || return 1 |
4776 | ||
4777 | local osd=0 | |
4778 | add_something $dir $poolname $objname scrub || return 1 | |
4779 | local primary=$(get_primary $poolname $objname) | |
4780 | local pg=$(get_pg $poolname $objname) | |
4781 | ||
4782 | # Add deep-scrub only error | |
4783 | local payload=UVWXYZ | |
4784 | echo $payload > $dir/CORRUPT | |
4785 | # Uses $ceph_osd_args for osd restart | |
4786 | objectstore_tool $dir $osd $objname set-bytes $dir/CORRUPT || return 1 | |
4787 | ||
4788 | # No scrub information available, so expect failure | |
4789 | set -o pipefail | |
4790 | ! rados list-inconsistent-obj $pg | jq '.' || return 1 | |
4791 | set +o pipefail | |
4792 | ||
4793 | pg_deep_scrub $pg || return 1 | |
4794 | ||
4795 | # Make sure bad object found | |
4796 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
4797 | ||
b5b8bbf5 | 4798 | flush_pg_stats |
7c673cae FG |
4799 | local last_scrub=$(get_last_scrub_stamp $pg) |
4800 | # Fake a schedule scrub | |
c07f9fc5 | 4801 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
4802 | trigger_scrub $pg || return 1 |
4803 | # Wait for schedule regular scrub | |
4804 | wait_for_scrub $pg "$last_scrub" | |
4805 | ||
4806 | # It needed to be upgraded | |
4807 | grep -q "Deep scrub errors, upgrading scrub to deep-scrub" $dir/osd.${primary}.log || return 1 | |
4808 | ||
4809 | # Bad object still known | |
4810 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
4811 | ||
4812 | # Can't upgrade with this set | |
4813 | ceph osd set nodeep-scrub | |
224ce89b | 4814 | # Let map change propagate to OSDs |
b5b8bbf5 FG |
4815 | flush pg_stats |
4816 | sleep 5 | |
7c673cae FG |
4817 | |
4818 | # Fake a schedule scrub | |
c07f9fc5 | 4819 | CEPH_ARGS='' ceph --admin-daemon $(get_asok_path osd.${primary}) \ |
7c673cae FG |
4820 | trigger_scrub $pg || return 1 |
4821 | # Wait for schedule regular scrub | |
4822 | # to notice scrub and skip it | |
4823 | local found=false | |
4824 | for i in $(seq 14 -1 0) | |
4825 | do | |
4826 | sleep 1 | |
4827 | ! grep -q "Regular scrub skipped due to deep-scrub errors and nodeep-scrub set" $dir/osd.${primary}.log || { found=true ; break; } | |
4828 | echo Time left: $i seconds | |
4829 | done | |
4830 | test $found = "true" || return 1 | |
4831 | ||
4832 | # Bad object still known | |
4833 | rados list-inconsistent-obj $pg | jq '.' | grep -q $objname || return 1 | |
4834 | ||
b5b8bbf5 | 4835 | flush_pg_stats |
7c673cae | 4836 | # Request a regular scrub and it will be done |
7c673cae | 4837 | pg_scrub $pg |
c07f9fc5 | 4838 | grep -q "Regular scrub request, deep-scrub details will be lost" $dir/osd.${primary}.log || return 1 |
7c673cae FG |
4839 | |
4840 | # deep-scrub error is no longer present | |
4841 | rados list-inconsistent-obj $pg | jq '.' | grep -qv $objname || return 1 | |
4842 | } | |
4843 | ||
3a9019d9 FG |
4844 | # |
4845 | # Corrupt snapset in replicated pool | |
4846 | # | |
4847 | function TEST_corrupt_snapset_scrub_rep() { | |
4848 | local dir=$1 | |
4849 | local poolname=csr_pool | |
4850 | local total_objs=2 | |
4851 | ||
4852 | setup $dir || return 1 | |
4853 | run_mon $dir a --osd_pool_default_size=2 || return 1 | |
4854 | run_mgr $dir x || return 1 | |
4855 | run_osd $dir 0 || return 1 | |
4856 | run_osd $dir 1 || return 1 | |
4857 | create_rbd_pool || return 1 | |
4858 | wait_for_clean || return 1 | |
4859 | ||
4860 | create_pool foo 1 || return 1 | |
4861 | create_pool $poolname 1 1 || return 1 | |
4862 | wait_for_clean || return 1 | |
4863 | ||
4864 | for i in $(seq 1 $total_objs) ; do | |
4865 | objname=ROBJ${i} | |
4866 | add_something $dir $poolname $objname || return 1 | |
4867 | ||
4868 | rados --pool $poolname setomapheader $objname hdr-$objname || return 1 | |
4869 | rados --pool $poolname setomapval $objname key-$objname val-$objname || return 1 | |
4870 | done | |
4871 | ||
4872 | local pg=$(get_pg $poolname ROBJ0) | |
4873 | ||
4874 | for i in $(seq 1 $total_objs) ; do | |
4875 | objname=ROBJ${i} | |
4876 | ||
4877 | # Alternate corruption between osd.0 and osd.1 | |
4878 | local osd=$(expr $i % 2) | |
4879 | ||
4880 | rados -p $poolname mksnap snap1 | |
4881 | echo -n head_of_snapshot_data > $dir/change | |
4882 | ||
4883 | case $i in | |
4884 | 1) | |
4885 | rados --pool $poolname put $objname $dir/change | |
4886 | objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1 | |
4887 | ;; | |
4888 | ||
4889 | 2) | |
4890 | rados --pool $poolname put $objname $dir/change | |
4891 | objectstore_tool $dir $osd --head $objname clear-snapset corrupt || return 1 | |
4892 | ;; | |
4893 | ||
4894 | esac | |
4895 | done | |
4896 | rm $dir/change | |
4897 | ||
4898 | pg_scrub $pg | |
4899 | ||
4900 | rados list-inconsistent-pg $poolname > $dir/json || return 1 | |
4901 | # Check pg count | |
4902 | test $(jq '. | length' $dir/json) = "1" || return 1 | |
4903 | # Check pgid | |
4904 | test $(jq -r '.[0]' $dir/json) = $pg || return 1 | |
4905 | ||
4906 | rados list-inconsistent-obj $pg > $dir/json || return 1 | |
4907 | ||
94b18763 | 4908 | jq "$jqfilter" << EOF | jq '.inconsistents' | python -c "$sortkeys" > $dir/checkcsjson |
3a9019d9 FG |
4909 | { |
4910 | "epoch": 34, | |
4911 | "inconsistents": [ | |
4912 | { | |
4913 | "object": { | |
4914 | "name": "ROBJ1", | |
4915 | "nspace": "", | |
4916 | "locator": "", | |
4917 | "snap": "head", | |
4918 | "version": 8 | |
4919 | }, | |
4920 | "errors": [ | |
4921 | "snapset_inconsistency" | |
4922 | ], | |
4923 | "union_shard_errors": [], | |
94b18763 FG |
4924 | "selected_object_info": { |
4925 | "oid": { | |
4926 | "oid": "ROBJ1", | |
4927 | "key": "", | |
4928 | "snapid": -2, | |
4929 | "hash": 1454963827, | |
4930 | "max": 0, | |
4931 | "pool": 3, | |
4932 | "namespace": "" | |
4933 | }, | |
4934 | "version": "24'8", | |
4935 | "prior_version": "21'3", | |
4936 | "last_reqid": "client.4195.0:1", | |
4937 | "user_version": 8, | |
4938 | "size": 21, | |
4939 | "mtime": "2018-04-05 14:35:43.286117", | |
4940 | "local_mtime": "2018-04-05 14:35:43.288990", | |
4941 | "lost": 0, | |
4942 | "flags": [ | |
4943 | "dirty", | |
4944 | "omap", | |
4945 | "data_digest" | |
4946 | ], | |
4947 | "truncate_seq": 0, | |
4948 | "truncate_size": 0, | |
4949 | "data_digest": "0x53acb008", | |
4950 | "omap_digest": "0xffffffff", | |
4951 | "expected_object_size": 0, | |
4952 | "expected_write_size": 0, | |
4953 | "alloc_hint_flags": 0, | |
4954 | "manifest": { | |
4955 | "type": 0 | |
4956 | }, | |
4957 | "watchers": {} | |
4958 | }, | |
3a9019d9 FG |
4959 | "shards": [ |
4960 | { | |
4961 | "osd": 0, | |
4962 | "primary": false, | |
4963 | "errors": [], | |
4964 | "size": 21, | |
94b18763 FG |
4965 | "snapset": { |
4966 | "clones": [ | |
4967 | { | |
4968 | "overlap": "[]", | |
4969 | "size": 7, | |
4970 | "snap": 1, | |
4971 | "snaps": [ | |
4972 | 1 | |
4973 | ] | |
4974 | } | |
4975 | ], | |
4976 | "head_exists": 1, | |
4977 | "snap_context": { | |
4978 | "seq": 1, | |
4979 | "snaps": [ | |
4980 | 1 | |
4981 | ] | |
4982 | } | |
4983 | } | |
3a9019d9 FG |
4984 | }, |
4985 | { | |
4986 | "osd": 1, | |
4987 | "primary": true, | |
4988 | "errors": [], | |
4989 | "size": 21, | |
94b18763 FG |
4990 | "snapset": { |
4991 | "clones": [], | |
4992 | "head_exists": 0, | |
4993 | "snap_context": { | |
4994 | "seq": 0, | |
4995 | "snaps": [] | |
4996 | } | |
4997 | } | |
3a9019d9 FG |
4998 | } |
4999 | ] | |
5000 | }, | |
5001 | { | |
5002 | "object": { | |
5003 | "name": "ROBJ2", | |
5004 | "nspace": "", | |
5005 | "locator": "", | |
5006 | "snap": "head", | |
5007 | "version": 10 | |
5008 | }, | |
5009 | "errors": [ | |
5010 | "snapset_inconsistency" | |
5011 | ], | |
5012 | "union_shard_errors": [], | |
94b18763 FG |
5013 | "selected_object_info": { |
5014 | "oid": { | |
5015 | "oid": "ROBJ2", | |
5016 | "key": "", | |
5017 | "snapid": -2, | |
5018 | "hash": 2026323607, | |
5019 | "max": 0, | |
5020 | "pool": 3, | |
5021 | "namespace": "" | |
5022 | }, | |
5023 | "version": "28'10", | |
5024 | "prior_version": "23'6", | |
5025 | "last_reqid": "client.4223.0:1", | |
5026 | "user_version": 10, | |
5027 | "size": 21, | |
5028 | "mtime": "2018-04-05 14:35:48.326856", | |
5029 | "local_mtime": "2018-04-05 14:35:48.328097", | |
5030 | "lost": 0, | |
5031 | "flags": [ | |
5032 | "dirty", | |
5033 | "omap", | |
5034 | "data_digest" | |
5035 | ], | |
5036 | "truncate_seq": 0, | |
5037 | "truncate_size": 0, | |
5038 | "data_digest": "0x53acb008", | |
5039 | "omap_digest": "0xffffffff", | |
5040 | "expected_object_size": 0, | |
5041 | "expected_write_size": 0, | |
5042 | "alloc_hint_flags": 0, | |
5043 | "manifest": { | |
5044 | "type": 0 | |
5045 | }, | |
5046 | "watchers": {} | |
5047 | }, | |
3a9019d9 FG |
5048 | "shards": [ |
5049 | { | |
5050 | "osd": 0, | |
5051 | "primary": false, | |
5052 | "errors": [], | |
5053 | "size": 21, | |
94b18763 FG |
5054 | "snapset": { |
5055 | "clones": [], | |
5056 | "head_exists": 0, | |
5057 | "snap_context": { | |
5058 | "seq": 0, | |
5059 | "snaps": [] | |
5060 | } | |
5061 | } | |
3a9019d9 FG |
5062 | }, |
5063 | { | |
5064 | "osd": 1, | |
5065 | "primary": true, | |
5066 | "errors": [], | |
5067 | "size": 21, | |
94b18763 FG |
5068 | "snapset": { |
5069 | "clones": [ | |
5070 | { | |
5071 | "overlap": "[]", | |
5072 | "size": 7, | |
5073 | "snap": 1, | |
5074 | "snaps": [ | |
5075 | 1 | |
5076 | ] | |
5077 | } | |
5078 | ], | |
5079 | "head_exists": 1, | |
5080 | "snap_context": { | |
5081 | "seq": 1, | |
5082 | "snaps": [ | |
5083 | 1 | |
5084 | ] | |
5085 | } | |
5086 | } | |
3a9019d9 FG |
5087 | } |
5088 | ] | |
5089 | } | |
5090 | ] | |
5091 | } | |
5092 | EOF | |
5093 | ||
94b18763 | 5094 | jq "$jqfilter" $dir/json | jq '.inconsistents' | python -c "$sortkeys" > $dir/csjson |
3a9019d9 FG |
5095 | diff ${DIFFCOLOPTS} $dir/checkcsjson $dir/csjson || test $getjson = "yes" || return 1 |
5096 | if test $getjson = "yes" | |
5097 | then | |
5098 | jq '.' $dir/json > save6.json | |
5099 | fi | |
5100 | ||
94b18763 | 5101 | if test "$LOCALRUN" = "yes" && which jsonschema > /dev/null; |
3a9019d9 FG |
5102 | then |
5103 | jsonschema -i $dir/json $CEPH_ROOT/doc/rados/command/list-inconsistent-obj.json || return 1 | |
5104 | fi | |
5105 | ||
5106 | rados rmpool $poolname $poolname --yes-i-really-really-mean-it | |
5107 | teardown $dir || return 1 | |
5108 | } | |
7c673cae FG |
5109 | |
5110 | main osd-scrub-repair "$@" | |
5111 | ||
5112 | # Local Variables: | |
28e407b8 AA |
5113 | # compile-command: "cd build ; make -j4 && \ |
5114 | # ../qa/run-standalone.sh osd-scrub-repair.sh" | |
7c673cae | 5115 | # End: |