]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | """ |
2 | Test Object locations going down | |
3 | """ | |
4 | import logging | |
7c673cae FG |
5 | import time |
6 | from teuthology import misc as teuthology | |
e306af50 TL |
7 | from tasks import ceph_manager |
8 | from tasks.util.rados import rados | |
7c673cae FG |
9 | |
10 | log = logging.getLogger(__name__) | |
11 | ||
12 | def task(ctx, config): | |
13 | """ | |
14 | Test handling of object location going down | |
15 | """ | |
16 | if config is None: | |
17 | config = {} | |
18 | assert isinstance(config, dict), \ | |
19 | 'lost_unfound task only accepts a dict for configuration' | |
20 | first_mon = teuthology.get_first_mon(ctx, config) | |
9f95a23c | 21 | (mon,) = ctx.cluster.only(first_mon).remotes.keys() |
7c673cae FG |
22 | |
23 | manager = ceph_manager.CephManager( | |
24 | mon, | |
25 | ctx=ctx, | |
26 | logger=log.getChild('ceph_manager'), | |
27 | ) | |
28 | ||
29 | while len(manager.get_osd_status()['up']) < 3: | |
30 | time.sleep(10) | |
31 | manager.wait_for_clean() | |
32 | ||
33 | # something that is always there | |
34 | dummyfile = '/etc/fstab' | |
35 | ||
36 | # take 0, 1 out | |
37 | manager.mark_out_osd(0) | |
38 | manager.mark_out_osd(1) | |
39 | manager.wait_for_clean() | |
40 | ||
41 | # delay recovery, and make the pg log very long (to prevent backfill) | |
42 | manager.raw_cluster_cmd( | |
43 | 'tell', 'osd.0', | |
44 | 'injectargs', | |
45 | '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000' | |
46 | ) | |
47 | # delay recovery, and make the pg log very long (to prevent backfill) | |
48 | manager.raw_cluster_cmd( | |
49 | 'tell', 'osd.1', | |
50 | 'injectargs', | |
51 | '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000' | |
52 | ) | |
53 | # delay recovery, and make the pg log very long (to prevent backfill) | |
54 | manager.raw_cluster_cmd( | |
55 | 'tell', 'osd.2', | |
56 | 'injectargs', | |
57 | '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000' | |
58 | ) | |
59 | # delay recovery, and make the pg log very long (to prevent backfill) | |
60 | manager.raw_cluster_cmd( | |
61 | 'tell', 'osd.3', | |
62 | 'injectargs', | |
63 | '--osd-recovery-delay-start 10000 --osd-min-pg-log-entries 100000000' | |
64 | ) | |
65 | ||
66 | # kludge to make sure they get a map | |
67 | rados(ctx, mon, ['-p', 'data', 'put', 'dummy', dummyfile]) | |
68 | ||
69 | # create old objects | |
70 | for f in range(1, 10): | |
71 | rados(ctx, mon, ['-p', 'data', 'put', 'existing_%d' % f, dummyfile]) | |
72 | ||
73 | manager.mark_out_osd(3) | |
74 | manager.wait_till_active() | |
75 | ||
76 | manager.mark_in_osd(0) | |
77 | manager.wait_till_active() | |
78 | ||
31f18b77 | 79 | manager.flush_pg_stats([2, 0]) |
7c673cae FG |
80 | |
81 | manager.mark_out_osd(2) | |
82 | manager.wait_till_active() | |
83 | ||
84 | # bring up 1 | |
85 | manager.mark_in_osd(1) | |
86 | manager.wait_till_active() | |
87 | ||
31f18b77 | 88 | manager.flush_pg_stats([0, 1]) |
7c673cae FG |
89 | log.info("Getting unfound objects") |
90 | unfound = manager.get_num_unfound_objects() | |
91 | assert not unfound | |
92 | ||
93 | manager.kill_osd(2) | |
94 | manager.mark_down_osd(2) | |
95 | manager.kill_osd(3) | |
96 | manager.mark_down_osd(3) | |
97 | ||
31f18b77 | 98 | manager.flush_pg_stats([0, 1]) |
7c673cae FG |
99 | log.info("Getting unfound objects") |
100 | unfound = manager.get_num_unfound_objects() | |
101 | assert unfound |