]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | # Copyright (c) 2011-present, Facebook, Inc. All rights reserved. |
2 | # This source code is licensed under both the GPLv2 (found in the | |
3 | # COPYING file in the root directory) and Apache 2.0 License | |
4 | # (found in the LICENSE.Apache file in the root directory). | |
5 | ||
1e59de90 TL |
6 | import os |
7 | import unittest | |
8 | ||
11fdf7f2 TL |
9 | from advisor.db_log_parser import NO_COL_FAMILY |
10 | from advisor.db_options_parser import DatabaseOptions | |
11 | from advisor.rule_parser import Condition, OptionCondition | |
11fdf7f2 TL |
12 | |
13 | ||
14 | class TestDatabaseOptions(unittest.TestCase): | |
15 | def setUp(self): | |
16 | self.this_path = os.path.abspath(os.path.dirname(__file__)) | |
1e59de90 TL |
17 | self.og_options = os.path.join(self.this_path, "input_files/OPTIONS-000005") |
18 | misc_options = ["bloom_bits = 4", "rate_limiter_bytes_per_sec = 1024000"] | |
11fdf7f2 TL |
19 | # create the options object |
20 | self.db_options = DatabaseOptions(self.og_options, misc_options) | |
21 | # perform clean-up before running tests | |
22 | self.generated_options = os.path.join( | |
1e59de90 | 23 | self.this_path, "../temp/OPTIONS_testing.tmp" |
11fdf7f2 TL |
24 | ) |
25 | if os.path.isfile(self.generated_options): | |
26 | os.remove(self.generated_options) | |
27 | ||
28 | def test_get_options_diff(self): | |
29 | old_opt = { | |
1e59de90 TL |
30 | "DBOptions.stats_dump_freq_sec": {NO_COL_FAMILY: "20"}, |
31 | "CFOptions.write_buffer_size": { | |
32 | "default": "1024000", | |
33 | "col_fam_A": "128000", | |
34 | "col_fam_B": "128000000", | |
11fdf7f2 | 35 | }, |
1e59de90 TL |
36 | "DBOptions.use_fsync": {NO_COL_FAMILY: "true"}, |
37 | "DBOptions.max_log_file_size": {NO_COL_FAMILY: "128000000"}, | |
11fdf7f2 TL |
38 | } |
39 | new_opt = { | |
1e59de90 TL |
40 | "bloom_bits": {NO_COL_FAMILY: "4"}, |
41 | "CFOptions.write_buffer_size": { | |
42 | "default": "128000000", | |
43 | "col_fam_A": "128000", | |
44 | "col_fam_C": "128000000", | |
11fdf7f2 | 45 | }, |
1e59de90 TL |
46 | "DBOptions.use_fsync": {NO_COL_FAMILY: "true"}, |
47 | "DBOptions.max_log_file_size": {NO_COL_FAMILY: "0"}, | |
11fdf7f2 TL |
48 | } |
49 | diff = DatabaseOptions.get_options_diff(old_opt, new_opt) | |
50 | ||
51 | expected_diff = { | |
1e59de90 TL |
52 | "DBOptions.stats_dump_freq_sec": {NO_COL_FAMILY: ("20", None)}, |
53 | "bloom_bits": {NO_COL_FAMILY: (None, "4")}, | |
54 | "CFOptions.write_buffer_size": { | |
55 | "default": ("1024000", "128000000"), | |
56 | "col_fam_B": ("128000000", None), | |
57 | "col_fam_C": (None, "128000000"), | |
11fdf7f2 | 58 | }, |
1e59de90 | 59 | "DBOptions.max_log_file_size": {NO_COL_FAMILY: ("128000000", "0")}, |
11fdf7f2 TL |
60 | } |
61 | self.assertDictEqual(diff, expected_diff) | |
62 | ||
63 | def test_is_misc_option(self): | |
1e59de90 | 64 | self.assertTrue(DatabaseOptions.is_misc_option("bloom_bits")) |
11fdf7f2 | 65 | self.assertFalse( |
1e59de90 | 66 | DatabaseOptions.is_misc_option("DBOptions.stats_dump_freq_sec") |
11fdf7f2 TL |
67 | ) |
68 | ||
69 | def test_set_up(self): | |
70 | options = self.db_options.get_all_options() | |
71 | self.assertEqual(22, len(options.keys())) | |
72 | expected_misc_options = { | |
1e59de90 TL |
73 | "bloom_bits": "4", |
74 | "rate_limiter_bytes_per_sec": "1024000", | |
11fdf7f2 | 75 | } |
1e59de90 | 76 | self.assertDictEqual(expected_misc_options, self.db_options.get_misc_options()) |
11fdf7f2 | 77 | self.assertListEqual( |
1e59de90 | 78 | ["default", "col_fam_A"], self.db_options.get_column_families() |
11fdf7f2 TL |
79 | ) |
80 | ||
81 | def test_get_options(self): | |
82 | opt_to_get = [ | |
1e59de90 TL |
83 | "DBOptions.manual_wal_flush", |
84 | "DBOptions.db_write_buffer_size", | |
85 | "bloom_bits", | |
86 | "CFOptions.compaction_filter_factory", | |
87 | "CFOptions.num_levels", | |
88 | "rate_limiter_bytes_per_sec", | |
89 | "TableOptions.BlockBasedTable.block_align", | |
90 | "random_option", | |
11fdf7f2 TL |
91 | ] |
92 | options = self.db_options.get_options(opt_to_get) | |
93 | expected_options = { | |
1e59de90 TL |
94 | "DBOptions.manual_wal_flush": {NO_COL_FAMILY: "false"}, |
95 | "DBOptions.db_write_buffer_size": {NO_COL_FAMILY: "0"}, | |
96 | "bloom_bits": {NO_COL_FAMILY: "4"}, | |
97 | "CFOptions.compaction_filter_factory": { | |
98 | "default": "nullptr", | |
99 | "col_fam_A": "nullptr", | |
100 | }, | |
101 | "CFOptions.num_levels": {"default": "7", "col_fam_A": "5"}, | |
102 | "rate_limiter_bytes_per_sec": {NO_COL_FAMILY: "1024000"}, | |
103 | "TableOptions.BlockBasedTable.block_align": { | |
104 | "default": "false", | |
105 | "col_fam_A": "true", | |
11fdf7f2 | 106 | }, |
11fdf7f2 TL |
107 | } |
108 | self.assertDictEqual(expected_options, options) | |
109 | ||
110 | def test_update_options(self): | |
111 | # add new, update old, set old | |
112 | # before updating | |
113 | expected_old_opts = { | |
1e59de90 TL |
114 | "DBOptions.db_log_dir": {NO_COL_FAMILY: None}, |
115 | "DBOptions.manual_wal_flush": {NO_COL_FAMILY: "false"}, | |
116 | "bloom_bits": {NO_COL_FAMILY: "4"}, | |
117 | "CFOptions.num_levels": {"default": "7", "col_fam_A": "5"}, | |
118 | "TableOptions.BlockBasedTable.block_restart_interval": {"col_fam_A": "16"}, | |
11fdf7f2 TL |
119 | } |
120 | get_opts = list(expected_old_opts.keys()) | |
121 | options = self.db_options.get_options(get_opts) | |
122 | self.assertEqual(expected_old_opts, options) | |
123 | # after updating options | |
124 | update_opts = { | |
1e59de90 TL |
125 | "DBOptions.db_log_dir": {NO_COL_FAMILY: "/dev/shm"}, |
126 | "DBOptions.manual_wal_flush": {NO_COL_FAMILY: "true"}, | |
127 | "bloom_bits": {NO_COL_FAMILY: "2"}, | |
128 | "CFOptions.num_levels": {"col_fam_A": "7"}, | |
129 | "TableOptions.BlockBasedTable.block_restart_interval": {"default": "32"}, | |
130 | "random_misc_option": {NO_COL_FAMILY: "something"}, | |
11fdf7f2 TL |
131 | } |
132 | self.db_options.update_options(update_opts) | |
1e59de90 TL |
133 | update_opts["CFOptions.num_levels"]["default"] = "7" |
134 | update_opts["TableOptions.BlockBasedTable.block_restart_interval"] = { | |
135 | "default": "32", | |
136 | "col_fam_A": "16", | |
11fdf7f2 | 137 | } |
1e59de90 | 138 | get_opts.append("random_misc_option") |
11fdf7f2 TL |
139 | options = self.db_options.get_options(get_opts) |
140 | self.assertDictEqual(update_opts, options) | |
141 | expected_misc_options = { | |
1e59de90 TL |
142 | "bloom_bits": "2", |
143 | "rate_limiter_bytes_per_sec": "1024000", | |
144 | "random_misc_option": "something", | |
11fdf7f2 | 145 | } |
1e59de90 | 146 | self.assertDictEqual(expected_misc_options, self.db_options.get_misc_options()) |
11fdf7f2 TL |
147 | |
148 | def test_generate_options_config(self): | |
149 | # make sure file does not exist from before | |
150 | self.assertFalse(os.path.isfile(self.generated_options)) | |
1e59de90 | 151 | self.db_options.generate_options_config("testing") |
11fdf7f2 TL |
152 | self.assertTrue(os.path.isfile(self.generated_options)) |
153 | ||
154 | def test_check_and_trigger_conditions(self): | |
155 | # options only from CFOptions | |
156 | # setup the OptionCondition objects to check and trigger | |
157 | update_dict = { | |
1e59de90 TL |
158 | "CFOptions.level0_file_num_compaction_trigger": {"col_fam_A": "4"}, |
159 | "CFOptions.max_bytes_for_level_base": {"col_fam_A": "10"}, | |
11fdf7f2 TL |
160 | } |
161 | self.db_options.update_options(update_dict) | |
1e59de90 | 162 | cond1 = Condition("opt-cond-1") |
11fdf7f2 TL |
163 | cond1 = OptionCondition.create(cond1) |
164 | cond1.set_parameter( | |
1e59de90 TL |
165 | "options", |
166 | [ | |
167 | "CFOptions.level0_file_num_compaction_trigger", | |
168 | "TableOptions.BlockBasedTable.block_restart_interval", | |
169 | "CFOptions.max_bytes_for_level_base", | |
170 | ], | |
11fdf7f2 TL |
171 | ) |
172 | cond1.set_parameter( | |
1e59de90 | 173 | "evaluate", "int(options[0])*int(options[1])-int(options[2])>=0" |
11fdf7f2 TL |
174 | ) |
175 | # only DBOptions | |
1e59de90 | 176 | cond2 = Condition("opt-cond-2") |
11fdf7f2 TL |
177 | cond2 = OptionCondition.create(cond2) |
178 | cond2.set_parameter( | |
1e59de90 TL |
179 | "options", |
180 | [ | |
181 | "DBOptions.db_write_buffer_size", | |
182 | "bloom_bits", | |
183 | "rate_limiter_bytes_per_sec", | |
184 | ], | |
11fdf7f2 TL |
185 | ) |
186 | cond2.set_parameter( | |
1e59de90 | 187 | "evaluate", "(int(options[2]) * int(options[1]) * int(options[0]))==0" |
11fdf7f2 TL |
188 | ) |
189 | # mix of CFOptions and DBOptions | |
1e59de90 | 190 | cond3 = Condition("opt-cond-3") |
11fdf7f2 TL |
191 | cond3 = OptionCondition.create(cond3) |
192 | cond3.set_parameter( | |
1e59de90 TL |
193 | "options", |
194 | [ | |
195 | "DBOptions.db_write_buffer_size", # 0 | |
196 | "CFOptions.num_levels", # 5, 7 | |
197 | "bloom_bits", # 4 | |
198 | ], | |
11fdf7f2 TL |
199 | ) |
200 | cond3.set_parameter( | |
1e59de90 | 201 | "evaluate", "int(options[2])*int(options[0])+int(options[1])>6" |
11fdf7f2 TL |
202 | ) |
203 | self.db_options.check_and_trigger_conditions([cond1, cond2, cond3]) | |
204 | ||
1e59de90 | 205 | cond1_trigger = {"col_fam_A": ["4", "16", "10"]} |
11fdf7f2 | 206 | self.assertDictEqual(cond1_trigger, cond1.get_trigger()) |
1e59de90 | 207 | cond2_trigger = {NO_COL_FAMILY: ["0", "4", "1024000"]} |
11fdf7f2 | 208 | self.assertDictEqual(cond2_trigger, cond2.get_trigger()) |
1e59de90 | 209 | cond3_trigger = {"default": ["0", "7", "4"]} |
11fdf7f2 TL |
210 | self.assertDictEqual(cond3_trigger, cond3.get_trigger()) |
211 | ||
212 | ||
1e59de90 | 213 | if __name__ == "__main__": |
11fdf7f2 | 214 | unittest.main() |