]>
Commit | Line | Data |
---|---|---|
7c673cae FG |
1 | // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- |
2 | // vim: ts=8 sw=2 smarttab | |
3 | /* | |
4 | * Ceph - scalable distributed file system | |
5 | * | |
6 | * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net> | |
7 | * | |
8 | * This is free software; you can redistribute it and/or | |
9 | * modify it under the terms of the GNU Lesser General Public | |
10 | * License version 2.1, as published by the Free Software | |
11 | * Foundation. See file COPYING. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <sys/types.h> | |
16 | #include <sys/stat.h> | |
17 | #include <fcntl.h> | |
18 | #include <boost/scoped_ptr.hpp> | |
19 | ||
20 | #include <iostream> | |
21 | #include <string> | |
22 | using namespace std; | |
23 | ||
24 | #include "osd/OSD.h" | |
25 | #include "os/ObjectStore.h" | |
26 | #include "mon/MonClient.h" | |
27 | #include "include/ceph_features.h" | |
28 | ||
29 | #include "common/config.h" | |
30 | ||
31 | #include "mon/MonMap.h" | |
32 | ||
33 | #include "msg/Messenger.h" | |
34 | ||
35 | #include "common/Timer.h" | |
36 | #include "common/TracepointProvider.h" | |
37 | #include "common/ceph_argparse.h" | |
38 | ||
39 | #include "global/global_init.h" | |
40 | #include "global/signal_handler.h" | |
41 | ||
42 | #include "include/color.h" | |
43 | #include "common/errno.h" | |
44 | #include "common/pick_address.h" | |
45 | ||
46 | #include "perfglue/heap_profiler.h" | |
47 | ||
48 | #include "include/assert.h" | |
49 | ||
50 | #define dout_context g_ceph_context | |
51 | #define dout_subsys ceph_subsys_osd | |
52 | ||
53 | namespace { | |
54 | ||
55 | TracepointProvider::Traits osd_tracepoint_traits("libosd_tp.so", | |
56 | "osd_tracing"); | |
57 | TracepointProvider::Traits os_tracepoint_traits("libos_tp.so", | |
58 | "osd_objectstore_tracing"); | |
31f18b77 FG |
59 | #ifdef WITH_OSD_INSTRUMENT_FUNCTIONS |
60 | TracepointProvider::Traits cyg_profile_traits("libcyg_profile_tp.so", | |
61 | "osd_function_tracing"); | |
62 | #endif | |
7c673cae FG |
63 | |
64 | } // anonymous namespace | |
65 | ||
66 | OSD *osd = NULL; | |
67 | ||
68 | void handle_osd_signal(int signum) | |
69 | { | |
70 | if (osd) | |
71 | osd->handle_signal(signum); | |
72 | } | |
73 | ||
74 | static void usage() | |
75 | { | |
31f18b77 | 76 | cout << "usage: ceph-osd -i <ID> [flags]\n" |
7c673cae FG |
77 | << " --osd-data PATH data directory\n" |
78 | << " --osd-journal PATH\n" | |
79 | << " journal file or block device\n" | |
80 | << " --mkfs create a [new] data directory\n" | |
31f18b77 | 81 | << " --mkkey generate a new secret key. This is normally used in combination with --mkfs\n" |
7c673cae FG |
82 | << " --convert-filestore\n" |
83 | << " run any pending upgrade operations\n" | |
84 | << " --flush-journal flush all data out of journal\n" | |
85 | << " --mkjournal initialize a new journal\n" | |
86 | << " --check-wants-journal\n" | |
87 | << " check whether a journal is desired\n" | |
88 | << " --check-allows-journal\n" | |
89 | << " check whether a journal is allowed\n" | |
90 | << " --check-needs-journal\n" | |
91 | << " check whether a journal is required\n" | |
92 | << " --debug_osd <N> set debug level (e.g. 10)\n" | |
93 | << " --get-device-fsid PATH\n" | |
94 | << " get OSD fsid for the given block device\n" | |
95 | << std::endl; | |
96 | generic_server_usage(); | |
97 | } | |
98 | ||
99 | #ifdef BUILDING_FOR_EMBEDDED | |
100 | void cephd_preload_embedded_plugins(); | |
101 | void cephd_preload_rados_classes(OSD *osd); | |
102 | extern "C" int cephd_osd(int argc, const char **argv) | |
103 | #else | |
104 | int main(int argc, const char **argv) | |
105 | #endif | |
106 | { | |
107 | vector<const char*> args; | |
108 | argv_to_vec(argc, argv, args); | |
109 | env_to_vec(args); | |
110 | ||
111 | vector<const char*> def_args; | |
112 | // We want to enable leveldb's log, while allowing users to override this | |
113 | // option, therefore we will pass it as a default argument to global_init(). | |
114 | def_args.push_back("--leveldb-log="); | |
115 | ||
116 | auto cct = global_init(&def_args, args, CEPH_ENTITY_TYPE_OSD, | |
117 | CODE_ENVIRONMENT_DAEMON, | |
118 | 0, "osd_data"); | |
119 | ceph_heap_profiler_init(); | |
120 | ||
121 | // osd specific args | |
122 | bool mkfs = false; | |
123 | bool mkjournal = false; | |
124 | bool check_wants_journal = false; | |
125 | bool check_allows_journal = false; | |
126 | bool check_needs_journal = false; | |
127 | bool mkkey = false; | |
128 | bool flushjournal = false; | |
129 | bool dump_journal = false; | |
130 | bool convertfilestore = false; | |
131 | bool get_osd_fsid = false; | |
132 | bool get_cluster_fsid = false; | |
133 | bool get_journal_fsid = false; | |
134 | bool get_device_fsid = false; | |
135 | string device_path; | |
136 | std::string dump_pg_log; | |
137 | ||
138 | std::string val; | |
139 | for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) { | |
140 | if (ceph_argparse_double_dash(args, i)) { | |
141 | break; | |
142 | } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) { | |
143 | usage(); | |
144 | } else if (ceph_argparse_flag(args, i, "--mkfs", (char*)NULL)) { | |
145 | mkfs = true; | |
146 | } else if (ceph_argparse_flag(args, i, "--mkjournal", (char*)NULL)) { | |
147 | mkjournal = true; | |
148 | } else if (ceph_argparse_flag(args, i, "--check-allows-journal", (char*)NULL)) { | |
149 | check_allows_journal = true; | |
150 | } else if (ceph_argparse_flag(args, i, "--check-wants-journal", (char*)NULL)) { | |
151 | check_wants_journal = true; | |
152 | } else if (ceph_argparse_flag(args, i, "--check-needs-journal", (char*)NULL)) { | |
153 | check_needs_journal = true; | |
154 | } else if (ceph_argparse_flag(args, i, "--mkkey", (char*)NULL)) { | |
155 | mkkey = true; | |
156 | } else if (ceph_argparse_flag(args, i, "--flush-journal", (char*)NULL)) { | |
157 | flushjournal = true; | |
158 | } else if (ceph_argparse_flag(args, i, "--convert-filestore", (char*)NULL)) { | |
159 | convertfilestore = true; | |
160 | } else if (ceph_argparse_witharg(args, i, &val, "--dump-pg-log", (char*)NULL)) { | |
161 | dump_pg_log = val; | |
162 | } else if (ceph_argparse_flag(args, i, "--dump-journal", (char*)NULL)) { | |
163 | dump_journal = true; | |
164 | } else if (ceph_argparse_flag(args, i, "--get-cluster-fsid", (char*)NULL)) { | |
165 | get_cluster_fsid = true; | |
166 | } else if (ceph_argparse_flag(args, i, "--get-osd-fsid", "--get-osd-uuid", (char*)NULL)) { | |
167 | get_osd_fsid = true; | |
168 | } else if (ceph_argparse_flag(args, i, "--get-journal-fsid", "--get-journal-uuid", (char*)NULL)) { | |
169 | get_journal_fsid = true; | |
170 | } else if (ceph_argparse_witharg(args, i, &device_path, | |
171 | "--get-device-fsid", (char*)NULL)) { | |
172 | get_device_fsid = true; | |
173 | } else { | |
174 | ++i; | |
175 | } | |
176 | } | |
177 | if (!args.empty()) { | |
178 | derr << "unrecognized arg " << args[0] << dendl; | |
179 | usage(); | |
180 | } | |
181 | ||
182 | if (get_journal_fsid) { | |
183 | device_path = g_conf->osd_journal; | |
184 | get_device_fsid = true; | |
185 | } | |
186 | if (get_device_fsid) { | |
187 | uuid_d uuid; | |
188 | int r = ObjectStore::probe_block_device_fsid(g_ceph_context, device_path, | |
189 | &uuid); | |
190 | if (r < 0) { | |
191 | cerr << "failed to get device fsid for " << device_path | |
192 | << ": " << cpp_strerror(r) << std::endl; | |
193 | exit(1); | |
194 | } | |
195 | cout << uuid << std::endl; | |
196 | return 0; | |
197 | } | |
198 | ||
199 | if (!dump_pg_log.empty()) { | |
200 | common_init_finish(g_ceph_context); | |
201 | bufferlist bl; | |
202 | std::string error; | |
203 | int r = bl.read_file(dump_pg_log.c_str(), &error); | |
204 | if (r >= 0) { | |
205 | pg_log_entry_t e; | |
206 | bufferlist::iterator p = bl.begin(); | |
207 | while (!p.end()) { | |
208 | uint64_t pos = p.get_off(); | |
209 | try { | |
210 | ::decode(e, p); | |
211 | } | |
212 | catch (const buffer::error &e) { | |
213 | derr << "failed to decode LogEntry at offset " << pos << dendl; | |
214 | return 1; | |
215 | } | |
216 | derr << pos << ":\t" << e << dendl; | |
217 | } | |
218 | } else { | |
219 | derr << "unable to open " << dump_pg_log << ": " << error << dendl; | |
220 | } | |
221 | return 0; | |
222 | } | |
223 | ||
224 | // whoami | |
225 | char *end; | |
226 | const char *id = g_conf->name.get_id().c_str(); | |
227 | int whoami = strtol(id, &end, 10); | |
228 | if (*end || end == id || whoami < 0) { | |
229 | derr << "must specify '-i #' where # is the osd number" << dendl; | |
230 | usage(); | |
231 | } | |
232 | ||
233 | if (g_conf->osd_data.empty()) { | |
234 | derr << "must specify '--osd-data=foo' data path" << dendl; | |
235 | usage(); | |
236 | } | |
237 | ||
238 | // the store | |
239 | string store_type = g_conf->osd_objectstore; | |
240 | { | |
241 | char fn[PATH_MAX]; | |
242 | snprintf(fn, sizeof(fn), "%s/type", g_conf->osd_data.c_str()); | |
243 | int fd = ::open(fn, O_RDONLY); | |
244 | if (fd >= 0) { | |
245 | bufferlist bl; | |
246 | bl.read_fd(fd, 64); | |
247 | if (bl.length()) { | |
248 | store_type = string(bl.c_str(), bl.length() - 1); // drop \n | |
d2e6a577 | 249 | g_conf->set_val("osd_objectstore", store_type); |
7c673cae FG |
250 | dout(5) << "object store type is " << store_type << dendl; |
251 | } | |
252 | ::close(fd); | |
253 | } | |
254 | } | |
255 | ObjectStore *store = ObjectStore::create(g_ceph_context, | |
256 | store_type, | |
257 | g_conf->osd_data, | |
258 | g_conf->osd_journal, | |
259 | g_conf->osd_os_flags); | |
260 | if (!store) { | |
261 | derr << "unable to create object store" << dendl; | |
262 | return -ENODEV; | |
263 | } | |
264 | ||
265 | #ifdef BUILDING_FOR_EMBEDDED | |
266 | cephd_preload_embedded_plugins(); | |
267 | #endif | |
268 | ||
269 | if (mkfs) { | |
270 | common_init_finish(g_ceph_context); | |
271 | MonClient mc(g_ceph_context); | |
272 | if (mc.build_initial_monmap() < 0) | |
273 | return -1; | |
274 | if (mc.get_monmap_privately() < 0) | |
275 | return -1; | |
276 | ||
277 | if (mc.monmap.fsid.is_zero()) { | |
278 | derr << "must specify cluster fsid" << dendl; | |
279 | return -EINVAL; | |
280 | } | |
281 | ||
282 | int err = OSD::mkfs(g_ceph_context, store, g_conf->osd_data, | |
283 | mc.monmap.fsid, whoami); | |
284 | if (err < 0) { | |
285 | derr << TEXT_RED << " ** ERROR: error creating empty object store in " | |
286 | << g_conf->osd_data << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
287 | exit(1); | |
288 | } | |
289 | derr << "created object store " << g_conf->osd_data | |
290 | << " for osd." << whoami << " fsid " << mc.monmap.fsid << dendl; | |
291 | } | |
292 | if (mkkey) { | |
293 | common_init_finish(g_ceph_context); | |
294 | KeyRing *keyring = KeyRing::create_empty(); | |
295 | if (!keyring) { | |
296 | derr << "Unable to get a Ceph keyring." << dendl; | |
297 | return 1; | |
298 | } | |
299 | ||
300 | EntityName ename(g_conf->name); | |
301 | EntityAuth eauth; | |
302 | ||
303 | int ret = keyring->load(g_ceph_context, g_conf->keyring); | |
304 | if (ret == 0 && | |
305 | keyring->get_auth(ename, eauth)) { | |
306 | derr << "already have key in keyring " << g_conf->keyring << dendl; | |
307 | } else { | |
308 | eauth.key.create(g_ceph_context, CEPH_CRYPTO_AES); | |
309 | keyring->add(ename, eauth); | |
310 | bufferlist bl; | |
311 | keyring->encode_plaintext(bl); | |
312 | int r = bl.write_file(g_conf->keyring.c_str(), 0600); | |
313 | if (r) | |
314 | derr << TEXT_RED << " ** ERROR: writing new keyring to " << g_conf->keyring | |
315 | << ": " << cpp_strerror(r) << TEXT_NORMAL << dendl; | |
316 | else | |
317 | derr << "created new key in keyring " << g_conf->keyring << dendl; | |
318 | } | |
319 | } | |
320 | if (mkfs || mkkey) | |
321 | exit(0); | |
322 | if (mkjournal) { | |
323 | common_init_finish(g_ceph_context); | |
324 | int err = store->mkjournal(); | |
325 | if (err < 0) { | |
326 | derr << TEXT_RED << " ** ERROR: error creating fresh journal " << g_conf->osd_journal | |
327 | << " for object store " << g_conf->osd_data | |
328 | << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
329 | exit(1); | |
330 | } | |
331 | derr << "created new journal " << g_conf->osd_journal | |
332 | << " for object store " << g_conf->osd_data << dendl; | |
333 | exit(0); | |
334 | } | |
335 | if (check_wants_journal) { | |
336 | if (store->wants_journal()) { | |
d2e6a577 | 337 | cout << "wants journal: yes" << std::endl; |
7c673cae FG |
338 | exit(0); |
339 | } else { | |
d2e6a577 | 340 | cout << "wants journal: no" << std::endl; |
7c673cae FG |
341 | exit(1); |
342 | } | |
343 | } | |
344 | if (check_allows_journal) { | |
345 | if (store->allows_journal()) { | |
d2e6a577 | 346 | cout << "allows journal: yes" << std::endl; |
7c673cae FG |
347 | exit(0); |
348 | } else { | |
d2e6a577 | 349 | cout << "allows journal: no" << std::endl; |
7c673cae FG |
350 | exit(1); |
351 | } | |
352 | } | |
353 | if (check_needs_journal) { | |
354 | if (store->needs_journal()) { | |
d2e6a577 | 355 | cout << "needs journal: yes" << std::endl; |
7c673cae FG |
356 | exit(0); |
357 | } else { | |
d2e6a577 | 358 | cout << "needs journal: no" << std::endl; |
7c673cae FG |
359 | exit(1); |
360 | } | |
361 | } | |
362 | if (flushjournal) { | |
363 | common_init_finish(g_ceph_context); | |
364 | int err = store->mount(); | |
365 | if (err < 0) { | |
366 | derr << TEXT_RED << " ** ERROR: error flushing journal " << g_conf->osd_journal | |
367 | << " for object store " << g_conf->osd_data | |
368 | << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
369 | goto flushjournal_out; | |
370 | } | |
371 | store->umount(); | |
372 | derr << "flushed journal " << g_conf->osd_journal | |
373 | << " for object store " << g_conf->osd_data | |
374 | << dendl; | |
375 | flushjournal_out: | |
376 | delete store; | |
377 | exit(err < 0 ? 1 : 0); | |
378 | } | |
379 | if (dump_journal) { | |
380 | common_init_finish(g_ceph_context); | |
381 | int err = store->dump_journal(cout); | |
382 | if (err < 0) { | |
383 | derr << TEXT_RED << " ** ERROR: error dumping journal " << g_conf->osd_journal | |
384 | << " for object store " << g_conf->osd_data | |
385 | << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
386 | exit(1); | |
387 | } | |
388 | derr << "dumped journal " << g_conf->osd_journal | |
389 | << " for object store " << g_conf->osd_data | |
390 | << dendl; | |
391 | exit(0); | |
392 | ||
393 | } | |
394 | ||
395 | ||
396 | if (convertfilestore) { | |
397 | int err = store->mount(); | |
398 | if (err < 0) { | |
399 | derr << TEXT_RED << " ** ERROR: error mounting store " << g_conf->osd_data | |
400 | << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
401 | exit(1); | |
402 | } | |
403 | err = store->upgrade(); | |
404 | store->umount(); | |
405 | if (err < 0) { | |
406 | derr << TEXT_RED << " ** ERROR: error converting store " << g_conf->osd_data | |
407 | << ": " << cpp_strerror(-err) << TEXT_NORMAL << dendl; | |
408 | exit(1); | |
409 | } | |
410 | exit(0); | |
411 | } | |
412 | ||
413 | string magic; | |
414 | uuid_d cluster_fsid, osd_fsid; | |
415 | int w; | |
416 | int r = OSD::peek_meta(store, magic, cluster_fsid, osd_fsid, w); | |
417 | if (r < 0) { | |
418 | derr << TEXT_RED << " ** ERROR: unable to open OSD superblock on " | |
419 | << g_conf->osd_data << ": " << cpp_strerror(-r) | |
420 | << TEXT_NORMAL << dendl; | |
421 | if (r == -ENOTSUP) { | |
422 | derr << TEXT_RED << " ** please verify that underlying storage " | |
423 | << "supports xattrs" << TEXT_NORMAL << dendl; | |
424 | } | |
425 | exit(1); | |
426 | } | |
427 | if (w != whoami) { | |
428 | derr << "OSD id " << w << " != my id " << whoami << dendl; | |
429 | exit(1); | |
430 | } | |
431 | if (strcmp(magic.c_str(), CEPH_OSD_ONDISK_MAGIC)) { | |
432 | derr << "OSD magic " << magic << " != my " << CEPH_OSD_ONDISK_MAGIC | |
433 | << dendl; | |
434 | exit(1); | |
435 | } | |
436 | ||
437 | if (get_cluster_fsid) { | |
438 | cout << cluster_fsid << std::endl; | |
439 | exit(0); | |
440 | } | |
441 | if (get_osd_fsid) { | |
442 | cout << osd_fsid << std::endl; | |
443 | exit(0); | |
444 | } | |
445 | ||
446 | pick_addresses(g_ceph_context, CEPH_PICK_ADDRESS_PUBLIC | |
447 | |CEPH_PICK_ADDRESS_CLUSTER); | |
448 | ||
449 | if (g_conf->public_addr.is_blank_ip() && !g_conf->cluster_addr.is_blank_ip()) { | |
450 | derr << TEXT_YELLOW | |
451 | << " ** WARNING: specified cluster addr but not public addr; we recommend **\n" | |
452 | << " ** you specify neither or both. **" | |
453 | << TEXT_NORMAL << dendl; | |
454 | } | |
455 | ||
456 | std::string public_msgr_type = g_conf->ms_public_type.empty() ? g_conf->get_val<std::string>("ms_type") : g_conf->ms_public_type; | |
457 | std::string cluster_msgr_type = g_conf->ms_cluster_type.empty() ? g_conf->get_val<std::string>("ms_type") : g_conf->ms_cluster_type; | |
458 | Messenger *ms_public = Messenger::create(g_ceph_context, public_msgr_type, | |
459 | entity_name_t::OSD(whoami), "client", | |
460 | getpid(), | |
461 | Messenger::HAS_HEAVY_TRAFFIC | | |
462 | Messenger::HAS_MANY_CONNECTIONS); | |
463 | Messenger *ms_cluster = Messenger::create(g_ceph_context, cluster_msgr_type, | |
464 | entity_name_t::OSD(whoami), "cluster", | |
465 | getpid(), | |
466 | Messenger::HAS_HEAVY_TRAFFIC | | |
467 | Messenger::HAS_MANY_CONNECTIONS); | |
468 | Messenger *ms_hb_back_client = Messenger::create(g_ceph_context, cluster_msgr_type, | |
469 | entity_name_t::OSD(whoami), "hb_back_client", | |
470 | getpid(), Messenger::HEARTBEAT); | |
471 | Messenger *ms_hb_front_client = Messenger::create(g_ceph_context, public_msgr_type, | |
472 | entity_name_t::OSD(whoami), "hb_front_client", | |
473 | getpid(), Messenger::HEARTBEAT); | |
474 | Messenger *ms_hb_back_server = Messenger::create(g_ceph_context, cluster_msgr_type, | |
475 | entity_name_t::OSD(whoami), "hb_back_server", | |
476 | getpid(), Messenger::HEARTBEAT); | |
477 | Messenger *ms_hb_front_server = Messenger::create(g_ceph_context, public_msgr_type, | |
478 | entity_name_t::OSD(whoami), "hb_front_server", | |
479 | getpid(), Messenger::HEARTBEAT); | |
480 | Messenger *ms_objecter = Messenger::create(g_ceph_context, public_msgr_type, | |
481 | entity_name_t::OSD(whoami), "ms_objecter", | |
482 | getpid(), 0); | |
483 | if (!ms_public || !ms_cluster || !ms_hb_front_client || !ms_hb_back_client || !ms_hb_back_server || !ms_hb_front_server || !ms_objecter) | |
484 | exit(1); | |
485 | ms_cluster->set_cluster_protocol(CEPH_OSD_PROTOCOL); | |
486 | ms_hb_front_client->set_cluster_protocol(CEPH_OSD_PROTOCOL); | |
487 | ms_hb_back_client->set_cluster_protocol(CEPH_OSD_PROTOCOL); | |
488 | ms_hb_back_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); | |
489 | ms_hb_front_server->set_cluster_protocol(CEPH_OSD_PROTOCOL); | |
490 | ||
491 | cout << "starting osd." << whoami | |
492 | << " at " << ms_public->get_myaddr() | |
493 | << " osd_data " << g_conf->osd_data | |
494 | << " " << ((g_conf->osd_journal.empty()) ? | |
495 | "(no journal)" : g_conf->osd_journal) | |
496 | << std::endl; | |
497 | ||
498 | boost::scoped_ptr<Throttle> client_byte_throttler( | |
499 | new Throttle(g_ceph_context, "osd_client_bytes", | |
500 | g_conf->osd_client_message_size_cap)); | |
7c673cae FG |
501 | |
502 | // All feature bits 0 - 34 should be present from dumpling v0.67 forward | |
503 | uint64_t osd_required = | |
504 | CEPH_FEATURE_UID | | |
505 | CEPH_FEATURE_PGID64 | | |
506 | CEPH_FEATURE_OSDENC; | |
507 | ||
508 | ms_public->set_default_policy(Messenger::Policy::stateless_server(0)); | |
509 | ms_public->set_policy_throttlers(entity_name_t::TYPE_CLIENT, | |
510 | client_byte_throttler.get(), | |
31f18b77 | 511 | nullptr); |
7c673cae FG |
512 | ms_public->set_policy(entity_name_t::TYPE_MON, |
513 | Messenger::Policy::lossy_client(CEPH_FEATURE_UID | | |
514 | CEPH_FEATURE_PGID64 | | |
515 | CEPH_FEATURE_OSDENC)); | |
516 | ms_public->set_policy(entity_name_t::TYPE_MGR, | |
517 | Messenger::Policy::lossy_client(CEPH_FEATURE_UID | | |
518 | CEPH_FEATURE_PGID64 | | |
519 | CEPH_FEATURE_OSDENC)); | |
520 | ||
521 | //try to poison pill any OSD connections on the wrong address | |
522 | ms_public->set_policy(entity_name_t::TYPE_OSD, | |
523 | Messenger::Policy::stateless_server(0)); | |
524 | ||
525 | ms_cluster->set_default_policy(Messenger::Policy::stateless_server(0)); | |
526 | ms_cluster->set_policy(entity_name_t::TYPE_MON, Messenger::Policy::lossy_client(0)); | |
527 | ms_cluster->set_policy(entity_name_t::TYPE_OSD, | |
528 | Messenger::Policy::lossless_peer(osd_required)); | |
529 | ms_cluster->set_policy(entity_name_t::TYPE_CLIENT, | |
530 | Messenger::Policy::stateless_server(0)); | |
531 | ||
532 | ms_hb_front_client->set_policy(entity_name_t::TYPE_OSD, | |
533 | Messenger::Policy::lossy_client(0)); | |
534 | ms_hb_back_client->set_policy(entity_name_t::TYPE_OSD, | |
535 | Messenger::Policy::lossy_client(0)); | |
536 | ms_hb_back_server->set_policy(entity_name_t::TYPE_OSD, | |
537 | Messenger::Policy::stateless_server(0)); | |
538 | ms_hb_front_server->set_policy(entity_name_t::TYPE_OSD, | |
539 | Messenger::Policy::stateless_server(0)); | |
540 | ||
541 | ms_objecter->set_default_policy(Messenger::Policy::lossy_client(CEPH_FEATURE_OSDREPLYMUX)); | |
542 | ||
543 | r = ms_public->bind(g_conf->public_addr); | |
544 | if (r < 0) | |
545 | exit(1); | |
546 | r = ms_cluster->bind(g_conf->cluster_addr); | |
547 | if (r < 0) | |
548 | exit(1); | |
549 | ||
550 | if (g_conf->osd_heartbeat_use_min_delay_socket) { | |
551 | ms_hb_front_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); | |
552 | ms_hb_back_client->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); | |
553 | ms_hb_back_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); | |
554 | ms_hb_front_server->set_socket_priority(SOCKET_PRIORITY_MIN_DELAY); | |
555 | } | |
556 | ||
557 | // hb back should bind to same ip as cluster_addr (if specified) | |
558 | entity_addr_t hb_back_addr = g_conf->osd_heartbeat_addr; | |
559 | if (hb_back_addr.is_blank_ip()) { | |
560 | hb_back_addr = g_conf->cluster_addr; | |
561 | if (hb_back_addr.is_ip()) | |
562 | hb_back_addr.set_port(0); | |
563 | } | |
564 | r = ms_hb_back_server->bind(hb_back_addr); | |
565 | if (r < 0) | |
566 | exit(1); | |
567 | r = ms_hb_back_client->client_bind(hb_back_addr); | |
568 | if (r < 0) | |
569 | exit(1); | |
570 | ||
571 | // hb front should bind to same ip as public_addr | |
572 | entity_addr_t hb_front_addr = g_conf->public_addr; | |
573 | if (hb_front_addr.is_ip()) | |
574 | hb_front_addr.set_port(0); | |
575 | r = ms_hb_front_server->bind(hb_front_addr); | |
576 | if (r < 0) | |
577 | exit(1); | |
578 | r = ms_hb_front_client->client_bind(hb_front_addr); | |
579 | if (r < 0) | |
580 | exit(1); | |
581 | ||
582 | // Set up crypto, daemonize, etc. | |
583 | global_init_daemonize(g_ceph_context); | |
584 | common_init_finish(g_ceph_context); | |
585 | ||
586 | TracepointProvider::initialize<osd_tracepoint_traits>(g_ceph_context); | |
587 | TracepointProvider::initialize<os_tracepoint_traits>(g_ceph_context); | |
31f18b77 FG |
588 | #ifdef WITH_OSD_INSTRUMENT_FUNCTIONS |
589 | TracepointProvider::initialize<cyg_profile_traits>(g_ceph_context); | |
590 | #endif | |
7c673cae FG |
591 | |
592 | MonClient mc(g_ceph_context); | |
593 | if (mc.build_initial_monmap() < 0) | |
594 | return -1; | |
595 | global_init_chdir(g_ceph_context); | |
596 | ||
597 | #ifndef BUILDING_FOR_EMBEDDED | |
598 | if (global_init_preload_erasure_code(g_ceph_context) < 0) | |
599 | return -1; | |
600 | #endif | |
601 | ||
224ce89b WB |
602 | srand(time(NULL) + getpid()); |
603 | ||
7c673cae FG |
604 | osd = new OSD(g_ceph_context, |
605 | store, | |
606 | whoami, | |
607 | ms_cluster, | |
608 | ms_public, | |
609 | ms_hb_front_client, | |
610 | ms_hb_back_client, | |
611 | ms_hb_front_server, | |
612 | ms_hb_back_server, | |
613 | ms_objecter, | |
614 | &mc, | |
615 | g_conf->osd_data, | |
616 | g_conf->osd_journal); | |
617 | ||
618 | int err = osd->pre_init(); | |
619 | if (err < 0) { | |
620 | derr << TEXT_RED << " ** ERROR: osd pre_init failed: " << cpp_strerror(-err) | |
621 | << TEXT_NORMAL << dendl; | |
622 | return 1; | |
623 | } | |
624 | ||
625 | ms_public->start(); | |
626 | ms_hb_front_client->start(); | |
627 | ms_hb_back_client->start(); | |
628 | ms_hb_front_server->start(); | |
629 | ms_hb_back_server->start(); | |
630 | ms_cluster->start(); | |
631 | ms_objecter->start(); | |
632 | ||
633 | // start osd | |
634 | err = osd->init(); | |
635 | if (err < 0) { | |
636 | derr << TEXT_RED << " ** ERROR: osd init failed: " << cpp_strerror(-err) | |
637 | << TEXT_NORMAL << dendl; | |
638 | return 1; | |
639 | } | |
640 | ||
641 | #ifdef BUILDING_FOR_EMBEDDED | |
642 | cephd_preload_rados_classes(osd); | |
643 | #endif | |
644 | ||
645 | // install signal handlers | |
646 | init_async_signal_handler(); | |
647 | register_async_signal_handler(SIGHUP, sighup_handler); | |
648 | register_async_signal_handler_oneshot(SIGINT, handle_osd_signal); | |
649 | register_async_signal_handler_oneshot(SIGTERM, handle_osd_signal); | |
650 | ||
651 | osd->final_init(); | |
652 | ||
653 | if (g_conf->inject_early_sigterm) | |
654 | kill(getpid(), SIGTERM); | |
655 | ||
656 | ms_public->wait(); | |
657 | ms_hb_front_client->wait(); | |
658 | ms_hb_back_client->wait(); | |
659 | ms_hb_front_server->wait(); | |
660 | ms_hb_back_server->wait(); | |
661 | ms_cluster->wait(); | |
662 | ms_objecter->wait(); | |
663 | ||
664 | unregister_async_signal_handler(SIGHUP, sighup_handler); | |
665 | unregister_async_signal_handler(SIGINT, handle_osd_signal); | |
666 | unregister_async_signal_handler(SIGTERM, handle_osd_signal); | |
667 | shutdown_async_signal_handler(); | |
668 | ||
669 | // done | |
670 | delete osd; | |
671 | delete ms_public; | |
672 | delete ms_hb_front_client; | |
673 | delete ms_hb_back_client; | |
674 | delete ms_hb_front_server; | |
675 | delete ms_hb_back_server; | |
676 | delete ms_cluster; | |
677 | delete ms_objecter; | |
678 | ||
679 | client_byte_throttler.reset(); | |
7c673cae FG |
680 | |
681 | // cd on exit, so that gmon.out (if any) goes into a separate directory for each node. | |
682 | char s[20]; | |
683 | snprintf(s, sizeof(s), "gmon/%d", getpid()); | |
684 | if ((mkdir(s, 0755) == 0) && (chdir(s) == 0)) { | |
685 | dout(0) << "ceph-osd: gmon.out should be in " << s << dendl; | |
686 | } | |
687 | ||
688 | return 0; | |
689 | } |