2 * (c) Copyright 2016 Hewlett Packard Enterprise Development LP
3 * Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014, 2016, 2017 Nicira, Inc.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
21 #include "condition.h"
23 #include "openvswitch/dynamic-string.h"
24 #include "openvswitch/hmap.h"
25 #include "openvswitch/json.h"
26 #include "openvswitch/vlog.h"
27 #include "ovsdb-error.h"
30 #include "replication.h"
36 #include "transaction.h"
39 VLOG_DEFINE_THIS_MODULE(replication
);
41 static char *sync_from
;
42 static struct uuid server_uuid
;
43 static struct jsonrpc_session
*session
;
44 static unsigned int session_seqno
= UINT_MAX
;
46 static struct jsonrpc_msg
*create_monitor_request(struct ovsdb
*db
);
47 static void add_monitored_table(struct ovsdb_table_schema
*table
,
48 struct json
*monitor_requests
);
50 static struct ovsdb_error
*reset_database(struct ovsdb
*db
);
52 static struct ovsdb_error
*process_notification(struct json
*, struct ovsdb
*);
53 static struct ovsdb_error
*process_table_update(struct json
*table_update
,
54 const char *table_name
,
55 struct ovsdb
*database
,
56 struct ovsdb_txn
*txn
);
58 static struct ovsdb_error
*execute_insert(struct ovsdb_txn
*txn
,
59 const struct uuid
*row_uuid
,
60 struct ovsdb_table
*table
,
62 static struct ovsdb_error
*execute_delete(struct ovsdb_txn
*txn
,
63 const struct uuid
*row_uuid
,
64 struct ovsdb_table
*table
);
65 static struct ovsdb_error
*execute_update(struct ovsdb_txn
*txn
,
66 const struct uuid
*row_uuid
,
67 struct ovsdb_table
*table
,
70 /* Maps from db name to sset of table names. */
71 static struct shash blacklist_tables
= SHASH_INITIALIZER(&blacklist_tables
);
73 static void blacklist_tables_clear(void);
74 static void blacklist_tables_add(const char *database
, const char *table
);
75 static bool blacklist_tables_find(const char *database
, const char* table
);
78 /* Keep track of request IDs of all outstanding OVSDB requests. */
79 static struct hmap request_ids
= HMAP_INITIALIZER(&request_ids
);
81 struct request_ids_hmap_node
{
82 struct hmap_node hmap
;
83 struct json
*request_id
;
84 struct ovsdb
*db
; /* associated database */
86 void request_ids_add(const struct json
*id
, struct ovsdb
*db
);
87 bool request_ids_lookup_and_free(const struct json
*id
, struct ovsdb
**db
);
88 static void request_ids_destroy(void);
89 void request_ids_clear(void);
91 enum ovsdb_replication_state
{
93 RPL_S_SERVER_ID_REQUESTED
,
95 RPL_S_SCHEMA_REQUESTED
,
96 RPL_S_MONITOR_REQUESTED
,
98 RPL_S_ERR
/* Error, no longer replicating. */
100 static enum ovsdb_replication_state state
;
103 /* All DBs known to ovsdb-server. The actual replication dbs are stored
104 * in 'replication dbs', which is a subset of all dbs and remote dbs whose
106 static struct shash local_dbs
= SHASH_INITIALIZER(&local_dbs
);
107 static struct shash
*replication_dbs
;
109 static struct shash
*replication_db_clone(struct shash
*dbs
);
110 static void replication_dbs_destroy(void);
111 /* Find 'struct ovsdb' by name within 'replication_dbs' */
112 static struct ovsdb
* find_db(const char *db_name
);
116 replication_init(const char *sync_from_
, const char *exclude_tables
,
117 const struct uuid
*server
)
120 sync_from
= xstrdup(sync_from_
);
121 char *err
= set_blacklist_tables(exclude_tables
, false);
122 /* Caller should have verified that the 'exclude_tables' is
123 * parseable. An error here is unexpected. */
126 replication_dbs_destroy();
128 shash_clear(&local_dbs
);
130 jsonrpc_session_close(session
);
133 session
= jsonrpc_session_open(sync_from
, true);
134 session_seqno
= UINT_MAX
;
136 /* Keep a copy of local server uuid. */
137 server_uuid
= *server
;
143 replication_add_local_db(const char *database
, struct ovsdb
*db
)
145 shash_add_assert(&local_dbs
, database
, db
);
149 send_schema_requests(const struct json
*result
)
151 for (size_t i
= 0; i
< result
->u
.array
.n
; i
++) {
152 const struct json
*name
= result
->u
.array
.elems
[i
];
153 if (name
->type
== JSON_STRING
) {
154 /* Send one schema request for each remote DB. */
155 const char *db_name
= json_string(name
);
156 struct ovsdb
*db
= find_db(db_name
);
158 struct jsonrpc_msg
*request
=
159 jsonrpc_create_request(
162 json_string_create(db_name
)),
165 request_ids_add(request
->id
, db
);
166 jsonrpc_session_send(session
, request
);
173 replication_run(void)
179 jsonrpc_session_run(session
);
181 for (int i
= 0; jsonrpc_session_is_connected(session
) && i
< 50; i
++) {
182 struct jsonrpc_msg
*msg
;
185 seqno
= jsonrpc_session_get_seqno(session
);
186 if (seqno
!= session_seqno
|| state
== RPL_S_INIT
) {
187 session_seqno
= seqno
;
189 struct jsonrpc_msg
*request
;
190 request
= jsonrpc_create_request("get_server_id",
191 json_array_create_empty(), NULL
);
192 request_ids_add(request
->id
, NULL
);
193 jsonrpc_session_send(session
, request
);
195 state
= RPL_S_SERVER_ID_REQUESTED
;
196 VLOG_DBG("send server ID request.");
199 msg
= jsonrpc_session_recv(session
);
204 if (msg
->type
== JSONRPC_NOTIFY
&& state
!= RPL_S_ERR
205 && !strcmp(msg
->method
, "update")) {
206 if (msg
->params
->type
== JSON_ARRAY
207 && msg
->params
->u
.array
.n
== 2
208 && msg
->params
->u
.array
.elems
[0]->type
== JSON_STRING
) {
209 char *db_name
= msg
->params
->u
.array
.elems
[0]->u
.string
;
210 struct ovsdb
*db
= find_db(db_name
);
212 struct ovsdb_error
*error
;
213 error
= process_notification(msg
->params
->u
.array
.elems
[1],
216 ovsdb_error_assert(error
);
221 } else if (msg
->type
== JSONRPC_REPLY
) {
223 if (!request_ids_lookup_and_free(msg
->id
, &db
)) {
224 VLOG_WARN("received unexpected reply");
229 case RPL_S_SERVER_ID_REQUESTED
: {
231 if (msg
->result
->type
!= JSON_STRING
||
232 !uuid_from_string(&uuid
, json_string(msg
->result
))) {
233 struct ovsdb_error
*error
;
234 error
= ovsdb_error("get_server_id failed",
235 "Server ID is not valid UUID");
237 ovsdb_error_assert(error
);
242 if (uuid_equals(&uuid
, &server_uuid
)) {
243 struct ovsdb_error
*error
;
244 error
= ovsdb_error("Server ID check failed",
245 "Self replicating is not allowed");
247 ovsdb_error_assert(error
);
252 struct jsonrpc_msg
*request
;
253 request
= jsonrpc_create_request("list_dbs",
254 json_array_create_empty(),
256 request_ids_add(request
->id
, NULL
);
257 jsonrpc_session_send(session
, request
);
259 replication_dbs_destroy();
260 replication_dbs
= replication_db_clone(&local_dbs
);
261 state
= RPL_S_DB_REQUESTED
;
264 case RPL_S_DB_REQUESTED
:
265 if (msg
->result
->type
!= JSON_ARRAY
) {
266 struct ovsdb_error
*error
;
267 error
= ovsdb_error("list_dbs failed",
268 "list_dbs response is not array");
269 ovsdb_error_assert(error
);
272 send_schema_requests(msg
->result
);
273 VLOG_DBG("Send schema requests");
274 state
= RPL_S_SCHEMA_REQUESTED
;
278 case RPL_S_SCHEMA_REQUESTED
: {
279 struct ovsdb_schema
*schema
;
280 struct ovsdb_error
*error
;
282 error
= ovsdb_schema_from_json(msg
->result
, &schema
);
284 ovsdb_error_assert(error
);
288 if (db
!= find_db(schema
->name
)) {
289 /* Unexpected schema. */
290 VLOG_WARN("unexpected schema %s", schema
->name
);
292 } else if (!ovsdb_schema_equal(schema
, db
->schema
)) {
293 /* Schmea version mismatch. */
294 VLOG_INFO("Schema version mismatch, %s not replicated",
296 shash_find_and_delete(replication_dbs
, schema
->name
);
298 ovsdb_schema_destroy(schema
);
300 /* After receiving schemas, reset the local databases that
301 * will be monitored and send out monitor requests for them. */
302 if (hmap_is_empty(&request_ids
)) {
303 struct shash_node
*node
, *next
;
305 SHASH_FOR_EACH_SAFE (node
, next
, replication_dbs
) {
307 error
= reset_database(db
);
309 const char *db_name
= db
->schema
->name
;
310 shash_find_and_delete(replication_dbs
, db_name
);
311 ovsdb_error_assert(error
);
312 VLOG_WARN("Failed to reset database, "
313 "%s not replicated.", db_name
);
317 if (shash_is_empty(replication_dbs
)) {
318 VLOG_WARN("Nothing to replicate.");
321 SHASH_FOR_EACH (node
, replication_dbs
) {
323 struct jsonrpc_msg
*request
=
324 create_monitor_request(db
);
326 request_ids_add(request
->id
, db
);
327 jsonrpc_session_send(session
, request
);
328 VLOG_DBG("Send monitor requests");
329 state
= RPL_S_MONITOR_REQUESTED
;
336 case RPL_S_MONITOR_REQUESTED
: {
337 /* Reply to monitor requests. */
338 struct ovsdb_error
*error
;
339 error
= process_notification(msg
->result
, db
);
341 ovsdb_error_assert(error
);
344 /* Transition to replicating state after receiving
345 * all replies of "monitor" requests. */
346 if (hmap_is_empty(&request_ids
)) {
347 VLOG_DBG("Listening to monitor updates");
348 state
= RPL_S_REPLICATING
;
355 /* Ignore all messages */
359 case RPL_S_REPLICATING
:
365 jsonrpc_msg_destroy(msg
);
370 replication_wait(void)
373 jsonrpc_session_wait(session
);
374 jsonrpc_session_recv_wait(session
);
378 /* Parse 'blacklist' to rebuild 'blacklist_tables'. If 'dryrun' is false, the
379 * current black list tables will be wiped out, regardless of whether
380 * 'blacklist' can be parsed. If 'dryrun' is true, only parses 'blacklist' and
381 * reports any errors, without modifying the blacklist.
383 * On error, returns the error string, which the caller is
384 * responsible for freeing. Returns NULL otherwise. */
385 char * OVS_WARN_UNUSED_RESULT
386 set_blacklist_tables(const char *blacklist
, bool dryrun
)
388 struct sset set
= SSET_INITIALIZER(&set
);
392 const char *longname
;
395 /* Can only add to an empty shash. */
396 blacklist_tables_clear();
399 sset_from_delimited_string(&set
, blacklist
, " ,");
400 SSET_FOR_EACH (longname
, &set
) {
401 char *database
= xstrdup(longname
), *table
= NULL
;
402 strtok_r(database
, ":", &table
);
403 if (table
&& !dryrun
) {
404 blacklist_tables_add(database
, table
);
409 err
= xasprintf("Can't parse black list table: %s", longname
);
417 if (err
&& !dryrun
) {
418 /* On error, destroy the partially built 'blacklist_tables'. */
419 blacklist_tables_clear();
424 char * OVS_WARN_UNUSED_RESULT
425 get_blacklist_tables(void)
427 struct shash_node
*node
;
428 struct sset set
= SSET_INITIALIZER(&set
);
430 SHASH_FOR_EACH (node
, &blacklist_tables
) {
431 const char *database
= node
->name
;
433 struct sset
*tables
= node
->data
;
435 SSET_FOR_EACH (table
, tables
) {
436 sset_add_and_free(&set
, xasprintf("%s:%s", database
, table
));
440 /* Output the table list in an sorted order, so that
441 * the output string will not depend on the hash function
442 * that used to implement the hmap data structure. This is
443 * only useful for writting unit tests. */
444 const char **sorted
= sset_sort(&set
);
445 struct ds ds
= DS_EMPTY_INITIALIZER
;
447 for (i
= 0; i
< sset_count(&set
); i
++) {
448 ds_put_format(&ds
, "%s,", sorted
[i
]);
456 return ds_steal_cstr(&ds
);
460 blacklist_tables_clear(void)
462 struct shash_node
*node
;
463 SHASH_FOR_EACH (node
, &blacklist_tables
) {
464 struct sset
*tables
= node
->data
;
465 sset_destroy(tables
);
468 shash_clear_free_data(&blacklist_tables
);
472 blacklist_tables_add(const char *database
, const char *table
)
474 struct sset
*tables
= shash_find_data(&blacklist_tables
, database
);
477 tables
= xmalloc(sizeof *tables
);
479 shash_add(&blacklist_tables
, database
, tables
);
482 sset_add(tables
, table
);
486 blacklist_tables_find(const char *database
, const char *table
)
488 struct sset
*tables
= shash_find_data(&blacklist_tables
, database
);
489 return tables
&& sset_contains(tables
, table
);
493 disconnect_active_server(void)
495 jsonrpc_session_close(session
);
500 replication_destroy(void)
502 blacklist_tables_clear();
503 shash_destroy(&blacklist_tables
);
510 request_ids_destroy();
511 replication_dbs_destroy();
513 shash_destroy(&local_dbs
);
516 static struct ovsdb
*
517 find_db(const char *db_name
)
519 return shash_find_data(replication_dbs
, db_name
);
522 static struct ovsdb_error
*
523 reset_database(struct ovsdb
*db
)
525 struct ovsdb_txn
*txn
= ovsdb_txn_create(db
);
526 struct shash_node
*table_node
;
528 SHASH_FOR_EACH (table_node
, &db
->tables
) {
529 /* Delete all rows if the table is not blacklisted. */
530 if (!blacklist_tables_find(db
->schema
->name
, table_node
->name
)) {
531 struct ovsdb_table
*table
= table_node
->data
;
532 struct ovsdb_row
*row
;
533 HMAP_FOR_EACH (row
, hmap_node
, &table
->rows
) {
534 ovsdb_txn_row_delete(txn
, row
);
539 return ovsdb_txn_commit(txn
, false);
542 /* Create a monitor request for 'db'. The monitor request will include
543 * any tables from 'blacklisted_tables'
545 * Caller is responsible for disposing 'request'.
547 static struct jsonrpc_msg
*
548 create_monitor_request(struct ovsdb
*db
)
550 struct jsonrpc_msg
*request
;
551 struct json
*monitor
;
552 struct ovsdb_schema
*schema
= db
->schema
;
553 const char *db_name
= schema
->name
;
555 struct json
*monitor_request
= json_object_create();
556 size_t n
= shash_count(&schema
->tables
);
557 const struct shash_node
**nodes
= shash_sort(&schema
->tables
);
559 for (int j
= 0; j
< n
; j
++) {
560 struct ovsdb_table_schema
*table
= nodes
[j
]->data
;
562 /* Monitor all tables not blacklisted. */
563 if (!blacklist_tables_find(db_name
, table
->name
)) {
564 add_monitored_table(table
, monitor_request
);
569 /* Create a monitor request. */
570 monitor
= json_array_create_3(
571 json_string_create(db_name
),
572 json_string_create(db_name
),
574 request
= jsonrpc_create_request("monitor", monitor
, NULL
);
580 add_monitored_table(struct ovsdb_table_schema
*table
,
581 struct json
*monitor_request
)
583 struct json
*monitor_request_array
;
585 monitor_request_array
= json_array_create_empty();
586 json_array_add(monitor_request_array
, json_object_create());
588 json_object_put(monitor_request
, table
->name
, monitor_request_array
);
592 static struct ovsdb_error
*
593 process_notification(struct json
*table_updates
, struct ovsdb
*db
)
595 struct ovsdb_error
*error
= NULL
;
596 struct ovsdb_txn
*txn
;
598 if (table_updates
->type
== JSON_OBJECT
) {
599 txn
= ovsdb_txn_create(db
);
601 /* Process each table update. */
602 struct shash_node
*node
;
603 SHASH_FOR_EACH (node
, json_object(table_updates
)) {
604 struct json
*table_update
= node
->data
;
606 error
= process_table_update(table_update
, node
->name
, db
, txn
);
614 ovsdb_txn_abort(txn
);
617 /* Commit transaction. */
618 error
= ovsdb_txn_commit(txn
, false);
625 static struct ovsdb_error
*
626 process_table_update(struct json
*table_update
, const char *table_name
,
627 struct ovsdb
*database
, struct ovsdb_txn
*txn
)
629 struct ovsdb_table
*table
= ovsdb_get_table(database
, table_name
);
631 return ovsdb_error("unknown table", "unknown table %s", table_name
);
634 if (table_update
->type
!= JSON_OBJECT
) {
635 return ovsdb_error("Not a JSON object",
636 "<table-update> for table is not object");
639 struct shash_node
*node
;
640 SHASH_FOR_EACH (node
, json_object(table_update
)) {
641 struct json
*row_update
= node
->data
;
642 struct json
*old
, *new;
644 if (row_update
->type
!= JSON_OBJECT
) {
645 return ovsdb_error("Not a JSON object",
646 "<row-update> is not object");
650 if (!uuid_from_string(&uuid
, node
->name
)) {
651 return ovsdb_syntax_error(table_update
, "bad row UUID",
652 "<table-update> names must be UUIDs");
655 old
= shash_find_data(json_object(row_update
), "old");
656 new = shash_find_data(json_object(row_update
), "new");
658 struct ovsdb_error
*error
;
659 error
= (!new ? execute_delete(txn
, &uuid
, table
)
660 : !old
? execute_insert(txn
, &uuid
, table
, new)
661 : execute_update(txn
, &uuid
, table
, new));
669 static struct ovsdb_error
*
670 execute_insert(struct ovsdb_txn
*txn
, const struct uuid
*row_uuid
,
671 struct ovsdb_table
*table
, struct json
*json_row
)
673 struct ovsdb_row
*row
= ovsdb_row_create(table
);
674 struct ovsdb_error
*error
= ovsdb_row_from_json(row
, json_row
, NULL
, NULL
);
676 *ovsdb_row_get_uuid_rw(row
) = *row_uuid
;
677 ovsdb_txn_row_insert(txn
, row
);
679 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
680 VLOG_WARN_RL(&rl
, "cannot add existing row "UUID_FMT
" to table %s",
681 UUID_ARGS(row_uuid
), table
->schema
->name
);
682 ovsdb_row_destroy(row
);
688 static struct ovsdb_error
*
689 execute_delete(struct ovsdb_txn
*txn
, const struct uuid
*row_uuid
,
690 struct ovsdb_table
*table
)
692 const struct ovsdb_row
*row
= ovsdb_table_get_row(table
, row_uuid
);
694 ovsdb_txn_row_delete(txn
, row
);
696 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
697 VLOG_WARN_RL(&rl
, "cannot delete missing row "UUID_FMT
" from table %s",
698 UUID_ARGS(row_uuid
), table
->schema
->name
);
703 static struct ovsdb_error
*
704 execute_update(struct ovsdb_txn
*txn
, const struct uuid
*row_uuid
,
705 struct ovsdb_table
*table
, struct json
*json_row
)
707 const struct ovsdb_row
*row
= ovsdb_table_get_row(table
, row_uuid
);
709 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
710 VLOG_WARN_RL(&rl
, "cannot modify missing row "UUID_FMT
" in table %s",
711 UUID_ARGS(row_uuid
), table
->schema
->name
);
715 struct ovsdb_column_set columns
= OVSDB_COLUMN_SET_INITIALIZER
;
716 struct ovsdb_row
*update
= ovsdb_row_create(table
);
717 struct ovsdb_error
*error
= ovsdb_row_from_json(update
, json_row
,
720 if (!error
&& !ovsdb_row_equal_columns(row
, update
, &columns
)) {
721 ovsdb_row_update_columns(ovsdb_txn_row_modify(txn
, row
),
725 ovsdb_column_set_destroy(&columns
);
726 ovsdb_row_destroy(update
);
731 request_ids_add(const struct json
*id
, struct ovsdb
*db
)
733 struct request_ids_hmap_node
*node
= xmalloc(sizeof *node
);
735 node
->request_id
= json_clone(id
);
737 hmap_insert(&request_ids
, &node
->hmap
, json_hash(id
, 0));
740 /* Look up 'id' from 'request_ids', if found, remove the found id from
741 * 'request_ids' and free its memory. If not found, 'request_ids' does
742 * not change. Sets '*db' to the database for the request (NULL if not
745 * Return true if 'id' is found, false otherwise.
748 request_ids_lookup_and_free(const struct json
*id
, struct ovsdb
**db
)
750 struct request_ids_hmap_node
*node
;
752 HMAP_FOR_EACH_WITH_HASH (node
, hmap
, json_hash(id
, 0), &request_ids
) {
753 if (json_equal(id
, node
->request_id
)) {
754 hmap_remove(&request_ids
, &node
->hmap
);
756 json_destroy(node
->request_id
);
767 request_ids_destroy(void)
769 struct request_ids_hmap_node
*node
;
771 HMAP_FOR_EACH_POP (node
, hmap
, &request_ids
) {
772 json_destroy(node
->request_id
);
775 hmap_destroy(&request_ids
);
779 request_ids_clear(void)
781 request_ids_destroy();
782 hmap_init(&request_ids
);
785 static struct shash
*
786 replication_db_clone(struct shash
*dbs
)
788 struct shash
*new = xmalloc(sizeof *new);
791 struct shash_node
*node
;
792 SHASH_FOR_EACH (node
, dbs
) {
793 shash_add(new, node
->name
, node
->data
);
800 replication_dbs_destroy(void)
802 shash_destroy(replication_dbs
);
803 free(replication_dbs
);
804 replication_dbs
= NULL
;
807 /* Return true if replication just started or is ongoing.
808 * Return false if the connection failed, or the replication
809 * was not able to start. */
811 replication_is_alive(void)
814 return jsonrpc_session_is_alive(session
) && state
!= RPL_S_ERR
;
819 /* Return the last error reported on a connection by 'session'. The
820 * return value is 0 if replication is not currently running, or
821 * if replication session has not encountered any error.
823 * Return a negative value if replication session has error, or the
824 * replication was not able to start. */
826 replication_get_last_error(void)
831 err
= jsonrpc_session_get_last_error(session
);
833 err
= (state
== RPL_S_ERR
) ? ENOENT
: 0;
841 replication_status(void)
843 bool alive
= session
&& jsonrpc_session_is_alive(session
);
844 struct ds ds
= DS_EMPTY_INITIALIZER
;
849 case RPL_S_SERVER_ID_REQUESTED
:
850 case RPL_S_DB_REQUESTED
:
851 case RPL_S_SCHEMA_REQUESTED
:
852 case RPL_S_MONITOR_REQUESTED
:
853 ds_put_format(&ds
, "connecting: %s", sync_from
);
855 case RPL_S_REPLICATING
: {
856 struct shash_node
*node
;
858 ds_put_format(&ds
, "replicating: %s\n", sync_from
);
859 ds_put_cstr(&ds
, "database:");
860 SHASH_FOR_EACH (node
, replication_dbs
) {
861 ds_put_format(&ds
, " %s,", node
->name
);
865 if (!shash_is_empty(&blacklist_tables
)) {
866 ds_put_char(&ds
, '\n');
867 ds_put_cstr(&ds
, "exclude: ");
868 ds_put_and_free_cstr(&ds
, get_blacklist_tables());
873 ds_put_format(&ds
, "Replication to (%s) failed\n", sync_from
);
880 ds_put_format(&ds
, "not connected to %s", sync_from
);
882 return ds_steal_cstr(&ds
);
886 replication_usage(void)
890 --sync-from=SERVER sync DATABASE from active SERVER and start in\n\
891 backup mode (except with --active)\n\
892 --sync-exclude-tables=DB:TABLE,...\n\
893 exclude the TABLE in DB from syncing\n\
894 --active with --sync-from, start in active mode\n");