4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
29 #include <linux/module.h>
30 #include <linux/drbd.h>
31 #include <asm/uaccess.h>
32 #include <asm/types.h>
34 #include <linux/ctype.h>
35 #include <linux/mutex.h>
37 #include <linux/file.h>
38 #include <linux/proc_fs.h>
39 #include <linux/init.h>
41 #include <linux/memcontrol.h>
42 #include <linux/mm_inline.h>
43 #include <linux/slab.h>
44 #include <linux/random.h>
45 #include <linux/reboot.h>
46 #include <linux/notifier.h>
47 #include <linux/kthread.h>
49 #define __KERNEL_SYSCALLS__
50 #include <linux/unistd.h>
51 #include <linux/vmalloc.h>
53 #include <linux/drbd_limits.h>
55 #include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
59 static DEFINE_MUTEX(drbd_main_mutex
);
60 int drbdd_init(struct drbd_thread
*);
61 int drbd_worker(struct drbd_thread
*);
62 int drbd_asender(struct drbd_thread
*);
65 static int drbd_open(struct block_device
*bdev
, fmode_t mode
);
66 static int drbd_release(struct gendisk
*gd
, fmode_t mode
);
67 static int w_md_sync(struct drbd_work
*w
, int unused
);
68 static void md_sync_timer_fn(unsigned long data
);
69 static int w_bitmap_io(struct drbd_work
*w
, int unused
);
70 static int w_go_diskless(struct drbd_work
*w
, int unused
);
72 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
73 "Lars Ellenberg <lars@linbit.com>");
74 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION
);
75 MODULE_VERSION(REL_VERSION
);
76 MODULE_LICENSE("GPL");
77 MODULE_PARM_DESC(minor_count
, "Approximate number of drbd devices ("
78 __stringify(DRBD_MINOR_COUNT_MIN
) "-" __stringify(DRBD_MINOR_COUNT_MAX
) ")");
79 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR
);
81 #include <linux/moduleparam.h>
82 /* allow_open_on_secondary */
83 MODULE_PARM_DESC(allow_oos
, "DONT USE!");
84 /* thanks to these macros, if compiled into the kernel (not-module),
85 * this becomes the boot parameter drbd.minor_count */
86 module_param(minor_count
, uint
, 0444);
87 module_param(disable_sendpage
, bool, 0644);
88 module_param(allow_oos
, bool, 0);
89 module_param(proc_details
, int, 0644);
91 #ifdef CONFIG_DRBD_FAULT_INJECTION
94 static int fault_count
;
96 /* bitmap of enabled faults */
97 module_param(enable_faults
, int, 0664);
98 /* fault rate % value - applies to all enabled faults */
99 module_param(fault_rate
, int, 0664);
100 /* count of faults inserted */
101 module_param(fault_count
, int, 0664);
102 /* bitmap of devices to insert faults on */
103 module_param(fault_devs
, int, 0644);
106 /* module parameter, defined */
107 unsigned int minor_count
= DRBD_MINOR_COUNT_DEF
;
108 int disable_sendpage
;
110 int proc_details
; /* Detail level in proc drbd*/
112 /* Module parameter for setting the user mode helper program
113 * to run. Default is /sbin/drbdadm */
114 char usermode_helper
[80] = "/sbin/drbdadm";
116 module_param_string(usermode_helper
, usermode_helper
, sizeof(usermode_helper
), 0644);
118 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
119 * as member "struct gendisk *vdisk;"
122 struct list_head drbd_tconns
; /* list of struct drbd_tconn */
123 DEFINE_MUTEX(drbd_cfg_mutex
);
125 struct kmem_cache
*drbd_request_cache
;
126 struct kmem_cache
*drbd_ee_cache
; /* peer requests */
127 struct kmem_cache
*drbd_bm_ext_cache
; /* bitmap extents */
128 struct kmem_cache
*drbd_al_ext_cache
; /* activity log extents */
129 mempool_t
*drbd_request_mempool
;
130 mempool_t
*drbd_ee_mempool
;
131 mempool_t
*drbd_md_io_page_pool
;
132 struct bio_set
*drbd_md_io_bio_set
;
134 /* I do not use a standard mempool, because:
135 1) I want to hand out the pre-allocated objects first.
136 2) I want to be able to interrupt sleeping allocation with a signal.
137 Note: This is a single linked list, the next pointer is the private
138 member of struct page.
140 struct page
*drbd_pp_pool
;
141 spinlock_t drbd_pp_lock
;
143 wait_queue_head_t drbd_pp_wait
;
145 DEFINE_RATELIMIT_STATE(drbd_ratelimit_state
, 5 * HZ
, 5);
147 static const struct block_device_operations drbd_ops
= {
148 .owner
= THIS_MODULE
,
150 .release
= drbd_release
,
153 static void bio_destructor_drbd(struct bio
*bio
)
155 bio_free(bio
, drbd_md_io_bio_set
);
158 struct bio
*bio_alloc_drbd(gfp_t gfp_mask
)
162 if (!drbd_md_io_bio_set
)
163 return bio_alloc(gfp_mask
, 1);
165 bio
= bio_alloc_bioset(gfp_mask
, 1, drbd_md_io_bio_set
);
168 bio
->bi_destructor
= bio_destructor_drbd
;
173 /* When checking with sparse, and this is an inline function, sparse will
174 give tons of false positives. When this is a real functions sparse works.
176 int _get_ldev_if_state(struct drbd_conf
*mdev
, enum drbd_disk_state mins
)
180 atomic_inc(&mdev
->local_cnt
);
181 io_allowed
= (mdev
->state
.disk
>= mins
);
183 if (atomic_dec_and_test(&mdev
->local_cnt
))
184 wake_up(&mdev
->misc_wait
);
192 * DOC: The transfer log
194 * The transfer log is a single linked list of &struct drbd_tl_epoch objects.
195 * mdev->tconn->newest_tle points to the head, mdev->tconn->oldest_tle points to the tail
196 * of the list. There is always at least one &struct drbd_tl_epoch object.
198 * Each &struct drbd_tl_epoch has a circular double linked list of requests
201 static int tl_init(struct drbd_tconn
*tconn
)
203 struct drbd_tl_epoch
*b
;
205 /* during device minor initialization, we may well use GFP_KERNEL */
206 b
= kmalloc(sizeof(struct drbd_tl_epoch
), GFP_KERNEL
);
209 INIT_LIST_HEAD(&b
->requests
);
210 INIT_LIST_HEAD(&b
->w
.list
);
214 b
->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
216 tconn
->oldest_tle
= b
;
217 tconn
->newest_tle
= b
;
218 INIT_LIST_HEAD(&tconn
->out_of_sequence_requests
);
223 static void tl_cleanup(struct drbd_tconn
*tconn
)
225 if (tconn
->oldest_tle
!= tconn
->newest_tle
)
226 conn_err(tconn
, "ASSERT FAILED: oldest_tle == newest_tle\n");
227 if (!list_empty(&tconn
->out_of_sequence_requests
))
228 conn_err(tconn
, "ASSERT FAILED: list_empty(out_of_sequence_requests)\n");
229 kfree(tconn
->oldest_tle
);
230 tconn
->oldest_tle
= NULL
;
231 kfree(tconn
->unused_spare_tle
);
232 tconn
->unused_spare_tle
= NULL
;
236 * _tl_add_barrier() - Adds a barrier to the transfer log
237 * @mdev: DRBD device.
238 * @new: Barrier to be added before the current head of the TL.
240 * The caller must hold the req_lock.
242 void _tl_add_barrier(struct drbd_tconn
*tconn
, struct drbd_tl_epoch
*new)
244 struct drbd_tl_epoch
*newest_before
;
246 INIT_LIST_HEAD(&new->requests
);
247 INIT_LIST_HEAD(&new->w
.list
);
248 new->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
252 newest_before
= tconn
->newest_tle
;
253 /* never send a barrier number == 0, because that is special-cased
254 * when using TCQ for our write ordering code */
255 new->br_number
= (newest_before
->br_number
+1) ?: 1;
256 if (tconn
->newest_tle
!= new) {
257 tconn
->newest_tle
->next
= new;
258 tconn
->newest_tle
= new;
263 * tl_release() - Free or recycle the oldest &struct drbd_tl_epoch object of the TL
264 * @mdev: DRBD device.
265 * @barrier_nr: Expected identifier of the DRBD write barrier packet.
266 * @set_size: Expected number of requests before that barrier.
268 * In case the passed barrier_nr or set_size does not match the oldest
269 * &struct drbd_tl_epoch objects this function will cause a termination
272 void tl_release(struct drbd_tconn
*tconn
, unsigned int barrier_nr
,
273 unsigned int set_size
)
275 struct drbd_conf
*mdev
;
276 struct drbd_tl_epoch
*b
, *nob
; /* next old barrier */
277 struct list_head
*le
, *tle
;
278 struct drbd_request
*r
;
280 spin_lock_irq(&tconn
->req_lock
);
282 b
= tconn
->oldest_tle
;
284 /* first some paranoia code */
286 conn_err(tconn
, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
290 if (b
->br_number
!= barrier_nr
) {
291 conn_err(tconn
, "BAD! BarrierAck #%u received, expected #%u!\n",
292 barrier_nr
, b
->br_number
);
295 if (b
->n_writes
!= set_size
) {
296 conn_err(tconn
, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
297 barrier_nr
, set_size
, b
->n_writes
);
301 /* Clean up list of requests processed during current epoch */
302 list_for_each_safe(le
, tle
, &b
->requests
) {
303 r
= list_entry(le
, struct drbd_request
, tl_requests
);
304 _req_mod(r
, BARRIER_ACKED
);
306 /* There could be requests on the list waiting for completion
307 of the write to the local disk. To avoid corruptions of
308 slab's data structures we have to remove the lists head.
310 Also there could have been a barrier ack out of sequence, overtaking
311 the write acks - which would be a bug and violating write ordering.
312 To not deadlock in case we lose connection while such requests are
313 still pending, we need some way to find them for the
314 _req_mode(CONNECTION_LOST_WHILE_PENDING).
316 These have been list_move'd to the out_of_sequence_requests list in
317 _req_mod(, BARRIER_ACKED) above.
319 list_del_init(&b
->requests
);
323 if (test_and_clear_bit(CREATE_BARRIER
, &mdev
->flags
)) {
324 _tl_add_barrier(tconn
, b
);
326 tconn
->oldest_tle
= nob
;
327 /* if nob == NULL b was the only barrier, and becomes the new
328 barrier. Therefore tconn->oldest_tle points already to b */
330 D_ASSERT(nob
!= NULL
);
331 tconn
->oldest_tle
= nob
;
335 spin_unlock_irq(&tconn
->req_lock
);
336 dec_ap_pending(mdev
);
341 spin_unlock_irq(&tconn
->req_lock
);
342 conn_request_state(tconn
, NS(conn
, C_PROTOCOL_ERROR
), CS_HARD
);
347 * _tl_restart() - Walks the transfer log, and applies an action to all requests
348 * @mdev: DRBD device.
349 * @what: The action/event to perform with all request objects
351 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
352 * RESTART_FROZEN_DISK_IO.
354 void _tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
356 struct drbd_tl_epoch
*b
, *tmp
, **pn
;
357 struct list_head
*le
, *tle
, carry_reads
;
358 struct drbd_request
*req
;
359 int rv
, n_writes
, n_reads
;
361 b
= tconn
->oldest_tle
;
362 pn
= &tconn
->oldest_tle
;
366 INIT_LIST_HEAD(&carry_reads
);
367 list_for_each_safe(le
, tle
, &b
->requests
) {
368 req
= list_entry(le
, struct drbd_request
, tl_requests
);
369 rv
= _req_mod(req
, what
);
371 n_writes
+= (rv
& MR_WRITE
) >> MR_WRITE_SHIFT
;
372 n_reads
+= (rv
& MR_READ
) >> MR_READ_SHIFT
;
377 if (what
== RESEND
) {
378 b
->n_writes
= n_writes
;
379 if (b
->w
.cb
== NULL
) {
380 b
->w
.cb
= w_send_barrier
;
381 inc_ap_pending(b
->w
.mdev
);
382 set_bit(CREATE_BARRIER
, &b
->w
.mdev
->flags
);
385 drbd_queue_work(&tconn
->data
.work
, &b
->w
);
390 list_add(&carry_reads
, &b
->requests
);
391 /* there could still be requests on that ring list,
392 * in case local io is still pending */
393 list_del(&b
->requests
);
395 /* dec_ap_pending corresponding to queue_barrier.
396 * the newest barrier may not have been queued yet,
397 * in which case w.cb is still NULL. */
399 dec_ap_pending(b
->w
.mdev
);
401 if (b
== tconn
->newest_tle
) {
402 /* recycle, but reinit! */
404 conn_err(tconn
, "ASSERT FAILED tmp == NULL");
405 INIT_LIST_HEAD(&b
->requests
);
406 list_splice(&carry_reads
, &b
->requests
);
407 INIT_LIST_HEAD(&b
->w
.list
);
409 b
->br_number
= net_random();
419 list_splice(&carry_reads
, &b
->requests
);
425 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
426 * @mdev: DRBD device.
428 * This is called after the connection to the peer was lost. The storage covered
429 * by the requests on the transfer gets marked as our of sync. Called from the
430 * receiver thread and the worker thread.
432 void tl_clear(struct drbd_tconn
*tconn
)
434 struct drbd_conf
*mdev
;
435 struct list_head
*le
, *tle
;
436 struct drbd_request
*r
;
439 spin_lock_irq(&tconn
->req_lock
);
441 _tl_restart(tconn
, CONNECTION_LOST_WHILE_PENDING
);
443 /* we expect this list to be empty. */
444 if (!list_empty(&tconn
->out_of_sequence_requests
))
445 conn_err(tconn
, "ASSERT FAILED list_empty(&out_of_sequence_requests)\n");
447 /* but just in case, clean it up anyways! */
448 list_for_each_safe(le
, tle
, &tconn
->out_of_sequence_requests
) {
449 r
= list_entry(le
, struct drbd_request
, tl_requests
);
450 /* It would be nice to complete outside of spinlock.
451 * But this is easier for now. */
452 _req_mod(r
, CONNECTION_LOST_WHILE_PENDING
);
455 /* ensure bit indicating barrier is required is clear */
456 idr_for_each_entry(&tconn
->volumes
, mdev
, vnr
)
457 clear_bit(CREATE_BARRIER
, &mdev
->flags
);
459 spin_unlock_irq(&tconn
->req_lock
);
462 void tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
464 spin_lock_irq(&tconn
->req_lock
);
465 _tl_restart(tconn
, what
);
466 spin_unlock_irq(&tconn
->req_lock
);
469 static int drbd_thread_setup(void *arg
)
471 struct drbd_thread
*thi
= (struct drbd_thread
*) arg
;
472 struct drbd_tconn
*tconn
= thi
->tconn
;
476 snprintf(current
->comm
, sizeof(current
->comm
), "drbd_%c_%s",
477 thi
->name
[0], thi
->tconn
->name
);
480 retval
= thi
->function(thi
);
482 spin_lock_irqsave(&thi
->t_lock
, flags
);
484 /* if the receiver has been "EXITING", the last thing it did
485 * was set the conn state to "StandAlone",
486 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
487 * and receiver thread will be "started".
488 * drbd_thread_start needs to set "RESTARTING" in that case.
489 * t_state check and assignment needs to be within the same spinlock,
490 * so either thread_start sees EXITING, and can remap to RESTARTING,
491 * or thread_start see NONE, and can proceed as normal.
494 if (thi
->t_state
== RESTARTING
) {
495 conn_info(tconn
, "Restarting %s thread\n", thi
->name
);
496 thi
->t_state
= RUNNING
;
497 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
504 complete(&thi
->stop
);
505 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
507 conn_info(tconn
, "Terminating %s\n", current
->comm
);
509 /* Release mod reference taken when thread was started */
510 module_put(THIS_MODULE
);
514 static void drbd_thread_init(struct drbd_tconn
*tconn
, struct drbd_thread
*thi
,
515 int (*func
) (struct drbd_thread
*), char *name
)
517 spin_lock_init(&thi
->t_lock
);
520 thi
->function
= func
;
522 strncpy(thi
->name
, name
, ARRAY_SIZE(thi
->name
));
525 int drbd_thread_start(struct drbd_thread
*thi
)
527 struct drbd_tconn
*tconn
= thi
->tconn
;
528 struct task_struct
*nt
;
531 /* is used from state engine doing drbd_thread_stop_nowait,
532 * while holding the req lock irqsave */
533 spin_lock_irqsave(&thi
->t_lock
, flags
);
535 switch (thi
->t_state
) {
537 conn_info(tconn
, "Starting %s thread (from %s [%d])\n",
538 thi
->name
, current
->comm
, current
->pid
);
540 /* Get ref on module for thread - this is released when thread exits */
541 if (!try_module_get(THIS_MODULE
)) {
542 conn_err(tconn
, "Failed to get module reference in drbd_thread_start\n");
543 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
547 init_completion(&thi
->stop
);
548 thi
->reset_cpu_mask
= 1;
549 thi
->t_state
= RUNNING
;
550 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
551 flush_signals(current
); /* otherw. may get -ERESTARTNOINTR */
553 nt
= kthread_create(drbd_thread_setup
, (void *) thi
,
554 "drbd_%c_%s", thi
->name
[0], thi
->tconn
->name
);
557 conn_err(tconn
, "Couldn't start thread\n");
559 module_put(THIS_MODULE
);
562 spin_lock_irqsave(&thi
->t_lock
, flags
);
564 thi
->t_state
= RUNNING
;
565 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
569 thi
->t_state
= RESTARTING
;
570 conn_info(tconn
, "Restarting %s thread (from %s [%d])\n",
571 thi
->name
, current
->comm
, current
->pid
);
576 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
584 void _drbd_thread_stop(struct drbd_thread
*thi
, int restart
, int wait
)
588 enum drbd_thread_state ns
= restart
? RESTARTING
: EXITING
;
590 /* may be called from state engine, holding the req lock irqsave */
591 spin_lock_irqsave(&thi
->t_lock
, flags
);
593 if (thi
->t_state
== NONE
) {
594 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
596 drbd_thread_start(thi
);
600 if (thi
->t_state
!= ns
) {
601 if (thi
->task
== NULL
) {
602 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
608 init_completion(&thi
->stop
);
609 if (thi
->task
!= current
)
610 force_sig(DRBD_SIGKILL
, thi
->task
);
613 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
616 wait_for_completion(&thi
->stop
);
619 static struct drbd_thread
*drbd_task_to_thread(struct drbd_tconn
*tconn
, struct task_struct
*task
)
621 struct drbd_thread
*thi
=
622 task
== tconn
->receiver
.task
? &tconn
->receiver
:
623 task
== tconn
->asender
.task
? &tconn
->asender
:
624 task
== tconn
->worker
.task
? &tconn
->worker
: NULL
;
629 char *drbd_task_to_thread_name(struct drbd_tconn
*tconn
, struct task_struct
*task
)
631 struct drbd_thread
*thi
= drbd_task_to_thread(tconn
, task
);
632 return thi
? thi
->name
: task
->comm
;
635 int conn_lowest_minor(struct drbd_tconn
*tconn
)
638 struct drbd_conf
*mdev
;
640 mdev
= idr_get_next(&tconn
->volumes
, &vnr
);
643 return mdev_to_minor(mdev
);
648 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
649 * @mdev: DRBD device.
651 * Forces all threads of a device onto the same CPU. This is beneficial for
652 * DRBD's performance. May be overwritten by user's configuration.
654 void drbd_calc_cpu_mask(struct drbd_tconn
*tconn
)
659 if (cpumask_weight(tconn
->cpu_mask
))
662 ord
= conn_lowest_minor(tconn
) % cpumask_weight(cpu_online_mask
);
663 for_each_online_cpu(cpu
) {
665 cpumask_set_cpu(cpu
, tconn
->cpu_mask
);
669 /* should not be reached */
670 cpumask_setall(tconn
->cpu_mask
);
674 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
675 * @mdev: DRBD device.
676 * @thi: drbd_thread object
678 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
681 void drbd_thread_current_set_cpu(struct drbd_thread
*thi
)
683 struct task_struct
*p
= current
;
685 if (!thi
->reset_cpu_mask
)
687 thi
->reset_cpu_mask
= 0;
688 set_cpus_allowed_ptr(p
, thi
->tconn
->cpu_mask
);
693 * drbd_header_size - size of a packet header
695 * The header size is a multiple of 8, so any payload following the header is
696 * word aligned on 64-bit architectures. (The bitmap send and receive code
699 unsigned int drbd_header_size(struct drbd_tconn
*tconn
)
701 if (tconn
->agreed_pro_version
>= 100) {
702 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100
), 8));
703 return sizeof(struct p_header100
);
705 BUILD_BUG_ON(sizeof(struct p_header80
) !=
706 sizeof(struct p_header95
));
707 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80
), 8));
708 return sizeof(struct p_header80
);
712 static unsigned int prepare_header80(struct p_header80
*h
, enum drbd_packet cmd
, int size
)
714 h
->magic
= cpu_to_be32(DRBD_MAGIC
);
715 h
->command
= cpu_to_be16(cmd
);
716 h
->length
= cpu_to_be16(size
);
717 return sizeof(struct p_header80
);
720 static unsigned int prepare_header95(struct p_header95
*h
, enum drbd_packet cmd
, int size
)
722 h
->magic
= cpu_to_be16(DRBD_MAGIC_BIG
);
723 h
->command
= cpu_to_be16(cmd
);
724 h
->length
= cpu_to_be32(size
);
725 return sizeof(struct p_header95
);
728 static unsigned int prepare_header100(struct p_header100
*h
, enum drbd_packet cmd
,
731 h
->magic
= cpu_to_be32(DRBD_MAGIC_100
);
732 h
->volume
= cpu_to_be16(vnr
);
733 h
->command
= cpu_to_be16(cmd
);
734 h
->length
= cpu_to_be32(size
);
736 return sizeof(struct p_header100
);
739 static unsigned int prepare_header(struct drbd_tconn
*tconn
, int vnr
,
740 void *buffer
, enum drbd_packet cmd
, int size
)
742 if (tconn
->agreed_pro_version
>= 100)
743 return prepare_header100(buffer
, cmd
, size
, vnr
);
744 else if (tconn
->agreed_pro_version
>= 95 &&
745 size
> DRBD_MAX_SIZE_H80_PACKET
)
746 return prepare_header95(buffer
, cmd
, size
);
748 return prepare_header80(buffer
, cmd
, size
);
751 void *conn_prepare_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
)
753 mutex_lock(&sock
->mutex
);
755 mutex_unlock(&sock
->mutex
);
758 return sock
->sbuf
+ drbd_header_size(tconn
);
761 void *drbd_prepare_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
)
763 return conn_prepare_command(mdev
->tconn
, sock
);
766 static int __send_command(struct drbd_tconn
*tconn
, int vnr
,
767 struct drbd_socket
*sock
, enum drbd_packet cmd
,
768 unsigned int header_size
, void *data
,
775 * Called with @data == NULL and the size of the data blocks in @size
776 * for commands that send data blocks. For those commands, omit the
777 * MSG_MORE flag: this will increase the likelihood that data blocks
778 * which are page aligned on the sender will end up page aligned on the
781 msg_flags
= data
? MSG_MORE
: 0;
783 header_size
+= prepare_header(tconn
, vnr
, sock
->sbuf
, cmd
,
785 err
= drbd_send_all(tconn
, sock
->socket
, sock
->sbuf
, header_size
,
788 err
= drbd_send_all(tconn
, sock
->socket
, data
, size
, 0);
792 int conn_send_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
,
793 enum drbd_packet cmd
, unsigned int header_size
,
794 void *data
, unsigned int size
)
798 err
= __send_command(tconn
, 0, sock
, cmd
, header_size
, data
, size
);
799 mutex_unlock(&sock
->mutex
);
803 int drbd_send_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
,
804 enum drbd_packet cmd
, unsigned int header_size
,
805 void *data
, unsigned int size
)
809 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, header_size
,
811 mutex_unlock(&sock
->mutex
);
815 int drbd_send_ping(struct drbd_tconn
*tconn
)
817 struct drbd_socket
*sock
;
820 if (!conn_prepare_command(tconn
, sock
))
822 return conn_send_command(tconn
, sock
, P_PING
, 0, NULL
, 0);
825 int drbd_send_ping_ack(struct drbd_tconn
*tconn
)
827 struct drbd_socket
*sock
;
830 if (!conn_prepare_command(tconn
, sock
))
832 return conn_send_command(tconn
, sock
, P_PING_ACK
, 0, NULL
, 0);
835 int drbd_send_sync_param(struct drbd_conf
*mdev
)
837 struct drbd_socket
*sock
;
838 struct p_rs_param_95
*p
;
840 const int apv
= mdev
->tconn
->agreed_pro_version
;
841 enum drbd_packet cmd
;
843 sock
= &mdev
->tconn
->data
;
844 p
= drbd_prepare_command(mdev
, sock
);
848 size
= apv
<= 87 ? sizeof(struct p_rs_param
)
849 : apv
== 88 ? sizeof(struct p_rs_param
)
850 + strlen(mdev
->tconn
->net_conf
->verify_alg
) + 1
851 : apv
<= 94 ? sizeof(struct p_rs_param_89
)
852 : /* apv >= 95 */ sizeof(struct p_rs_param_95
);
854 cmd
= apv
>= 89 ? P_SYNC_PARAM89
: P_SYNC_PARAM
;
856 /* initialize verify_alg and csums_alg */
857 memset(p
->verify_alg
, 0, 2 * SHARED_SECRET_MAX
);
859 if (get_ldev(mdev
)) {
860 p
->rate
= cpu_to_be32(mdev
->ldev
->dc
.resync_rate
);
861 p
->c_plan_ahead
= cpu_to_be32(mdev
->ldev
->dc
.c_plan_ahead
);
862 p
->c_delay_target
= cpu_to_be32(mdev
->ldev
->dc
.c_delay_target
);
863 p
->c_fill_target
= cpu_to_be32(mdev
->ldev
->dc
.c_fill_target
);
864 p
->c_max_rate
= cpu_to_be32(mdev
->ldev
->dc
.c_max_rate
);
867 p
->rate
= cpu_to_be32(DRBD_RATE_DEF
);
868 p
->c_plan_ahead
= cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF
);
869 p
->c_delay_target
= cpu_to_be32(DRBD_C_DELAY_TARGET_DEF
);
870 p
->c_fill_target
= cpu_to_be32(DRBD_C_FILL_TARGET_DEF
);
871 p
->c_max_rate
= cpu_to_be32(DRBD_C_MAX_RATE_DEF
);
875 strcpy(p
->verify_alg
, mdev
->tconn
->net_conf
->verify_alg
);
877 strcpy(p
->csums_alg
, mdev
->tconn
->net_conf
->csums_alg
);
879 return drbd_send_command(mdev
, sock
, cmd
, size
, NULL
, 0);
882 int drbd_send_protocol(struct drbd_tconn
*tconn
)
884 struct drbd_socket
*sock
;
885 struct p_protocol
*p
;
888 if (tconn
->net_conf
->dry_run
&& tconn
->agreed_pro_version
< 92) {
889 conn_err(tconn
, "--dry-run is not supported by peer");
894 p
= conn_prepare_command(tconn
, sock
);
899 if (tconn
->agreed_pro_version
>= 87)
900 size
+= strlen(tconn
->net_conf
->integrity_alg
) + 1;
902 p
->protocol
= cpu_to_be32(tconn
->net_conf
->wire_protocol
);
903 p
->after_sb_0p
= cpu_to_be32(tconn
->net_conf
->after_sb_0p
);
904 p
->after_sb_1p
= cpu_to_be32(tconn
->net_conf
->after_sb_1p
);
905 p
->after_sb_2p
= cpu_to_be32(tconn
->net_conf
->after_sb_2p
);
906 p
->two_primaries
= cpu_to_be32(tconn
->net_conf
->two_primaries
);
908 if (tconn
->net_conf
->want_lose
)
910 if (tconn
->net_conf
->dry_run
)
912 p
->conn_flags
= cpu_to_be32(cf
);
914 if (tconn
->agreed_pro_version
>= 87)
915 strcpy(p
->integrity_alg
, tconn
->net_conf
->integrity_alg
);
916 return conn_send_command(tconn
, sock
, P_PROTOCOL
, size
, NULL
, 0);
919 int _drbd_send_uuids(struct drbd_conf
*mdev
, u64 uuid_flags
)
921 struct drbd_socket
*sock
;
925 if (!get_ldev_if_state(mdev
, D_NEGOTIATING
))
928 sock
= &mdev
->tconn
->data
;
929 p
= drbd_prepare_command(mdev
, sock
);
934 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
935 p
->uuid
[i
] = mdev
->ldev
? cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]) : 0;
937 mdev
->comm_bm_set
= drbd_bm_total_weight(mdev
);
938 p
->uuid
[UI_SIZE
] = cpu_to_be64(mdev
->comm_bm_set
);
939 uuid_flags
|= mdev
->tconn
->net_conf
->want_lose
? 1 : 0;
940 uuid_flags
|= test_bit(CRASHED_PRIMARY
, &mdev
->flags
) ? 2 : 0;
941 uuid_flags
|= mdev
->new_state_tmp
.disk
== D_INCONSISTENT
? 4 : 0;
942 p
->uuid
[UI_FLAGS
] = cpu_to_be64(uuid_flags
);
945 return drbd_send_command(mdev
, sock
, P_UUIDS
, sizeof(*p
), NULL
, 0);
948 int drbd_send_uuids(struct drbd_conf
*mdev
)
950 return _drbd_send_uuids(mdev
, 0);
953 int drbd_send_uuids_skip_initial_sync(struct drbd_conf
*mdev
)
955 return _drbd_send_uuids(mdev
, 8);
958 void drbd_print_uuids(struct drbd_conf
*mdev
, const char *text
)
960 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
961 u64
*uuid
= mdev
->ldev
->md
.uuid
;
962 dev_info(DEV
, "%s %016llX:%016llX:%016llX:%016llX\n",
964 (unsigned long long)uuid
[UI_CURRENT
],
965 (unsigned long long)uuid
[UI_BITMAP
],
966 (unsigned long long)uuid
[UI_HISTORY_START
],
967 (unsigned long long)uuid
[UI_HISTORY_END
]);
970 dev_info(DEV
, "%s effective data uuid: %016llX\n",
972 (unsigned long long)mdev
->ed_uuid
);
976 void drbd_gen_and_send_sync_uuid(struct drbd_conf
*mdev
)
978 struct drbd_socket
*sock
;
982 D_ASSERT(mdev
->state
.disk
== D_UP_TO_DATE
);
984 uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
] + UUID_NEW_BM_OFFSET
;
985 drbd_uuid_set(mdev
, UI_BITMAP
, uuid
);
986 drbd_print_uuids(mdev
, "updated sync UUID");
989 sock
= &mdev
->tconn
->data
;
990 p
= drbd_prepare_command(mdev
, sock
);
992 p
->uuid
= cpu_to_be64(uuid
);
993 drbd_send_command(mdev
, sock
, P_SYNC_UUID
, sizeof(*p
), NULL
, 0);
997 int drbd_send_sizes(struct drbd_conf
*mdev
, int trigger_reply
, enum dds_flags flags
)
999 struct drbd_socket
*sock
;
1001 sector_t d_size
, u_size
;
1002 int q_order_type
, max_bio_size
;
1004 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
1005 D_ASSERT(mdev
->ldev
->backing_bdev
);
1006 d_size
= drbd_get_max_capacity(mdev
->ldev
);
1007 u_size
= mdev
->ldev
->dc
.disk_size
;
1008 q_order_type
= drbd_queue_order_type(mdev
);
1009 max_bio_size
= queue_max_hw_sectors(mdev
->ldev
->backing_bdev
->bd_disk
->queue
) << 9;
1010 max_bio_size
= min_t(int, max_bio_size
, DRBD_MAX_BIO_SIZE
);
1015 q_order_type
= QUEUE_ORDERED_NONE
;
1016 max_bio_size
= DRBD_MAX_BIO_SIZE
; /* ... multiple BIOs per peer_request */
1019 sock
= &mdev
->tconn
->data
;
1020 p
= drbd_prepare_command(mdev
, sock
);
1023 p
->d_size
= cpu_to_be64(d_size
);
1024 p
->u_size
= cpu_to_be64(u_size
);
1025 p
->c_size
= cpu_to_be64(trigger_reply
? 0 : drbd_get_capacity(mdev
->this_bdev
));
1026 p
->max_bio_size
= cpu_to_be32(max_bio_size
);
1027 p
->queue_order_type
= cpu_to_be16(q_order_type
);
1028 p
->dds_flags
= cpu_to_be16(flags
);
1029 return drbd_send_command(mdev
, sock
, P_SIZES
, sizeof(*p
), NULL
, 0);
1033 * drbd_send_state() - Sends the drbd state to the peer
1034 * @mdev: DRBD device.
1036 int drbd_send_state(struct drbd_conf
*mdev
)
1038 struct drbd_socket
*sock
;
1041 sock
= &mdev
->tconn
->data
;
1042 p
= drbd_prepare_command(mdev
, sock
);
1045 p
->state
= cpu_to_be32(mdev
->state
.i
); /* Within the send mutex */
1046 return drbd_send_command(mdev
, sock
, P_STATE
, sizeof(*p
), NULL
, 0);
1049 int drbd_send_state_req(struct drbd_conf
*mdev
, union drbd_state mask
, union drbd_state val
)
1051 struct drbd_socket
*sock
;
1052 struct p_req_state
*p
;
1054 sock
= &mdev
->tconn
->data
;
1055 p
= drbd_prepare_command(mdev
, sock
);
1058 p
->mask
= cpu_to_be32(mask
.i
);
1059 p
->val
= cpu_to_be32(val
.i
);
1060 return drbd_send_command(mdev
, sock
, P_STATE_CHG_REQ
, sizeof(*p
), NULL
, 0);
1064 int conn_send_state_req(struct drbd_tconn
*tconn
, union drbd_state mask
, union drbd_state val
)
1066 enum drbd_packet cmd
;
1067 struct drbd_socket
*sock
;
1068 struct p_req_state
*p
;
1070 cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REQ
: P_CONN_ST_CHG_REQ
;
1071 sock
= &tconn
->data
;
1072 p
= conn_prepare_command(tconn
, sock
);
1075 p
->mask
= cpu_to_be32(mask
.i
);
1076 p
->val
= cpu_to_be32(val
.i
);
1077 return conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1080 void drbd_send_sr_reply(struct drbd_conf
*mdev
, enum drbd_state_rv retcode
)
1082 struct drbd_socket
*sock
;
1083 struct p_req_state_reply
*p
;
1085 sock
= &mdev
->tconn
->meta
;
1086 p
= drbd_prepare_command(mdev
, sock
);
1088 p
->retcode
= cpu_to_be32(retcode
);
1089 drbd_send_command(mdev
, sock
, P_STATE_CHG_REPLY
, sizeof(*p
), NULL
, 0);
1093 void conn_send_sr_reply(struct drbd_tconn
*tconn
, enum drbd_state_rv retcode
)
1095 struct drbd_socket
*sock
;
1096 struct p_req_state_reply
*p
;
1097 enum drbd_packet cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REPLY
: P_CONN_ST_CHG_REPLY
;
1099 sock
= &tconn
->meta
;
1100 p
= conn_prepare_command(tconn
, sock
);
1102 p
->retcode
= cpu_to_be32(retcode
);
1103 conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1107 static void dcbp_set_code(struct p_compressed_bm
*p
, enum drbd_bitmap_code code
)
1109 BUG_ON(code
& ~0xf);
1110 p
->encoding
= (p
->encoding
& ~0xf) | code
;
1113 static void dcbp_set_start(struct p_compressed_bm
*p
, int set
)
1115 p
->encoding
= (p
->encoding
& ~0x80) | (set
? 0x80 : 0);
1118 static void dcbp_set_pad_bits(struct p_compressed_bm
*p
, int n
)
1121 p
->encoding
= (p
->encoding
& (~0x7 << 4)) | (n
<< 4);
1124 int fill_bitmap_rle_bits(struct drbd_conf
*mdev
,
1125 struct p_compressed_bm
*p
,
1127 struct bm_xfer_ctx
*c
)
1129 struct bitstream bs
;
1130 unsigned long plain_bits
;
1137 /* may we use this feature? */
1138 if ((mdev
->tconn
->net_conf
->use_rle
== 0) ||
1139 (mdev
->tconn
->agreed_pro_version
< 90))
1142 if (c
->bit_offset
>= c
->bm_bits
)
1143 return 0; /* nothing to do. */
1145 /* use at most thus many bytes */
1146 bitstream_init(&bs
, p
->code
, size
, 0);
1147 memset(p
->code
, 0, size
);
1148 /* plain bits covered in this code string */
1151 /* p->encoding & 0x80 stores whether the first run length is set.
1152 * bit offset is implicit.
1153 * start with toggle == 2 to be able to tell the first iteration */
1156 /* see how much plain bits we can stuff into one packet
1157 * using RLE and VLI. */
1159 tmp
= (toggle
== 0) ? _drbd_bm_find_next_zero(mdev
, c
->bit_offset
)
1160 : _drbd_bm_find_next(mdev
, c
->bit_offset
);
1163 rl
= tmp
- c
->bit_offset
;
1165 if (toggle
== 2) { /* first iteration */
1167 /* the first checked bit was set,
1168 * store start value, */
1169 dcbp_set_start(p
, 1);
1170 /* but skip encoding of zero run length */
1174 dcbp_set_start(p
, 0);
1177 /* paranoia: catch zero runlength.
1178 * can only happen if bitmap is modified while we scan it. */
1180 dev_err(DEV
, "unexpected zero runlength while encoding bitmap "
1181 "t:%u bo:%lu\n", toggle
, c
->bit_offset
);
1185 bits
= vli_encode_bits(&bs
, rl
);
1186 if (bits
== -ENOBUFS
) /* buffer full */
1189 dev_err(DEV
, "error while encoding bitmap: %d\n", bits
);
1195 c
->bit_offset
= tmp
;
1196 } while (c
->bit_offset
< c
->bm_bits
);
1198 len
= bs
.cur
.b
- p
->code
+ !!bs
.cur
.bit
;
1200 if (plain_bits
< (len
<< 3)) {
1201 /* incompressible with this method.
1202 * we need to rewind both word and bit position. */
1203 c
->bit_offset
-= plain_bits
;
1204 bm_xfer_ctx_bit_to_word_offset(c
);
1205 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1209 /* RLE + VLI was able to compress it just fine.
1210 * update c->word_offset. */
1211 bm_xfer_ctx_bit_to_word_offset(c
);
1213 /* store pad_bits */
1214 dcbp_set_pad_bits(p
, (8 - bs
.cur
.bit
) & 0x7);
1220 * send_bitmap_rle_or_plain
1222 * Return 0 when done, 1 when another iteration is needed, and a negative error
1223 * code upon failure.
1226 send_bitmap_rle_or_plain(struct drbd_conf
*mdev
, struct bm_xfer_ctx
*c
)
1228 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1229 unsigned int header_size
= drbd_header_size(mdev
->tconn
);
1230 struct p_compressed_bm
*p
= sock
->sbuf
+ header_size
;
1233 len
= fill_bitmap_rle_bits(mdev
, p
,
1234 DRBD_SOCKET_BUFFER_SIZE
- header_size
- sizeof(*p
), c
);
1239 dcbp_set_code(p
, RLE_VLI_Bits
);
1240 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
,
1241 P_COMPRESSED_BITMAP
, sizeof(*p
) + len
,
1244 c
->bytes
[0] += header_size
+ sizeof(*p
) + len
;
1246 if (c
->bit_offset
>= c
->bm_bits
)
1249 /* was not compressible.
1250 * send a buffer full of plain text bits instead. */
1251 unsigned int data_size
;
1252 unsigned long num_words
;
1253 unsigned long *p
= sock
->sbuf
+ header_size
;
1255 data_size
= DRBD_SOCKET_BUFFER_SIZE
- header_size
;
1256 num_words
= min_t(size_t, data_size
/ sizeof(*p
),
1257 c
->bm_words
- c
->word_offset
);
1258 len
= num_words
* sizeof(*p
);
1260 drbd_bm_get_lel(mdev
, c
->word_offset
, num_words
, p
);
1261 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_BITMAP
, len
, NULL
, 0);
1262 c
->word_offset
+= num_words
;
1263 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1266 c
->bytes
[1] += header_size
+ len
;
1268 if (c
->bit_offset
> c
->bm_bits
)
1269 c
->bit_offset
= c
->bm_bits
;
1273 INFO_bm_xfer_stats(mdev
, "send", c
);
1281 /* See the comment at receive_bitmap() */
1282 static int _drbd_send_bitmap(struct drbd_conf
*mdev
)
1284 struct bm_xfer_ctx c
;
1287 if (!expect(mdev
->bitmap
))
1290 if (get_ldev(mdev
)) {
1291 if (drbd_md_test_flag(mdev
->ldev
, MDF_FULL_SYNC
)) {
1292 dev_info(DEV
, "Writing the whole bitmap, MDF_FullSync was set.\n");
1293 drbd_bm_set_all(mdev
);
1294 if (drbd_bm_write(mdev
)) {
1295 /* write_bm did fail! Leave full sync flag set in Meta P_DATA
1296 * but otherwise process as per normal - need to tell other
1297 * side that a full resync is required! */
1298 dev_err(DEV
, "Failed to write bitmap to disk!\n");
1300 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
1307 c
= (struct bm_xfer_ctx
) {
1308 .bm_bits
= drbd_bm_bits(mdev
),
1309 .bm_words
= drbd_bm_words(mdev
),
1313 err
= send_bitmap_rle_or_plain(mdev
, &c
);
1319 int drbd_send_bitmap(struct drbd_conf
*mdev
)
1321 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1324 mutex_lock(&sock
->mutex
);
1326 err
= !_drbd_send_bitmap(mdev
);
1327 mutex_unlock(&sock
->mutex
);
1331 void drbd_send_b_ack(struct drbd_conf
*mdev
, u32 barrier_nr
, u32 set_size
)
1333 struct drbd_socket
*sock
;
1334 struct p_barrier_ack
*p
;
1336 if (mdev
->state
.conn
< C_CONNECTED
)
1339 sock
= &mdev
->tconn
->meta
;
1340 p
= drbd_prepare_command(mdev
, sock
);
1343 p
->barrier
= barrier_nr
;
1344 p
->set_size
= cpu_to_be32(set_size
);
1345 drbd_send_command(mdev
, sock
, P_BARRIER_ACK
, sizeof(*p
), NULL
, 0);
1349 * _drbd_send_ack() - Sends an ack packet
1350 * @mdev: DRBD device.
1351 * @cmd: Packet command code.
1352 * @sector: sector, needs to be in big endian byte order
1353 * @blksize: size in byte, needs to be in big endian byte order
1354 * @block_id: Id, big endian byte order
1356 static int _drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1357 u64 sector
, u32 blksize
, u64 block_id
)
1359 struct drbd_socket
*sock
;
1360 struct p_block_ack
*p
;
1362 if (mdev
->state
.conn
< C_CONNECTED
)
1365 sock
= &mdev
->tconn
->meta
;
1366 p
= drbd_prepare_command(mdev
, sock
);
1370 p
->block_id
= block_id
;
1371 p
->blksize
= blksize
;
1372 p
->seq_num
= cpu_to_be32(atomic_inc_return(&mdev
->packet_seq
));
1373 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1376 /* dp->sector and dp->block_id already/still in network byte order,
1377 * data_size is payload size according to dp->head,
1378 * and may need to be corrected for digest size. */
1379 void drbd_send_ack_dp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1380 struct p_data
*dp
, int data_size
)
1382 data_size
-= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_r_tfm
) ?
1383 crypto_hash_digestsize(mdev
->tconn
->integrity_r_tfm
) : 0;
1384 _drbd_send_ack(mdev
, cmd
, dp
->sector
, cpu_to_be32(data_size
),
1388 void drbd_send_ack_rp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1389 struct p_block_req
*rp
)
1391 _drbd_send_ack(mdev
, cmd
, rp
->sector
, rp
->blksize
, rp
->block_id
);
1395 * drbd_send_ack() - Sends an ack packet
1396 * @mdev: DRBD device
1397 * @cmd: packet command code
1398 * @peer_req: peer request
1400 int drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1401 struct drbd_peer_request
*peer_req
)
1403 return _drbd_send_ack(mdev
, cmd
,
1404 cpu_to_be64(peer_req
->i
.sector
),
1405 cpu_to_be32(peer_req
->i
.size
),
1406 peer_req
->block_id
);
1409 /* This function misuses the block_id field to signal if the blocks
1410 * are is sync or not. */
1411 int drbd_send_ack_ex(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1412 sector_t sector
, int blksize
, u64 block_id
)
1414 return _drbd_send_ack(mdev
, cmd
,
1415 cpu_to_be64(sector
),
1416 cpu_to_be32(blksize
),
1417 cpu_to_be64(block_id
));
1420 int drbd_send_drequest(struct drbd_conf
*mdev
, int cmd
,
1421 sector_t sector
, int size
, u64 block_id
)
1423 struct drbd_socket
*sock
;
1424 struct p_block_req
*p
;
1426 sock
= &mdev
->tconn
->data
;
1427 p
= drbd_prepare_command(mdev
, sock
);
1430 p
->sector
= cpu_to_be64(sector
);
1431 p
->block_id
= block_id
;
1432 p
->blksize
= cpu_to_be32(size
);
1433 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1436 int drbd_send_drequest_csum(struct drbd_conf
*mdev
, sector_t sector
, int size
,
1437 void *digest
, int digest_size
, enum drbd_packet cmd
)
1439 struct drbd_socket
*sock
;
1440 struct p_block_req
*p
;
1442 /* FIXME: Put the digest into the preallocated socket buffer. */
1444 sock
= &mdev
->tconn
->data
;
1445 p
= drbd_prepare_command(mdev
, sock
);
1448 p
->sector
= cpu_to_be64(sector
);
1449 p
->block_id
= ID_SYNCER
/* unused */;
1450 p
->blksize
= cpu_to_be32(size
);
1451 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
),
1452 digest
, digest_size
);
1455 int drbd_send_ov_request(struct drbd_conf
*mdev
, sector_t sector
, int size
)
1457 struct drbd_socket
*sock
;
1458 struct p_block_req
*p
;
1460 sock
= &mdev
->tconn
->data
;
1461 p
= drbd_prepare_command(mdev
, sock
);
1464 p
->sector
= cpu_to_be64(sector
);
1465 p
->block_id
= ID_SYNCER
/* unused */;
1466 p
->blksize
= cpu_to_be32(size
);
1467 return drbd_send_command(mdev
, sock
, P_OV_REQUEST
, sizeof(*p
), NULL
, 0);
1470 /* called on sndtimeo
1471 * returns false if we should retry,
1472 * true if we think connection is dead
1474 static int we_should_drop_the_connection(struct drbd_tconn
*tconn
, struct socket
*sock
)
1477 /* long elapsed = (long)(jiffies - mdev->last_received); */
1479 drop_it
= tconn
->meta
.socket
== sock
1480 || !tconn
->asender
.task
1481 || get_t_state(&tconn
->asender
) != RUNNING
1482 || tconn
->cstate
< C_WF_REPORT_PARAMS
;
1487 drop_it
= !--tconn
->ko_count
;
1489 conn_err(tconn
, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1490 current
->comm
, current
->pid
, tconn
->ko_count
);
1491 request_ping(tconn
);
1494 return drop_it
; /* && (mdev->state == R_PRIMARY) */;
1497 static void drbd_update_congested(struct drbd_tconn
*tconn
)
1499 struct sock
*sk
= tconn
->data
.socket
->sk
;
1500 if (sk
->sk_wmem_queued
> sk
->sk_sndbuf
* 4 / 5)
1501 set_bit(NET_CONGESTED
, &tconn
->flags
);
1504 /* The idea of sendpage seems to be to put some kind of reference
1505 * to the page into the skb, and to hand it over to the NIC. In
1506 * this process get_page() gets called.
1508 * As soon as the page was really sent over the network put_page()
1509 * gets called by some part of the network layer. [ NIC driver? ]
1511 * [ get_page() / put_page() increment/decrement the count. If count
1512 * reaches 0 the page will be freed. ]
1514 * This works nicely with pages from FSs.
1515 * But this means that in protocol A we might signal IO completion too early!
1517 * In order not to corrupt data during a resync we must make sure
1518 * that we do not reuse our own buffer pages (EEs) to early, therefore
1519 * we have the net_ee list.
1521 * XFS seems to have problems, still, it submits pages with page_count == 0!
1522 * As a workaround, we disable sendpage on pages
1523 * with page_count == 0 or PageSlab.
1525 static int _drbd_no_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1526 int offset
, size_t size
, unsigned msg_flags
)
1528 struct socket
*socket
;
1532 socket
= mdev
->tconn
->data
.socket
;
1533 addr
= kmap(page
) + offset
;
1534 err
= drbd_send_all(mdev
->tconn
, socket
, addr
, size
, msg_flags
);
1537 mdev
->send_cnt
+= size
>> 9;
1541 static int _drbd_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1542 int offset
, size_t size
, unsigned msg_flags
)
1544 struct socket
*socket
= mdev
->tconn
->data
.socket
;
1545 mm_segment_t oldfs
= get_fs();
1549 /* e.g. XFS meta- & log-data is in slab pages, which have a
1550 * page_count of 0 and/or have PageSlab() set.
1551 * we cannot use send_page for those, as that does get_page();
1552 * put_page(); and would cause either a VM_BUG directly, or
1553 * __page_cache_release a page that would actually still be referenced
1554 * by someone, leading to some obscure delayed Oops somewhere else. */
1555 if (disable_sendpage
|| (page_count(page
) < 1) || PageSlab(page
))
1556 return _drbd_no_send_page(mdev
, page
, offset
, size
, msg_flags
);
1558 msg_flags
|= MSG_NOSIGNAL
;
1559 drbd_update_congested(mdev
->tconn
);
1564 sent
= socket
->ops
->sendpage(socket
, page
, offset
, len
, msg_flags
);
1566 if (sent
== -EAGAIN
) {
1567 if (we_should_drop_the_connection(mdev
->tconn
, socket
))
1571 dev_warn(DEV
, "%s: size=%d len=%d sent=%d\n",
1572 __func__
, (int)size
, len
, sent
);
1579 } while (len
> 0 /* THINK && mdev->cstate >= C_CONNECTED*/);
1581 clear_bit(NET_CONGESTED
, &mdev
->tconn
->flags
);
1585 mdev
->send_cnt
+= size
>> 9;
1590 static int _drbd_send_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1592 struct bio_vec
*bvec
;
1594 /* hint all but last page with MSG_MORE */
1595 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1598 err
= _drbd_no_send_page(mdev
, bvec
->bv_page
,
1599 bvec
->bv_offset
, bvec
->bv_len
,
1600 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1607 static int _drbd_send_zc_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1609 struct bio_vec
*bvec
;
1611 /* hint all but last page with MSG_MORE */
1612 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1615 err
= _drbd_send_page(mdev
, bvec
->bv_page
,
1616 bvec
->bv_offset
, bvec
->bv_len
,
1617 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1624 static int _drbd_send_zc_ee(struct drbd_conf
*mdev
,
1625 struct drbd_peer_request
*peer_req
)
1627 struct page
*page
= peer_req
->pages
;
1628 unsigned len
= peer_req
->i
.size
;
1631 /* hint all but last page with MSG_MORE */
1632 page_chain_for_each(page
) {
1633 unsigned l
= min_t(unsigned, len
, PAGE_SIZE
);
1635 err
= _drbd_send_page(mdev
, page
, 0, l
,
1636 page_chain_next(page
) ? MSG_MORE
: 0);
1644 static u32
bio_flags_to_wire(struct drbd_conf
*mdev
, unsigned long bi_rw
)
1646 if (mdev
->tconn
->agreed_pro_version
>= 95)
1647 return (bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0) |
1648 (bi_rw
& REQ_FUA
? DP_FUA
: 0) |
1649 (bi_rw
& REQ_FLUSH
? DP_FLUSH
: 0) |
1650 (bi_rw
& REQ_DISCARD
? DP_DISCARD
: 0);
1652 return bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0;
1655 /* Used to send write requests
1656 * R_PRIMARY -> Peer (P_DATA)
1658 int drbd_send_dblock(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1660 struct drbd_socket
*sock
;
1662 unsigned int dp_flags
= 0;
1666 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1667 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1669 sock
= &mdev
->tconn
->data
;
1670 p
= drbd_prepare_command(mdev
, sock
);
1673 p
->sector
= cpu_to_be64(req
->i
.sector
);
1674 p
->block_id
= (unsigned long)req
;
1675 p
->seq_num
= cpu_to_be32(req
->seq_num
= atomic_inc_return(&mdev
->packet_seq
));
1676 dp_flags
= bio_flags_to_wire(mdev
, req
->master_bio
->bi_rw
);
1677 if (mdev
->state
.conn
>= C_SYNC_SOURCE
&&
1678 mdev
->state
.conn
<= C_PAUSED_SYNC_T
)
1679 dp_flags
|= DP_MAY_SET_IN_SYNC
;
1680 p
->dp_flags
= cpu_to_be32(dp_flags
);
1682 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, p
+ 1);
1683 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_DATA
, sizeof(*p
) + dgs
, NULL
, req
->i
.size
);
1685 /* For protocol A, we have to memcpy the payload into
1686 * socket buffers, as we may complete right away
1687 * as soon as we handed it over to tcp, at which point the data
1688 * pages may become invalid.
1690 * For data-integrity enabled, we copy it as well, so we can be
1691 * sure that even if the bio pages may still be modified, it
1692 * won't change the data on the wire, thus if the digest checks
1693 * out ok after sending on this side, but does not fit on the
1694 * receiving side, we sure have detected corruption elsewhere.
1696 if (mdev
->tconn
->net_conf
->wire_protocol
== DRBD_PROT_A
|| dgs
)
1697 err
= _drbd_send_bio(mdev
, req
->master_bio
);
1699 err
= _drbd_send_zc_bio(mdev
, req
->master_bio
);
1701 /* double check digest, sometimes buffers have been modified in flight. */
1702 if (dgs
> 0 && dgs
<= 64) {
1703 /* 64 byte, 512 bit, is the largest digest size
1704 * currently supported in kernel crypto. */
1705 unsigned char digest
[64];
1706 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, digest
);
1707 if (memcmp(p
+ 1, digest
, dgs
)) {
1709 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1710 (unsigned long long)req
->i
.sector
, req
->i
.size
);
1712 } /* else if (dgs > 64) {
1713 ... Be noisy about digest too large ...
1716 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1721 /* answer packet, used to send data back for read requests:
1722 * Peer -> (diskless) R_PRIMARY (P_DATA_REPLY)
1723 * C_SYNC_SOURCE -> C_SYNC_TARGET (P_RS_DATA_REPLY)
1725 int drbd_send_block(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1726 struct drbd_peer_request
*peer_req
)
1728 struct drbd_socket
*sock
;
1733 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1734 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1736 sock
= &mdev
->tconn
->data
;
1737 p
= drbd_prepare_command(mdev
, sock
);
1740 p
->sector
= cpu_to_be64(peer_req
->i
.sector
);
1741 p
->block_id
= peer_req
->block_id
;
1742 p
->seq_num
= 0; /* unused */
1744 drbd_csum_ee(mdev
, mdev
->tconn
->integrity_w_tfm
, peer_req
, p
+ 1);
1745 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, sizeof(*p
) + dgs
, NULL
, peer_req
->i
.size
);
1747 err
= _drbd_send_zc_ee(mdev
, peer_req
);
1748 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1753 int drbd_send_out_of_sync(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1755 struct drbd_socket
*sock
;
1756 struct p_block_desc
*p
;
1758 sock
= &mdev
->tconn
->data
;
1759 p
= drbd_prepare_command(mdev
, sock
);
1762 p
->sector
= cpu_to_be64(req
->i
.sector
);
1763 p
->blksize
= cpu_to_be32(req
->i
.size
);
1764 return drbd_send_command(mdev
, sock
, P_OUT_OF_SYNC
, sizeof(*p
), NULL
, 0);
1768 drbd_send distinguishes two cases:
1770 Packets sent via the data socket "sock"
1771 and packets sent via the meta data socket "msock"
1774 -----------------+-------------------------+------------------------------
1775 timeout conf.timeout / 2 conf.timeout / 2
1776 timeout action send a ping via msock Abort communication
1777 and close all sockets
1781 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
1783 int drbd_send(struct drbd_tconn
*tconn
, struct socket
*sock
,
1784 void *buf
, size_t size
, unsigned msg_flags
)
1793 /* THINK if (signal_pending) return ... ? */
1798 msg
.msg_name
= NULL
;
1799 msg
.msg_namelen
= 0;
1800 msg
.msg_control
= NULL
;
1801 msg
.msg_controllen
= 0;
1802 msg
.msg_flags
= msg_flags
| MSG_NOSIGNAL
;
1804 if (sock
== tconn
->data
.socket
) {
1805 tconn
->ko_count
= tconn
->net_conf
->ko_count
;
1806 drbd_update_congested(tconn
);
1810 * tcp_sendmsg does _not_ use its size parameter at all ?
1812 * -EAGAIN on timeout, -EINTR on signal.
1815 * do we need to block DRBD_SIG if sock == &meta.socket ??
1816 * otherwise wake_asender() might interrupt some send_*Ack !
1818 rv
= kernel_sendmsg(sock
, &msg
, &iov
, 1, size
);
1819 if (rv
== -EAGAIN
) {
1820 if (we_should_drop_the_connection(tconn
, sock
))
1826 flush_signals(current
);
1834 } while (sent
< size
);
1836 if (sock
== tconn
->data
.socket
)
1837 clear_bit(NET_CONGESTED
, &tconn
->flags
);
1840 if (rv
!= -EAGAIN
) {
1841 conn_err(tconn
, "%s_sendmsg returned %d\n",
1842 sock
== tconn
->meta
.socket
? "msock" : "sock",
1844 conn_request_state(tconn
, NS(conn
, C_BROKEN_PIPE
), CS_HARD
);
1846 conn_request_state(tconn
, NS(conn
, C_TIMEOUT
), CS_HARD
);
1853 * drbd_send_all - Send an entire buffer
1855 * Returns 0 upon success and a negative error value otherwise.
1857 int drbd_send_all(struct drbd_tconn
*tconn
, struct socket
*sock
, void *buffer
,
1858 size_t size
, unsigned msg_flags
)
1862 err
= drbd_send(tconn
, sock
, buffer
, size
, msg_flags
);
1870 static int drbd_open(struct block_device
*bdev
, fmode_t mode
)
1872 struct drbd_conf
*mdev
= bdev
->bd_disk
->private_data
;
1873 unsigned long flags
;
1876 mutex_lock(&drbd_main_mutex
);
1877 spin_lock_irqsave(&mdev
->tconn
->req_lock
, flags
);
1878 /* to have a stable mdev->state.role
1879 * and no race with updating open_cnt */
1881 if (mdev
->state
.role
!= R_PRIMARY
) {
1882 if (mode
& FMODE_WRITE
)
1884 else if (!allow_oos
)
1890 spin_unlock_irqrestore(&mdev
->tconn
->req_lock
, flags
);
1891 mutex_unlock(&drbd_main_mutex
);
1896 static int drbd_release(struct gendisk
*gd
, fmode_t mode
)
1898 struct drbd_conf
*mdev
= gd
->private_data
;
1899 mutex_lock(&drbd_main_mutex
);
1901 mutex_unlock(&drbd_main_mutex
);
1905 static void drbd_set_defaults(struct drbd_conf
*mdev
)
1907 /* Beware! The actual layout differs
1908 * between big endian and little endian */
1909 mdev
->state
= (union drbd_dev_state
) {
1910 { .role
= R_SECONDARY
,
1912 .conn
= C_STANDALONE
,
1918 void drbd_init_set_defaults(struct drbd_conf
*mdev
)
1920 /* the memset(,0,) did most of this.
1921 * note: only assignments, no allocation in here */
1923 drbd_set_defaults(mdev
);
1925 atomic_set(&mdev
->ap_bio_cnt
, 0);
1926 atomic_set(&mdev
->ap_pending_cnt
, 0);
1927 atomic_set(&mdev
->rs_pending_cnt
, 0);
1928 atomic_set(&mdev
->unacked_cnt
, 0);
1929 atomic_set(&mdev
->local_cnt
, 0);
1930 atomic_set(&mdev
->pp_in_use_by_net
, 0);
1931 atomic_set(&mdev
->rs_sect_in
, 0);
1932 atomic_set(&mdev
->rs_sect_ev
, 0);
1933 atomic_set(&mdev
->ap_in_flight
, 0);
1935 mutex_init(&mdev
->md_io_mutex
);
1936 mutex_init(&mdev
->own_state_mutex
);
1937 mdev
->state_mutex
= &mdev
->own_state_mutex
;
1939 spin_lock_init(&mdev
->al_lock
);
1940 spin_lock_init(&mdev
->peer_seq_lock
);
1941 spin_lock_init(&mdev
->epoch_lock
);
1943 INIT_LIST_HEAD(&mdev
->active_ee
);
1944 INIT_LIST_HEAD(&mdev
->sync_ee
);
1945 INIT_LIST_HEAD(&mdev
->done_ee
);
1946 INIT_LIST_HEAD(&mdev
->read_ee
);
1947 INIT_LIST_HEAD(&mdev
->net_ee
);
1948 INIT_LIST_HEAD(&mdev
->resync_reads
);
1949 INIT_LIST_HEAD(&mdev
->resync_work
.list
);
1950 INIT_LIST_HEAD(&mdev
->unplug_work
.list
);
1951 INIT_LIST_HEAD(&mdev
->go_diskless
.list
);
1952 INIT_LIST_HEAD(&mdev
->md_sync_work
.list
);
1953 INIT_LIST_HEAD(&mdev
->start_resync_work
.list
);
1954 INIT_LIST_HEAD(&mdev
->bm_io_work
.w
.list
);
1956 mdev
->resync_work
.cb
= w_resync_timer
;
1957 mdev
->unplug_work
.cb
= w_send_write_hint
;
1958 mdev
->go_diskless
.cb
= w_go_diskless
;
1959 mdev
->md_sync_work
.cb
= w_md_sync
;
1960 mdev
->bm_io_work
.w
.cb
= w_bitmap_io
;
1961 mdev
->start_resync_work
.cb
= w_start_resync
;
1963 mdev
->resync_work
.mdev
= mdev
;
1964 mdev
->unplug_work
.mdev
= mdev
;
1965 mdev
->go_diskless
.mdev
= mdev
;
1966 mdev
->md_sync_work
.mdev
= mdev
;
1967 mdev
->bm_io_work
.w
.mdev
= mdev
;
1968 mdev
->start_resync_work
.mdev
= mdev
;
1970 init_timer(&mdev
->resync_timer
);
1971 init_timer(&mdev
->md_sync_timer
);
1972 init_timer(&mdev
->start_resync_timer
);
1973 init_timer(&mdev
->request_timer
);
1974 mdev
->resync_timer
.function
= resync_timer_fn
;
1975 mdev
->resync_timer
.data
= (unsigned long) mdev
;
1976 mdev
->md_sync_timer
.function
= md_sync_timer_fn
;
1977 mdev
->md_sync_timer
.data
= (unsigned long) mdev
;
1978 mdev
->start_resync_timer
.function
= start_resync_timer_fn
;
1979 mdev
->start_resync_timer
.data
= (unsigned long) mdev
;
1980 mdev
->request_timer
.function
= request_timer_fn
;
1981 mdev
->request_timer
.data
= (unsigned long) mdev
;
1983 init_waitqueue_head(&mdev
->misc_wait
);
1984 init_waitqueue_head(&mdev
->state_wait
);
1985 init_waitqueue_head(&mdev
->ee_wait
);
1986 init_waitqueue_head(&mdev
->al_wait
);
1987 init_waitqueue_head(&mdev
->seq_wait
);
1989 /* mdev->tconn->agreed_pro_version gets initialized in drbd_connect() */
1990 mdev
->write_ordering
= WO_bdev_flush
;
1991 mdev
->resync_wenr
= LC_FREE
;
1992 mdev
->peer_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1993 mdev
->local_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1996 void drbd_mdev_cleanup(struct drbd_conf
*mdev
)
1999 if (mdev
->tconn
->receiver
.t_state
!= NONE
)
2000 dev_err(DEV
, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
2001 mdev
->tconn
->receiver
.t_state
);
2003 /* no need to lock it, I'm the only thread alive */
2004 if (atomic_read(&mdev
->current_epoch
->epoch_size
) != 0)
2005 dev_err(DEV
, "epoch_size:%d\n", atomic_read(&mdev
->current_epoch
->epoch_size
));
2015 mdev
->rs_failed
= 0;
2016 mdev
->rs_last_events
= 0;
2017 mdev
->rs_last_sect_ev
= 0;
2018 for (i
= 0; i
< DRBD_SYNC_MARKS
; i
++) {
2019 mdev
->rs_mark_left
[i
] = 0;
2020 mdev
->rs_mark_time
[i
] = 0;
2022 D_ASSERT(mdev
->tconn
->net_conf
== NULL
);
2024 drbd_set_my_capacity(mdev
, 0);
2026 /* maybe never allocated. */
2027 drbd_bm_resize(mdev
, 0, 1);
2028 drbd_bm_cleanup(mdev
);
2031 drbd_free_resources(mdev
);
2032 clear_bit(AL_SUSPENDED
, &mdev
->flags
);
2035 * currently we drbd_init_ee only on module load, so
2036 * we may do drbd_release_ee only on module unload!
2038 D_ASSERT(list_empty(&mdev
->active_ee
));
2039 D_ASSERT(list_empty(&mdev
->sync_ee
));
2040 D_ASSERT(list_empty(&mdev
->done_ee
));
2041 D_ASSERT(list_empty(&mdev
->read_ee
));
2042 D_ASSERT(list_empty(&mdev
->net_ee
));
2043 D_ASSERT(list_empty(&mdev
->resync_reads
));
2044 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2045 D_ASSERT(list_empty(&mdev
->tconn
->meta
.work
.q
));
2046 D_ASSERT(list_empty(&mdev
->resync_work
.list
));
2047 D_ASSERT(list_empty(&mdev
->unplug_work
.list
));
2048 D_ASSERT(list_empty(&mdev
->go_diskless
.list
));
2050 drbd_set_defaults(mdev
);
2054 static void drbd_destroy_mempools(void)
2058 while (drbd_pp_pool
) {
2059 page
= drbd_pp_pool
;
2060 drbd_pp_pool
= (struct page
*)page_private(page
);
2065 /* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */
2067 if (drbd_md_io_bio_set
)
2068 bioset_free(drbd_md_io_bio_set
);
2069 if (drbd_md_io_page_pool
)
2070 mempool_destroy(drbd_md_io_page_pool
);
2071 if (drbd_ee_mempool
)
2072 mempool_destroy(drbd_ee_mempool
);
2073 if (drbd_request_mempool
)
2074 mempool_destroy(drbd_request_mempool
);
2076 kmem_cache_destroy(drbd_ee_cache
);
2077 if (drbd_request_cache
)
2078 kmem_cache_destroy(drbd_request_cache
);
2079 if (drbd_bm_ext_cache
)
2080 kmem_cache_destroy(drbd_bm_ext_cache
);
2081 if (drbd_al_ext_cache
)
2082 kmem_cache_destroy(drbd_al_ext_cache
);
2084 drbd_md_io_bio_set
= NULL
;
2085 drbd_md_io_page_pool
= NULL
;
2086 drbd_ee_mempool
= NULL
;
2087 drbd_request_mempool
= NULL
;
2088 drbd_ee_cache
= NULL
;
2089 drbd_request_cache
= NULL
;
2090 drbd_bm_ext_cache
= NULL
;
2091 drbd_al_ext_cache
= NULL
;
2096 static int drbd_create_mempools(void)
2099 const int number
= (DRBD_MAX_BIO_SIZE
/PAGE_SIZE
) * minor_count
;
2102 /* prepare our caches and mempools */
2103 drbd_request_mempool
= NULL
;
2104 drbd_ee_cache
= NULL
;
2105 drbd_request_cache
= NULL
;
2106 drbd_bm_ext_cache
= NULL
;
2107 drbd_al_ext_cache
= NULL
;
2108 drbd_pp_pool
= NULL
;
2109 drbd_md_io_page_pool
= NULL
;
2110 drbd_md_io_bio_set
= NULL
;
2113 drbd_request_cache
= kmem_cache_create(
2114 "drbd_req", sizeof(struct drbd_request
), 0, 0, NULL
);
2115 if (drbd_request_cache
== NULL
)
2118 drbd_ee_cache
= kmem_cache_create(
2119 "drbd_ee", sizeof(struct drbd_peer_request
), 0, 0, NULL
);
2120 if (drbd_ee_cache
== NULL
)
2123 drbd_bm_ext_cache
= kmem_cache_create(
2124 "drbd_bm", sizeof(struct bm_extent
), 0, 0, NULL
);
2125 if (drbd_bm_ext_cache
== NULL
)
2128 drbd_al_ext_cache
= kmem_cache_create(
2129 "drbd_al", sizeof(struct lc_element
), 0, 0, NULL
);
2130 if (drbd_al_ext_cache
== NULL
)
2134 drbd_md_io_bio_set
= bioset_create(DRBD_MIN_POOL_PAGES
, 0);
2135 if (drbd_md_io_bio_set
== NULL
)
2138 drbd_md_io_page_pool
= mempool_create_page_pool(DRBD_MIN_POOL_PAGES
, 0);
2139 if (drbd_md_io_page_pool
== NULL
)
2142 drbd_request_mempool
= mempool_create(number
,
2143 mempool_alloc_slab
, mempool_free_slab
, drbd_request_cache
);
2144 if (drbd_request_mempool
== NULL
)
2147 drbd_ee_mempool
= mempool_create(number
,
2148 mempool_alloc_slab
, mempool_free_slab
, drbd_ee_cache
);
2149 if (drbd_ee_mempool
== NULL
)
2152 /* drbd's page pool */
2153 spin_lock_init(&drbd_pp_lock
);
2155 for (i
= 0; i
< number
; i
++) {
2156 page
= alloc_page(GFP_HIGHUSER
);
2159 set_page_private(page
, (unsigned long)drbd_pp_pool
);
2160 drbd_pp_pool
= page
;
2162 drbd_pp_vacant
= number
;
2167 drbd_destroy_mempools(); /* in case we allocated some */
2171 static int drbd_notify_sys(struct notifier_block
*this, unsigned long code
,
2174 /* just so we have it. you never know what interesting things we
2175 * might want to do here some day...
2181 static struct notifier_block drbd_notifier
= {
2182 .notifier_call
= drbd_notify_sys
,
2185 static void drbd_release_ee_lists(struct drbd_conf
*mdev
)
2189 rr
= drbd_release_ee(mdev
, &mdev
->active_ee
);
2191 dev_err(DEV
, "%d EEs in active list found!\n", rr
);
2193 rr
= drbd_release_ee(mdev
, &mdev
->sync_ee
);
2195 dev_err(DEV
, "%d EEs in sync list found!\n", rr
);
2197 rr
= drbd_release_ee(mdev
, &mdev
->read_ee
);
2199 dev_err(DEV
, "%d EEs in read list found!\n", rr
);
2201 rr
= drbd_release_ee(mdev
, &mdev
->done_ee
);
2203 dev_err(DEV
, "%d EEs in done list found!\n", rr
);
2205 rr
= drbd_release_ee(mdev
, &mdev
->net_ee
);
2207 dev_err(DEV
, "%d EEs in net list found!\n", rr
);
2210 /* caution. no locking. */
2211 void drbd_delete_device(unsigned int minor
)
2213 struct drbd_conf
*mdev
= minor_to_mdev(minor
);
2218 idr_remove(&mdev
->tconn
->volumes
, mdev
->vnr
);
2219 idr_remove(&minors
, minor
);
2222 /* paranoia asserts */
2223 D_ASSERT(mdev
->open_cnt
== 0);
2224 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2225 /* end paranoia asserts */
2227 del_gendisk(mdev
->vdisk
);
2229 /* cleanup stuff that may have been allocated during
2230 * device (re-)configuration or state changes */
2232 if (mdev
->this_bdev
)
2233 bdput(mdev
->this_bdev
);
2235 drbd_free_resources(mdev
);
2237 drbd_release_ee_lists(mdev
);
2239 lc_destroy(mdev
->act_log
);
2240 lc_destroy(mdev
->resync
);
2242 kfree(mdev
->p_uuid
);
2243 /* mdev->p_uuid = NULL; */
2245 /* cleanup the rest that has been
2246 * allocated from drbd_new_device
2247 * and actually free the mdev itself */
2248 drbd_free_mdev(mdev
);
2251 static void drbd_cleanup(void)
2254 struct drbd_conf
*mdev
;
2256 unregister_reboot_notifier(&drbd_notifier
);
2258 /* first remove proc,
2259 * drbdsetup uses it's presence to detect
2260 * whether DRBD is loaded.
2261 * If we would get stuck in proc removal,
2262 * but have netlink already deregistered,
2263 * some drbdsetup commands may wait forever
2267 remove_proc_entry("drbd", NULL
);
2269 drbd_genl_unregister();
2271 idr_for_each_entry(&minors
, mdev
, i
)
2272 drbd_delete_device(i
);
2273 drbd_destroy_mempools();
2274 unregister_blkdev(DRBD_MAJOR
, "drbd");
2276 idr_destroy(&minors
);
2278 printk(KERN_INFO
"drbd: module cleanup done.\n");
2282 * drbd_congested() - Callback for pdflush
2283 * @congested_data: User data
2284 * @bdi_bits: Bits pdflush is currently interested in
2286 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
2288 static int drbd_congested(void *congested_data
, int bdi_bits
)
2290 struct drbd_conf
*mdev
= congested_data
;
2291 struct request_queue
*q
;
2295 if (!may_inc_ap_bio(mdev
)) {
2296 /* DRBD has frozen IO */
2302 if (get_ldev(mdev
)) {
2303 q
= bdev_get_queue(mdev
->ldev
->backing_bdev
);
2304 r
= bdi_congested(&q
->backing_dev_info
, bdi_bits
);
2310 if (bdi_bits
& (1 << BDI_async_congested
) && test_bit(NET_CONGESTED
, &mdev
->tconn
->flags
)) {
2311 r
|= (1 << BDI_async_congested
);
2312 reason
= reason
== 'b' ? 'a' : 'n';
2316 mdev
->congestion_reason
= reason
;
2320 static void drbd_init_workqueue(struct drbd_work_queue
* wq
)
2322 sema_init(&wq
->s
, 0);
2323 spin_lock_init(&wq
->q_lock
);
2324 INIT_LIST_HEAD(&wq
->q
);
2327 struct drbd_tconn
*conn_by_name(const char *name
)
2329 struct drbd_tconn
*tconn
;
2331 if (!name
|| !name
[0])
2334 mutex_lock(&drbd_cfg_mutex
);
2335 list_for_each_entry(tconn
, &drbd_tconns
, all_tconn
) {
2336 if (!strcmp(tconn
->name
, name
))
2341 mutex_unlock(&drbd_cfg_mutex
);
2345 static int drbd_alloc_socket(struct drbd_socket
*socket
)
2347 socket
->rbuf
= (void *) __get_free_page(GFP_KERNEL
);
2350 socket
->sbuf
= (void *) __get_free_page(GFP_KERNEL
);
2356 static void drbd_free_socket(struct drbd_socket
*socket
)
2358 free_page((unsigned long) socket
->sbuf
);
2359 free_page((unsigned long) socket
->rbuf
);
2362 struct drbd_tconn
*drbd_new_tconn(const char *name
)
2364 struct drbd_tconn
*tconn
;
2366 tconn
= kzalloc(sizeof(struct drbd_tconn
), GFP_KERNEL
);
2370 tconn
->name
= kstrdup(name
, GFP_KERNEL
);
2374 if (drbd_alloc_socket(&tconn
->data
))
2376 if (drbd_alloc_socket(&tconn
->meta
))
2379 if (!zalloc_cpumask_var(&tconn
->cpu_mask
, GFP_KERNEL
))
2382 if (!tl_init(tconn
))
2385 tconn
->cstate
= C_STANDALONE
;
2386 mutex_init(&tconn
->cstate_mutex
);
2387 spin_lock_init(&tconn
->req_lock
);
2388 atomic_set(&tconn
->net_cnt
, 0);
2389 init_waitqueue_head(&tconn
->net_cnt_wait
);
2390 init_waitqueue_head(&tconn
->ping_wait
);
2391 idr_init(&tconn
->volumes
);
2393 drbd_init_workqueue(&tconn
->data
.work
);
2394 mutex_init(&tconn
->data
.mutex
);
2396 drbd_init_workqueue(&tconn
->meta
.work
);
2397 mutex_init(&tconn
->meta
.mutex
);
2399 drbd_thread_init(tconn
, &tconn
->receiver
, drbdd_init
, "receiver");
2400 drbd_thread_init(tconn
, &tconn
->worker
, drbd_worker
, "worker");
2401 drbd_thread_init(tconn
, &tconn
->asender
, drbd_asender
, "asender");
2403 tconn
->res_opts
= (struct res_opts
) {
2404 {}, 0, /* cpu_mask */
2405 DRBD_ON_NO_DATA_DEF
, /* on_no_data */
2408 mutex_lock(&drbd_cfg_mutex
);
2409 list_add_tail(&tconn
->all_tconn
, &drbd_tconns
);
2410 mutex_unlock(&drbd_cfg_mutex
);
2416 free_cpumask_var(tconn
->cpu_mask
);
2417 drbd_free_socket(&tconn
->meta
);
2418 drbd_free_socket(&tconn
->data
);
2425 void drbd_free_tconn(struct drbd_tconn
*tconn
)
2427 list_del(&tconn
->all_tconn
);
2428 idr_destroy(&tconn
->volumes
);
2430 free_cpumask_var(tconn
->cpu_mask
);
2431 drbd_free_socket(&tconn
->meta
);
2432 drbd_free_socket(&tconn
->data
);
2434 kfree(tconn
->int_dig_in
);
2435 kfree(tconn
->int_dig_vv
);
2439 enum drbd_ret_code
conn_new_minor(struct drbd_tconn
*tconn
, unsigned int minor
, int vnr
)
2441 struct drbd_conf
*mdev
;
2442 struct gendisk
*disk
;
2443 struct request_queue
*q
;
2445 int minor_got
= minor
;
2446 enum drbd_ret_code err
= ERR_NOMEM
;
2448 mdev
= minor_to_mdev(minor
);
2450 return ERR_MINOR_EXISTS
;
2452 /* GFP_KERNEL, we are outside of all write-out paths */
2453 mdev
= kzalloc(sizeof(struct drbd_conf
), GFP_KERNEL
);
2457 mdev
->tconn
= tconn
;
2458 mdev
->minor
= minor
;
2461 drbd_init_set_defaults(mdev
);
2463 q
= blk_alloc_queue(GFP_KERNEL
);
2467 q
->queuedata
= mdev
;
2469 disk
= alloc_disk(1);
2474 set_disk_ro(disk
, true);
2477 disk
->major
= DRBD_MAJOR
;
2478 disk
->first_minor
= minor
;
2479 disk
->fops
= &drbd_ops
;
2480 sprintf(disk
->disk_name
, "drbd%d", minor
);
2481 disk
->private_data
= mdev
;
2483 mdev
->this_bdev
= bdget(MKDEV(DRBD_MAJOR
, minor
));
2484 /* we have no partitions. we contain only ourselves. */
2485 mdev
->this_bdev
->bd_contains
= mdev
->this_bdev
;
2487 q
->backing_dev_info
.congested_fn
= drbd_congested
;
2488 q
->backing_dev_info
.congested_data
= mdev
;
2490 blk_queue_make_request(q
, drbd_make_request
);
2491 /* Setting the max_hw_sectors to an odd value of 8kibyte here
2492 This triggers a max_bio_size message upon first attach or connect */
2493 blk_queue_max_hw_sectors(q
, DRBD_MAX_BIO_SIZE_SAFE
>> 8);
2494 blk_queue_bounce_limit(q
, BLK_BOUNCE_ANY
);
2495 blk_queue_merge_bvec(q
, drbd_merge_bvec
);
2496 q
->queue_lock
= &mdev
->tconn
->req_lock
; /* needed since we use */
2498 mdev
->md_io_page
= alloc_page(GFP_KERNEL
);
2499 if (!mdev
->md_io_page
)
2500 goto out_no_io_page
;
2502 if (drbd_bm_init(mdev
))
2504 mdev
->read_requests
= RB_ROOT
;
2505 mdev
->write_requests
= RB_ROOT
;
2507 mdev
->current_epoch
= kzalloc(sizeof(struct drbd_epoch
), GFP_KERNEL
);
2508 if (!mdev
->current_epoch
)
2511 INIT_LIST_HEAD(&mdev
->current_epoch
->list
);
2514 if (!idr_pre_get(&minors
, GFP_KERNEL
))
2515 goto out_no_minor_idr
;
2516 if (idr_get_new_above(&minors
, mdev
, minor
, &minor_got
))
2517 goto out_no_minor_idr
;
2518 if (minor_got
!= minor
) {
2519 err
= ERR_MINOR_EXISTS
;
2520 drbd_msg_put_info("requested minor exists already");
2521 goto out_idr_remove_minor
;
2524 if (!idr_pre_get(&tconn
->volumes
, GFP_KERNEL
))
2525 goto out_idr_remove_minor
;
2526 if (idr_get_new_above(&tconn
->volumes
, mdev
, vnr
, &vnr_got
))
2527 goto out_idr_remove_minor
;
2528 if (vnr_got
!= vnr
) {
2529 err
= ERR_INVALID_REQUEST
;
2530 drbd_msg_put_info("requested volume exists already");
2531 goto out_idr_remove_vol
;
2535 /* inherit the connection state */
2536 mdev
->state
.conn
= tconn
->cstate
;
2537 if (mdev
->state
.conn
== C_WF_REPORT_PARAMS
)
2538 drbd_connected(vnr
, mdev
, tconn
);
2543 idr_remove(&tconn
->volumes
, vnr_got
);
2544 out_idr_remove_minor
:
2545 idr_remove(&minors
, minor_got
);
2548 kfree(mdev
->current_epoch
);
2550 drbd_bm_cleanup(mdev
);
2552 __free_page(mdev
->md_io_page
);
2556 blk_cleanup_queue(q
);
2562 /* counterpart of drbd_new_device.
2563 * last part of drbd_delete_device. */
2564 void drbd_free_mdev(struct drbd_conf
*mdev
)
2566 kfree(mdev
->current_epoch
);
2567 if (mdev
->bitmap
) /* should no longer be there. */
2568 drbd_bm_cleanup(mdev
);
2569 __free_page(mdev
->md_io_page
);
2570 put_disk(mdev
->vdisk
);
2571 blk_cleanup_queue(mdev
->rq_queue
);
2576 int __init
drbd_init(void)
2580 if (minor_count
< DRBD_MINOR_COUNT_MIN
|| minor_count
> DRBD_MINOR_COUNT_MAX
) {
2582 "drbd: invalid minor_count (%d)\n", minor_count
);
2590 err
= register_blkdev(DRBD_MAJOR
, "drbd");
2593 "drbd: unable to register block device major %d\n",
2598 err
= drbd_genl_register();
2600 printk(KERN_ERR
"drbd: unable to register generic netlink family\n");
2605 register_reboot_notifier(&drbd_notifier
);
2608 * allocate all necessary structs
2612 init_waitqueue_head(&drbd_pp_wait
);
2614 drbd_proc
= NULL
; /* play safe for drbd_cleanup */
2617 err
= drbd_create_mempools();
2621 drbd_proc
= proc_create_data("drbd", S_IFREG
| S_IRUGO
, NULL
, &drbd_proc_fops
, NULL
);
2623 printk(KERN_ERR
"drbd: unable to register proc file\n");
2627 rwlock_init(&global_state_lock
);
2628 INIT_LIST_HEAD(&drbd_tconns
);
2630 printk(KERN_INFO
"drbd: initialized. "
2631 "Version: " REL_VERSION
" (api:%d/proto:%d-%d)\n",
2632 API_VERSION
, PRO_VERSION_MIN
, PRO_VERSION_MAX
);
2633 printk(KERN_INFO
"drbd: %s\n", drbd_buildtag());
2634 printk(KERN_INFO
"drbd: registered as block device major %d\n",
2637 return 0; /* Success! */
2642 /* currently always the case */
2643 printk(KERN_ERR
"drbd: ran out of memory\n");
2645 printk(KERN_ERR
"drbd: initialization failure\n");
2649 void drbd_free_bc(struct drbd_backing_dev
*ldev
)
2654 blkdev_put(ldev
->backing_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2655 blkdev_put(ldev
->md_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2660 void drbd_free_sock(struct drbd_tconn
*tconn
)
2662 if (tconn
->data
.socket
) {
2663 mutex_lock(&tconn
->data
.mutex
);
2664 kernel_sock_shutdown(tconn
->data
.socket
, SHUT_RDWR
);
2665 sock_release(tconn
->data
.socket
);
2666 tconn
->data
.socket
= NULL
;
2667 mutex_unlock(&tconn
->data
.mutex
);
2669 if (tconn
->meta
.socket
) {
2670 mutex_lock(&tconn
->meta
.mutex
);
2671 kernel_sock_shutdown(tconn
->meta
.socket
, SHUT_RDWR
);
2672 sock_release(tconn
->meta
.socket
);
2673 tconn
->meta
.socket
= NULL
;
2674 mutex_unlock(&tconn
->meta
.mutex
);
2679 void drbd_free_resources(struct drbd_conf
*mdev
)
2681 crypto_free_hash(mdev
->tconn
->csums_tfm
);
2682 mdev
->tconn
->csums_tfm
= NULL
;
2683 crypto_free_hash(mdev
->tconn
->verify_tfm
);
2684 mdev
->tconn
->verify_tfm
= NULL
;
2685 crypto_free_hash(mdev
->tconn
->cram_hmac_tfm
);
2686 mdev
->tconn
->cram_hmac_tfm
= NULL
;
2687 crypto_free_hash(mdev
->tconn
->integrity_w_tfm
);
2688 mdev
->tconn
->integrity_w_tfm
= NULL
;
2689 crypto_free_hash(mdev
->tconn
->integrity_r_tfm
);
2690 mdev
->tconn
->integrity_r_tfm
= NULL
;
2692 drbd_free_sock(mdev
->tconn
);
2695 drbd_free_bc(mdev
->ldev
);
2696 mdev
->ldev
= NULL
;);
2699 /* meta data management */
2701 struct meta_data_on_disk
{
2702 u64 la_size
; /* last agreed size. */
2703 u64 uuid
[UI_SIZE
]; /* UUIDs. */
2706 u32 flags
; /* MDF */
2709 u32 al_offset
; /* offset to this block */
2710 u32 al_nr_extents
; /* important for restoring the AL */
2711 /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
2712 u32 bm_offset
; /* offset to the bitmap, from here */
2713 u32 bm_bytes_per_bit
; /* BM_BLOCK_SIZE */
2714 u32 la_peer_max_bio_size
; /* last peer max_bio_size */
2715 u32 reserved_u32
[3];
2720 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2721 * @mdev: DRBD device.
2723 void drbd_md_sync(struct drbd_conf
*mdev
)
2725 struct meta_data_on_disk
*buffer
;
2729 del_timer(&mdev
->md_sync_timer
);
2730 /* timer may be rearmed by drbd_md_mark_dirty() now. */
2731 if (!test_and_clear_bit(MD_DIRTY
, &mdev
->flags
))
2734 /* We use here D_FAILED and not D_ATTACHING because we try to write
2735 * metadata even if we detach due to a disk failure! */
2736 if (!get_ldev_if_state(mdev
, D_FAILED
))
2739 mutex_lock(&mdev
->md_io_mutex
);
2740 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2741 memset(buffer
, 0, 512);
2743 buffer
->la_size
= cpu_to_be64(drbd_get_capacity(mdev
->this_bdev
));
2744 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2745 buffer
->uuid
[i
] = cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]);
2746 buffer
->flags
= cpu_to_be32(mdev
->ldev
->md
.flags
);
2747 buffer
->magic
= cpu_to_be32(DRBD_MD_MAGIC
);
2749 buffer
->md_size_sect
= cpu_to_be32(mdev
->ldev
->md
.md_size_sect
);
2750 buffer
->al_offset
= cpu_to_be32(mdev
->ldev
->md
.al_offset
);
2751 buffer
->al_nr_extents
= cpu_to_be32(mdev
->act_log
->nr_elements
);
2752 buffer
->bm_bytes_per_bit
= cpu_to_be32(BM_BLOCK_SIZE
);
2753 buffer
->device_uuid
= cpu_to_be64(mdev
->ldev
->md
.device_uuid
);
2755 buffer
->bm_offset
= cpu_to_be32(mdev
->ldev
->md
.bm_offset
);
2756 buffer
->la_peer_max_bio_size
= cpu_to_be32(mdev
->peer_max_bio_size
);
2758 D_ASSERT(drbd_md_ss__(mdev
, mdev
->ldev
) == mdev
->ldev
->md
.md_offset
);
2759 sector
= mdev
->ldev
->md
.md_offset
;
2761 if (drbd_md_sync_page_io(mdev
, mdev
->ldev
, sector
, WRITE
)) {
2762 /* this was a try anyways ... */
2763 dev_err(DEV
, "meta data update failed!\n");
2764 drbd_chk_io_error(mdev
, 1, true);
2767 /* Update mdev->ldev->md.la_size_sect,
2768 * since we updated it on metadata. */
2769 mdev
->ldev
->md
.la_size_sect
= drbd_get_capacity(mdev
->this_bdev
);
2771 mutex_unlock(&mdev
->md_io_mutex
);
2776 * drbd_md_read() - Reads in the meta data super block
2777 * @mdev: DRBD device.
2778 * @bdev: Device from which the meta data should be read in.
2780 * Return 0 (NO_ERROR) on success, and an enum drbd_ret_code in case
2781 * something goes wrong. Currently only: ERR_IO_MD_DISK, ERR_MD_INVALID.
2783 int drbd_md_read(struct drbd_conf
*mdev
, struct drbd_backing_dev
*bdev
)
2785 struct meta_data_on_disk
*buffer
;
2786 int i
, rv
= NO_ERROR
;
2788 if (!get_ldev_if_state(mdev
, D_ATTACHING
))
2789 return ERR_IO_MD_DISK
;
2791 mutex_lock(&mdev
->md_io_mutex
);
2792 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2794 if (drbd_md_sync_page_io(mdev
, bdev
, bdev
->md
.md_offset
, READ
)) {
2795 /* NOTE: can't do normal error processing here as this is
2796 called BEFORE disk is attached */
2797 dev_err(DEV
, "Error while reading metadata.\n");
2798 rv
= ERR_IO_MD_DISK
;
2802 if (buffer
->magic
!= cpu_to_be32(DRBD_MD_MAGIC
)) {
2803 dev_err(DEV
, "Error while reading metadata, magic not found.\n");
2804 rv
= ERR_MD_INVALID
;
2807 if (be32_to_cpu(buffer
->al_offset
) != bdev
->md
.al_offset
) {
2808 dev_err(DEV
, "unexpected al_offset: %d (expected %d)\n",
2809 be32_to_cpu(buffer
->al_offset
), bdev
->md
.al_offset
);
2810 rv
= ERR_MD_INVALID
;
2813 if (be32_to_cpu(buffer
->bm_offset
) != bdev
->md
.bm_offset
) {
2814 dev_err(DEV
, "unexpected bm_offset: %d (expected %d)\n",
2815 be32_to_cpu(buffer
->bm_offset
), bdev
->md
.bm_offset
);
2816 rv
= ERR_MD_INVALID
;
2819 if (be32_to_cpu(buffer
->md_size_sect
) != bdev
->md
.md_size_sect
) {
2820 dev_err(DEV
, "unexpected md_size: %u (expected %u)\n",
2821 be32_to_cpu(buffer
->md_size_sect
), bdev
->md
.md_size_sect
);
2822 rv
= ERR_MD_INVALID
;
2826 if (be32_to_cpu(buffer
->bm_bytes_per_bit
) != BM_BLOCK_SIZE
) {
2827 dev_err(DEV
, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
2828 be32_to_cpu(buffer
->bm_bytes_per_bit
), BM_BLOCK_SIZE
);
2829 rv
= ERR_MD_INVALID
;
2833 bdev
->md
.la_size_sect
= be64_to_cpu(buffer
->la_size
);
2834 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2835 bdev
->md
.uuid
[i
] = be64_to_cpu(buffer
->uuid
[i
]);
2836 bdev
->md
.flags
= be32_to_cpu(buffer
->flags
);
2837 bdev
->dc
.al_extents
= be32_to_cpu(buffer
->al_nr_extents
);
2838 bdev
->md
.device_uuid
= be64_to_cpu(buffer
->device_uuid
);
2840 spin_lock_irq(&mdev
->tconn
->req_lock
);
2841 if (mdev
->state
.conn
< C_CONNECTED
) {
2843 peer
= be32_to_cpu(buffer
->la_peer_max_bio_size
);
2844 peer
= max_t(int, peer
, DRBD_MAX_BIO_SIZE_SAFE
);
2845 mdev
->peer_max_bio_size
= peer
;
2847 spin_unlock_irq(&mdev
->tconn
->req_lock
);
2849 if (bdev
->dc
.al_extents
< 7)
2850 bdev
->dc
.al_extents
= 127;
2853 mutex_unlock(&mdev
->md_io_mutex
);
2860 * drbd_md_mark_dirty() - Mark meta data super block as dirty
2861 * @mdev: DRBD device.
2863 * Call this function if you change anything that should be written to
2864 * the meta-data super block. This function sets MD_DIRTY, and starts a
2865 * timer that ensures that within five seconds you have to call drbd_md_sync().
2868 void drbd_md_mark_dirty_(struct drbd_conf
*mdev
, unsigned int line
, const char *func
)
2870 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
)) {
2871 mod_timer(&mdev
->md_sync_timer
, jiffies
+ HZ
);
2872 mdev
->last_md_mark_dirty
.line
= line
;
2873 mdev
->last_md_mark_dirty
.func
= func
;
2877 void drbd_md_mark_dirty(struct drbd_conf
*mdev
)
2879 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
))
2880 mod_timer(&mdev
->md_sync_timer
, jiffies
+ 5*HZ
);
2884 static void drbd_uuid_move_history(struct drbd_conf
*mdev
) __must_hold(local
)
2888 for (i
= UI_HISTORY_START
; i
< UI_HISTORY_END
; i
++)
2889 mdev
->ldev
->md
.uuid
[i
+1] = mdev
->ldev
->md
.uuid
[i
];
2892 void _drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2894 if (idx
== UI_CURRENT
) {
2895 if (mdev
->state
.role
== R_PRIMARY
)
2900 drbd_set_ed_uuid(mdev
, val
);
2903 mdev
->ldev
->md
.uuid
[idx
] = val
;
2904 drbd_md_mark_dirty(mdev
);
2908 void drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2910 if (mdev
->ldev
->md
.uuid
[idx
]) {
2911 drbd_uuid_move_history(mdev
);
2912 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[idx
];
2914 _drbd_uuid_set(mdev
, idx
, val
);
2918 * drbd_uuid_new_current() - Creates a new current UUID
2919 * @mdev: DRBD device.
2921 * Creates a new current UUID, and rotates the old current UUID into
2922 * the bitmap slot. Causes an incremental resync upon next connect.
2924 void drbd_uuid_new_current(struct drbd_conf
*mdev
) __must_hold(local
)
2927 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2930 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2932 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = mdev
->ldev
->md
.uuid
[UI_CURRENT
];
2934 get_random_bytes(&val
, sizeof(u64
));
2935 _drbd_uuid_set(mdev
, UI_CURRENT
, val
);
2936 drbd_print_uuids(mdev
, "new current UUID");
2937 /* get it to stable storage _now_ */
2941 void drbd_uuid_set_bm(struct drbd_conf
*mdev
, u64 val
) __must_hold(local
)
2943 if (mdev
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && val
== 0)
2947 drbd_uuid_move_history(mdev
);
2948 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2949 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = 0;
2951 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2953 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2955 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = val
& ~((u64
)1);
2957 drbd_md_mark_dirty(mdev
);
2961 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2962 * @mdev: DRBD device.
2964 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
2966 int drbd_bmio_set_n_write(struct drbd_conf
*mdev
)
2970 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2971 drbd_md_set_flag(mdev
, MDF_FULL_SYNC
);
2973 drbd_bm_set_all(mdev
);
2975 rv
= drbd_bm_write(mdev
);
2978 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
2989 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2990 * @mdev: DRBD device.
2992 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
2994 int drbd_bmio_clear_n_write(struct drbd_conf
*mdev
)
2998 drbd_resume_al(mdev
);
2999 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
3000 drbd_bm_clear_all(mdev
);
3001 rv
= drbd_bm_write(mdev
);
3008 static int w_bitmap_io(struct drbd_work
*w
, int unused
)
3010 struct bm_io_work
*work
= container_of(w
, struct bm_io_work
, w
);
3011 struct drbd_conf
*mdev
= w
->mdev
;
3014 D_ASSERT(atomic_read(&mdev
->ap_bio_cnt
) == 0);
3016 if (get_ldev(mdev
)) {
3017 drbd_bm_lock(mdev
, work
->why
, work
->flags
);
3018 rv
= work
->io_fn(mdev
);
3019 drbd_bm_unlock(mdev
);
3023 clear_bit_unlock(BITMAP_IO
, &mdev
->flags
);
3024 wake_up(&mdev
->misc_wait
);
3027 work
->done(mdev
, rv
);
3029 clear_bit(BITMAP_IO_QUEUED
, &mdev
->flags
);
3036 void drbd_ldev_destroy(struct drbd_conf
*mdev
)
3038 lc_destroy(mdev
->resync
);
3039 mdev
->resync
= NULL
;
3040 lc_destroy(mdev
->act_log
);
3041 mdev
->act_log
= NULL
;
3043 drbd_free_bc(mdev
->ldev
);
3044 mdev
->ldev
= NULL
;);
3046 clear_bit(GO_DISKLESS
, &mdev
->flags
);
3049 static int w_go_diskless(struct drbd_work
*w
, int unused
)
3051 struct drbd_conf
*mdev
= w
->mdev
;
3053 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3054 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
3055 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
3056 * the protected members anymore, though, so once put_ldev reaches zero
3057 * again, it will be safe to free them. */
3058 drbd_force_state(mdev
, NS(disk
, D_DISKLESS
));
3062 void drbd_go_diskless(struct drbd_conf
*mdev
)
3064 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3065 if (!test_and_set_bit(GO_DISKLESS
, &mdev
->flags
))
3066 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->go_diskless
);
3070 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3071 * @mdev: DRBD device.
3072 * @io_fn: IO callback to be called when bitmap IO is possible
3073 * @done: callback to be called after the bitmap IO was performed
3074 * @why: Descriptive text of the reason for doing the IO
3076 * While IO on the bitmap happens we freeze application IO thus we ensure
3077 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
3078 * called from worker context. It MUST NOT be used while a previous such
3079 * work is still pending!
3081 void drbd_queue_bitmap_io(struct drbd_conf
*mdev
,
3082 int (*io_fn
)(struct drbd_conf
*),
3083 void (*done
)(struct drbd_conf
*, int),
3084 char *why
, enum bm_flag flags
)
3086 D_ASSERT(current
== mdev
->tconn
->worker
.task
);
3088 D_ASSERT(!test_bit(BITMAP_IO_QUEUED
, &mdev
->flags
));
3089 D_ASSERT(!test_bit(BITMAP_IO
, &mdev
->flags
));
3090 D_ASSERT(list_empty(&mdev
->bm_io_work
.w
.list
));
3091 if (mdev
->bm_io_work
.why
)
3092 dev_err(DEV
, "FIXME going to queue '%s' but '%s' still pending?\n",
3093 why
, mdev
->bm_io_work
.why
);
3095 mdev
->bm_io_work
.io_fn
= io_fn
;
3096 mdev
->bm_io_work
.done
= done
;
3097 mdev
->bm_io_work
.why
= why
;
3098 mdev
->bm_io_work
.flags
= flags
;
3100 spin_lock_irq(&mdev
->tconn
->req_lock
);
3101 set_bit(BITMAP_IO
, &mdev
->flags
);
3102 if (atomic_read(&mdev
->ap_bio_cnt
) == 0) {
3103 if (!test_and_set_bit(BITMAP_IO_QUEUED
, &mdev
->flags
))
3104 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->bm_io_work
.w
);
3106 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3110 * drbd_bitmap_io() - Does an IO operation on the whole bitmap
3111 * @mdev: DRBD device.
3112 * @io_fn: IO callback to be called when bitmap IO is possible
3113 * @why: Descriptive text of the reason for doing the IO
3115 * freezes application IO while that the actual IO operations runs. This
3116 * functions MAY NOT be called from worker context.
3118 int drbd_bitmap_io(struct drbd_conf
*mdev
, int (*io_fn
)(struct drbd_conf
*),
3119 char *why
, enum bm_flag flags
)
3123 D_ASSERT(current
!= mdev
->tconn
->worker
.task
);
3125 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3126 drbd_suspend_io(mdev
);
3128 drbd_bm_lock(mdev
, why
, flags
);
3130 drbd_bm_unlock(mdev
);
3132 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3133 drbd_resume_io(mdev
);
3138 void drbd_md_set_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3140 if ((mdev
->ldev
->md
.flags
& flag
) != flag
) {
3141 drbd_md_mark_dirty(mdev
);
3142 mdev
->ldev
->md
.flags
|= flag
;
3146 void drbd_md_clear_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3148 if ((mdev
->ldev
->md
.flags
& flag
) != 0) {
3149 drbd_md_mark_dirty(mdev
);
3150 mdev
->ldev
->md
.flags
&= ~flag
;
3153 int drbd_md_test_flag(struct drbd_backing_dev
*bdev
, int flag
)
3155 return (bdev
->md
.flags
& flag
) != 0;
3158 static void md_sync_timer_fn(unsigned long data
)
3160 struct drbd_conf
*mdev
= (struct drbd_conf
*) data
;
3162 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &mdev
->md_sync_work
);
3165 static int w_md_sync(struct drbd_work
*w
, int unused
)
3167 struct drbd_conf
*mdev
= w
->mdev
;
3169 dev_warn(DEV
, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3171 dev_warn(DEV
, "last md_mark_dirty: %s:%u\n",
3172 mdev
->last_md_mark_dirty
.func
, mdev
->last_md_mark_dirty
.line
);
3178 const char *cmdname(enum drbd_packet cmd
)
3180 /* THINK may need to become several global tables
3181 * when we want to support more than
3182 * one PRO_VERSION */
3183 static const char *cmdnames
[] = {
3185 [P_DATA_REPLY
] = "DataReply",
3186 [P_RS_DATA_REPLY
] = "RSDataReply",
3187 [P_BARRIER
] = "Barrier",
3188 [P_BITMAP
] = "ReportBitMap",
3189 [P_BECOME_SYNC_TARGET
] = "BecomeSyncTarget",
3190 [P_BECOME_SYNC_SOURCE
] = "BecomeSyncSource",
3191 [P_UNPLUG_REMOTE
] = "UnplugRemote",
3192 [P_DATA_REQUEST
] = "DataRequest",
3193 [P_RS_DATA_REQUEST
] = "RSDataRequest",
3194 [P_SYNC_PARAM
] = "SyncParam",
3195 [P_SYNC_PARAM89
] = "SyncParam89",
3196 [P_PROTOCOL
] = "ReportProtocol",
3197 [P_UUIDS
] = "ReportUUIDs",
3198 [P_SIZES
] = "ReportSizes",
3199 [P_STATE
] = "ReportState",
3200 [P_SYNC_UUID
] = "ReportSyncUUID",
3201 [P_AUTH_CHALLENGE
] = "AuthChallenge",
3202 [P_AUTH_RESPONSE
] = "AuthResponse",
3204 [P_PING_ACK
] = "PingAck",
3205 [P_RECV_ACK
] = "RecvAck",
3206 [P_WRITE_ACK
] = "WriteAck",
3207 [P_RS_WRITE_ACK
] = "RSWriteAck",
3208 [P_DISCARD_WRITE
] = "DiscardWrite",
3209 [P_NEG_ACK
] = "NegAck",
3210 [P_NEG_DREPLY
] = "NegDReply",
3211 [P_NEG_RS_DREPLY
] = "NegRSDReply",
3212 [P_BARRIER_ACK
] = "BarrierAck",
3213 [P_STATE_CHG_REQ
] = "StateChgRequest",
3214 [P_STATE_CHG_REPLY
] = "StateChgReply",
3215 [P_OV_REQUEST
] = "OVRequest",
3216 [P_OV_REPLY
] = "OVReply",
3217 [P_OV_RESULT
] = "OVResult",
3218 [P_CSUM_RS_REQUEST
] = "CsumRSRequest",
3219 [P_RS_IS_IN_SYNC
] = "CsumRSIsInSync",
3220 [P_COMPRESSED_BITMAP
] = "CBitmap",
3221 [P_DELAY_PROBE
] = "DelayProbe",
3222 [P_OUT_OF_SYNC
] = "OutOfSync",
3223 [P_RETRY_WRITE
] = "RetryWrite",
3226 if (cmd
== P_INITIAL_META
)
3227 return "InitialMeta";
3228 if (cmd
== P_INITIAL_DATA
)
3229 return "InitialData";
3230 if (cmd
== P_CONNECTION_FEATURES
)
3231 return "ConnectionFeatures";
3232 if (cmd
>= ARRAY_SIZE(cmdnames
))
3234 return cmdnames
[cmd
];
3238 * drbd_wait_misc - wait for a request to make progress
3239 * @mdev: device associated with the request
3240 * @i: the struct drbd_interval embedded in struct drbd_request or
3241 * struct drbd_peer_request
3243 int drbd_wait_misc(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
3245 struct net_conf
*net_conf
= mdev
->tconn
->net_conf
;
3251 timeout
= MAX_SCHEDULE_TIMEOUT
;
3252 if (net_conf
->ko_count
)
3253 timeout
= net_conf
->timeout
* HZ
/ 10 * net_conf
->ko_count
;
3255 /* Indicate to wake up mdev->misc_wait on progress. */
3257 prepare_to_wait(&mdev
->misc_wait
, &wait
, TASK_INTERRUPTIBLE
);
3258 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3259 timeout
= schedule_timeout(timeout
);
3260 finish_wait(&mdev
->misc_wait
, &wait
);
3261 spin_lock_irq(&mdev
->tconn
->req_lock
);
3262 if (!timeout
|| mdev
->state
.conn
< C_CONNECTED
)
3264 if (signal_pending(current
))
3265 return -ERESTARTSYS
;
3269 #ifdef CONFIG_DRBD_FAULT_INJECTION
3270 /* Fault insertion support including random number generator shamelessly
3271 * stolen from kernel/rcutorture.c */
3272 struct fault_random_state
{
3273 unsigned long state
;
3274 unsigned long count
;
3277 #define FAULT_RANDOM_MULT 39916801 /* prime */
3278 #define FAULT_RANDOM_ADD 479001701 /* prime */
3279 #define FAULT_RANDOM_REFRESH 10000
3282 * Crude but fast random-number generator. Uses a linear congruential
3283 * generator, with occasional help from get_random_bytes().
3285 static unsigned long
3286 _drbd_fault_random(struct fault_random_state
*rsp
)
3290 if (!rsp
->count
--) {
3291 get_random_bytes(&refresh
, sizeof(refresh
));
3292 rsp
->state
+= refresh
;
3293 rsp
->count
= FAULT_RANDOM_REFRESH
;
3295 rsp
->state
= rsp
->state
* FAULT_RANDOM_MULT
+ FAULT_RANDOM_ADD
;
3296 return swahw32(rsp
->state
);
3300 _drbd_fault_str(unsigned int type
) {
3301 static char *_faults
[] = {
3302 [DRBD_FAULT_MD_WR
] = "Meta-data write",
3303 [DRBD_FAULT_MD_RD
] = "Meta-data read",
3304 [DRBD_FAULT_RS_WR
] = "Resync write",
3305 [DRBD_FAULT_RS_RD
] = "Resync read",
3306 [DRBD_FAULT_DT_WR
] = "Data write",
3307 [DRBD_FAULT_DT_RD
] = "Data read",
3308 [DRBD_FAULT_DT_RA
] = "Data read ahead",
3309 [DRBD_FAULT_BM_ALLOC
] = "BM allocation",
3310 [DRBD_FAULT_AL_EE
] = "EE allocation",
3311 [DRBD_FAULT_RECEIVE
] = "receive data corruption",
3314 return (type
< DRBD_FAULT_MAX
) ? _faults
[type
] : "**Unknown**";
3318 _drbd_insert_fault(struct drbd_conf
*mdev
, unsigned int type
)
3320 static struct fault_random_state rrs
= {0, 0};
3322 unsigned int ret
= (
3324 ((1 << mdev_to_minor(mdev
)) & fault_devs
) != 0) &&
3325 (((_drbd_fault_random(&rrs
) % 100) + 1) <= fault_rate
));
3330 if (__ratelimit(&drbd_ratelimit_state
))
3331 dev_warn(DEV
, "***Simulating %s failure\n",
3332 _drbd_fault_str(type
));
3339 const char *drbd_buildtag(void)
3341 /* DRBD built from external sources has here a reference to the
3342 git hash of the source code. */
3344 static char buildtag
[38] = "\0uilt-in";
3346 if (buildtag
[0] == 0) {
3347 #ifdef CONFIG_MODULES
3348 if (THIS_MODULE
!= NULL
)
3349 sprintf(buildtag
, "srcversion: %-24s", THIS_MODULE
->srcversion
);
3358 module_init(drbd_init
)
3359 module_exit(drbd_cleanup
)
3361 EXPORT_SYMBOL(drbd_conn_str
);
3362 EXPORT_SYMBOL(drbd_role_str
);
3363 EXPORT_SYMBOL(drbd_disk_str
);
3364 EXPORT_SYMBOL(drbd_set_st_err_str
);