4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
29 #include <linux/module.h>
30 #include <linux/drbd.h>
31 #include <asm/uaccess.h>
32 #include <asm/types.h>
34 #include <linux/ctype.h>
35 #include <linux/mutex.h>
37 #include <linux/file.h>
38 #include <linux/proc_fs.h>
39 #include <linux/init.h>
41 #include <linux/memcontrol.h>
42 #include <linux/mm_inline.h>
43 #include <linux/slab.h>
44 #include <linux/random.h>
45 #include <linux/reboot.h>
46 #include <linux/notifier.h>
47 #include <linux/kthread.h>
49 #define __KERNEL_SYSCALLS__
50 #include <linux/unistd.h>
51 #include <linux/vmalloc.h>
53 #include <linux/drbd_limits.h>
55 #include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
59 static DEFINE_MUTEX(drbd_main_mutex
);
60 int drbdd_init(struct drbd_thread
*);
61 int drbd_worker(struct drbd_thread
*);
62 int drbd_asender(struct drbd_thread
*);
65 static int drbd_open(struct block_device
*bdev
, fmode_t mode
);
66 static int drbd_release(struct gendisk
*gd
, fmode_t mode
);
67 static int w_md_sync(struct drbd_work
*w
, int unused
);
68 static void md_sync_timer_fn(unsigned long data
);
69 static int w_bitmap_io(struct drbd_work
*w
, int unused
);
70 static int w_go_diskless(struct drbd_work
*w
, int unused
);
72 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
73 "Lars Ellenberg <lars@linbit.com>");
74 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION
);
75 MODULE_VERSION(REL_VERSION
);
76 MODULE_LICENSE("GPL");
77 MODULE_PARM_DESC(minor_count
, "Approximate number of drbd devices ("
78 __stringify(DRBD_MINOR_COUNT_MIN
) "-" __stringify(DRBD_MINOR_COUNT_MAX
) ")");
79 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR
);
81 #include <linux/moduleparam.h>
82 /* allow_open_on_secondary */
83 MODULE_PARM_DESC(allow_oos
, "DONT USE!");
84 /* thanks to these macros, if compiled into the kernel (not-module),
85 * this becomes the boot parameter drbd.minor_count */
86 module_param(minor_count
, uint
, 0444);
87 module_param(disable_sendpage
, bool, 0644);
88 module_param(allow_oos
, bool, 0);
89 module_param(proc_details
, int, 0644);
91 #ifdef CONFIG_DRBD_FAULT_INJECTION
94 static int fault_count
;
96 /* bitmap of enabled faults */
97 module_param(enable_faults
, int, 0664);
98 /* fault rate % value - applies to all enabled faults */
99 module_param(fault_rate
, int, 0664);
100 /* count of faults inserted */
101 module_param(fault_count
, int, 0664);
102 /* bitmap of devices to insert faults on */
103 module_param(fault_devs
, int, 0644);
106 /* module parameter, defined */
107 unsigned int minor_count
= DRBD_MINOR_COUNT_DEF
;
108 int disable_sendpage
;
110 int proc_details
; /* Detail level in proc drbd*/
112 /* Module parameter for setting the user mode helper program
113 * to run. Default is /sbin/drbdadm */
114 char usermode_helper
[80] = "/sbin/drbdadm";
116 module_param_string(usermode_helper
, usermode_helper
, sizeof(usermode_helper
), 0644);
118 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
119 * as member "struct gendisk *vdisk;"
122 struct list_head drbd_tconns
; /* list of struct drbd_tconn */
123 DEFINE_MUTEX(drbd_cfg_mutex
);
125 struct kmem_cache
*drbd_request_cache
;
126 struct kmem_cache
*drbd_ee_cache
; /* peer requests */
127 struct kmem_cache
*drbd_bm_ext_cache
; /* bitmap extents */
128 struct kmem_cache
*drbd_al_ext_cache
; /* activity log extents */
129 mempool_t
*drbd_request_mempool
;
130 mempool_t
*drbd_ee_mempool
;
131 mempool_t
*drbd_md_io_page_pool
;
132 struct bio_set
*drbd_md_io_bio_set
;
134 /* I do not use a standard mempool, because:
135 1) I want to hand out the pre-allocated objects first.
136 2) I want to be able to interrupt sleeping allocation with a signal.
137 Note: This is a single linked list, the next pointer is the private
138 member of struct page.
140 struct page
*drbd_pp_pool
;
141 spinlock_t drbd_pp_lock
;
143 wait_queue_head_t drbd_pp_wait
;
145 DEFINE_RATELIMIT_STATE(drbd_ratelimit_state
, 5 * HZ
, 5);
147 static const struct block_device_operations drbd_ops
= {
148 .owner
= THIS_MODULE
,
150 .release
= drbd_release
,
153 static void bio_destructor_drbd(struct bio
*bio
)
155 bio_free(bio
, drbd_md_io_bio_set
);
158 struct bio
*bio_alloc_drbd(gfp_t gfp_mask
)
162 if (!drbd_md_io_bio_set
)
163 return bio_alloc(gfp_mask
, 1);
165 bio
= bio_alloc_bioset(gfp_mask
, 1, drbd_md_io_bio_set
);
168 bio
->bi_destructor
= bio_destructor_drbd
;
173 /* When checking with sparse, and this is an inline function, sparse will
174 give tons of false positives. When this is a real functions sparse works.
176 int _get_ldev_if_state(struct drbd_conf
*mdev
, enum drbd_disk_state mins
)
180 atomic_inc(&mdev
->local_cnt
);
181 io_allowed
= (mdev
->state
.disk
>= mins
);
183 if (atomic_dec_and_test(&mdev
->local_cnt
))
184 wake_up(&mdev
->misc_wait
);
192 * DOC: The transfer log
194 * The transfer log is a single linked list of &struct drbd_tl_epoch objects.
195 * mdev->tconn->newest_tle points to the head, mdev->tconn->oldest_tle points to the tail
196 * of the list. There is always at least one &struct drbd_tl_epoch object.
198 * Each &struct drbd_tl_epoch has a circular double linked list of requests
201 static int tl_init(struct drbd_tconn
*tconn
)
203 struct drbd_tl_epoch
*b
;
205 /* during device minor initialization, we may well use GFP_KERNEL */
206 b
= kmalloc(sizeof(struct drbd_tl_epoch
), GFP_KERNEL
);
209 INIT_LIST_HEAD(&b
->requests
);
210 INIT_LIST_HEAD(&b
->w
.list
);
214 b
->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
216 tconn
->oldest_tle
= b
;
217 tconn
->newest_tle
= b
;
218 INIT_LIST_HEAD(&tconn
->out_of_sequence_requests
);
223 static void tl_cleanup(struct drbd_tconn
*tconn
)
225 if (tconn
->oldest_tle
!= tconn
->newest_tle
)
226 conn_err(tconn
, "ASSERT FAILED: oldest_tle == newest_tle\n");
227 if (!list_empty(&tconn
->out_of_sequence_requests
))
228 conn_err(tconn
, "ASSERT FAILED: list_empty(out_of_sequence_requests)\n");
229 kfree(tconn
->oldest_tle
);
230 tconn
->oldest_tle
= NULL
;
231 kfree(tconn
->unused_spare_tle
);
232 tconn
->unused_spare_tle
= NULL
;
236 * _tl_add_barrier() - Adds a barrier to the transfer log
237 * @mdev: DRBD device.
238 * @new: Barrier to be added before the current head of the TL.
240 * The caller must hold the req_lock.
242 void _tl_add_barrier(struct drbd_tconn
*tconn
, struct drbd_tl_epoch
*new)
244 struct drbd_tl_epoch
*newest_before
;
246 INIT_LIST_HEAD(&new->requests
);
247 INIT_LIST_HEAD(&new->w
.list
);
248 new->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
252 newest_before
= tconn
->newest_tle
;
253 /* never send a barrier number == 0, because that is special-cased
254 * when using TCQ for our write ordering code */
255 new->br_number
= (newest_before
->br_number
+1) ?: 1;
256 if (tconn
->newest_tle
!= new) {
257 tconn
->newest_tle
->next
= new;
258 tconn
->newest_tle
= new;
263 * tl_release() - Free or recycle the oldest &struct drbd_tl_epoch object of the TL
264 * @mdev: DRBD device.
265 * @barrier_nr: Expected identifier of the DRBD write barrier packet.
266 * @set_size: Expected number of requests before that barrier.
268 * In case the passed barrier_nr or set_size does not match the oldest
269 * &struct drbd_tl_epoch objects this function will cause a termination
272 void tl_release(struct drbd_tconn
*tconn
, unsigned int barrier_nr
,
273 unsigned int set_size
)
275 struct drbd_conf
*mdev
;
276 struct drbd_tl_epoch
*b
, *nob
; /* next old barrier */
277 struct list_head
*le
, *tle
;
278 struct drbd_request
*r
;
280 spin_lock_irq(&tconn
->req_lock
);
282 b
= tconn
->oldest_tle
;
284 /* first some paranoia code */
286 conn_err(tconn
, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
290 if (b
->br_number
!= barrier_nr
) {
291 conn_err(tconn
, "BAD! BarrierAck #%u received, expected #%u!\n",
292 barrier_nr
, b
->br_number
);
295 if (b
->n_writes
!= set_size
) {
296 conn_err(tconn
, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
297 barrier_nr
, set_size
, b
->n_writes
);
301 /* Clean up list of requests processed during current epoch */
302 list_for_each_safe(le
, tle
, &b
->requests
) {
303 r
= list_entry(le
, struct drbd_request
, tl_requests
);
304 _req_mod(r
, BARRIER_ACKED
);
306 /* There could be requests on the list waiting for completion
307 of the write to the local disk. To avoid corruptions of
308 slab's data structures we have to remove the lists head.
310 Also there could have been a barrier ack out of sequence, overtaking
311 the write acks - which would be a bug and violating write ordering.
312 To not deadlock in case we lose connection while such requests are
313 still pending, we need some way to find them for the
314 _req_mode(CONNECTION_LOST_WHILE_PENDING).
316 These have been list_move'd to the out_of_sequence_requests list in
317 _req_mod(, BARRIER_ACKED) above.
319 list_del_init(&b
->requests
);
323 if (test_and_clear_bit(CREATE_BARRIER
, &mdev
->flags
)) {
324 _tl_add_barrier(tconn
, b
);
326 tconn
->oldest_tle
= nob
;
327 /* if nob == NULL b was the only barrier, and becomes the new
328 barrier. Therefore tconn->oldest_tle points already to b */
330 D_ASSERT(nob
!= NULL
);
331 tconn
->oldest_tle
= nob
;
335 spin_unlock_irq(&tconn
->req_lock
);
336 dec_ap_pending(mdev
);
341 spin_unlock_irq(&tconn
->req_lock
);
342 conn_request_state(tconn
, NS(conn
, C_PROTOCOL_ERROR
), CS_HARD
);
347 * _tl_restart() - Walks the transfer log, and applies an action to all requests
348 * @mdev: DRBD device.
349 * @what: The action/event to perform with all request objects
351 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
352 * RESTART_FROZEN_DISK_IO.
354 void _tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
356 struct drbd_tl_epoch
*b
, *tmp
, **pn
;
357 struct list_head
*le
, *tle
, carry_reads
;
358 struct drbd_request
*req
;
359 int rv
, n_writes
, n_reads
;
361 b
= tconn
->oldest_tle
;
362 pn
= &tconn
->oldest_tle
;
366 INIT_LIST_HEAD(&carry_reads
);
367 list_for_each_safe(le
, tle
, &b
->requests
) {
368 req
= list_entry(le
, struct drbd_request
, tl_requests
);
369 rv
= _req_mod(req
, what
);
371 n_writes
+= (rv
& MR_WRITE
) >> MR_WRITE_SHIFT
;
372 n_reads
+= (rv
& MR_READ
) >> MR_READ_SHIFT
;
377 if (what
== RESEND
) {
378 b
->n_writes
= n_writes
;
379 if (b
->w
.cb
== NULL
) {
380 b
->w
.cb
= w_send_barrier
;
381 inc_ap_pending(b
->w
.mdev
);
382 set_bit(CREATE_BARRIER
, &b
->w
.mdev
->flags
);
385 drbd_queue_work(&tconn
->data
.work
, &b
->w
);
390 list_add(&carry_reads
, &b
->requests
);
391 /* there could still be requests on that ring list,
392 * in case local io is still pending */
393 list_del(&b
->requests
);
395 /* dec_ap_pending corresponding to queue_barrier.
396 * the newest barrier may not have been queued yet,
397 * in which case w.cb is still NULL. */
399 dec_ap_pending(b
->w
.mdev
);
401 if (b
== tconn
->newest_tle
) {
402 /* recycle, but reinit! */
404 conn_err(tconn
, "ASSERT FAILED tmp == NULL");
405 INIT_LIST_HEAD(&b
->requests
);
406 list_splice(&carry_reads
, &b
->requests
);
407 INIT_LIST_HEAD(&b
->w
.list
);
409 b
->br_number
= net_random();
419 list_splice(&carry_reads
, &b
->requests
);
425 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
426 * @mdev: DRBD device.
428 * This is called after the connection to the peer was lost. The storage covered
429 * by the requests on the transfer gets marked as our of sync. Called from the
430 * receiver thread and the worker thread.
432 void tl_clear(struct drbd_tconn
*tconn
)
434 struct drbd_conf
*mdev
;
435 struct list_head
*le
, *tle
;
436 struct drbd_request
*r
;
439 spin_lock_irq(&tconn
->req_lock
);
441 _tl_restart(tconn
, CONNECTION_LOST_WHILE_PENDING
);
443 /* we expect this list to be empty. */
444 if (!list_empty(&tconn
->out_of_sequence_requests
))
445 conn_err(tconn
, "ASSERT FAILED list_empty(&out_of_sequence_requests)\n");
447 /* but just in case, clean it up anyways! */
448 list_for_each_safe(le
, tle
, &tconn
->out_of_sequence_requests
) {
449 r
= list_entry(le
, struct drbd_request
, tl_requests
);
450 /* It would be nice to complete outside of spinlock.
451 * But this is easier for now. */
452 _req_mod(r
, CONNECTION_LOST_WHILE_PENDING
);
455 /* ensure bit indicating barrier is required is clear */
456 idr_for_each_entry(&tconn
->volumes
, mdev
, vnr
)
457 clear_bit(CREATE_BARRIER
, &mdev
->flags
);
459 spin_unlock_irq(&tconn
->req_lock
);
462 void tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
464 spin_lock_irq(&tconn
->req_lock
);
465 _tl_restart(tconn
, what
);
466 spin_unlock_irq(&tconn
->req_lock
);
469 static int drbd_thread_setup(void *arg
)
471 struct drbd_thread
*thi
= (struct drbd_thread
*) arg
;
472 struct drbd_tconn
*tconn
= thi
->tconn
;
476 snprintf(current
->comm
, sizeof(current
->comm
), "drbd_%c_%s",
477 thi
->name
[0], thi
->tconn
->name
);
480 retval
= thi
->function(thi
);
482 spin_lock_irqsave(&thi
->t_lock
, flags
);
484 /* if the receiver has been "EXITING", the last thing it did
485 * was set the conn state to "StandAlone",
486 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
487 * and receiver thread will be "started".
488 * drbd_thread_start needs to set "RESTARTING" in that case.
489 * t_state check and assignment needs to be within the same spinlock,
490 * so either thread_start sees EXITING, and can remap to RESTARTING,
491 * or thread_start see NONE, and can proceed as normal.
494 if (thi
->t_state
== RESTARTING
) {
495 conn_info(tconn
, "Restarting %s thread\n", thi
->name
);
496 thi
->t_state
= RUNNING
;
497 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
504 complete(&thi
->stop
);
505 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
507 conn_info(tconn
, "Terminating %s\n", current
->comm
);
509 /* Release mod reference taken when thread was started */
510 module_put(THIS_MODULE
);
514 static void drbd_thread_init(struct drbd_tconn
*tconn
, struct drbd_thread
*thi
,
515 int (*func
) (struct drbd_thread
*), char *name
)
517 spin_lock_init(&thi
->t_lock
);
520 thi
->function
= func
;
522 strncpy(thi
->name
, name
, ARRAY_SIZE(thi
->name
));
525 int drbd_thread_start(struct drbd_thread
*thi
)
527 struct drbd_tconn
*tconn
= thi
->tconn
;
528 struct task_struct
*nt
;
531 /* is used from state engine doing drbd_thread_stop_nowait,
532 * while holding the req lock irqsave */
533 spin_lock_irqsave(&thi
->t_lock
, flags
);
535 switch (thi
->t_state
) {
537 conn_info(tconn
, "Starting %s thread (from %s [%d])\n",
538 thi
->name
, current
->comm
, current
->pid
);
540 /* Get ref on module for thread - this is released when thread exits */
541 if (!try_module_get(THIS_MODULE
)) {
542 conn_err(tconn
, "Failed to get module reference in drbd_thread_start\n");
543 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
547 init_completion(&thi
->stop
);
548 thi
->reset_cpu_mask
= 1;
549 thi
->t_state
= RUNNING
;
550 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
551 flush_signals(current
); /* otherw. may get -ERESTARTNOINTR */
553 nt
= kthread_create(drbd_thread_setup
, (void *) thi
,
554 "drbd_%c_%s", thi
->name
[0], thi
->tconn
->name
);
557 conn_err(tconn
, "Couldn't start thread\n");
559 module_put(THIS_MODULE
);
562 spin_lock_irqsave(&thi
->t_lock
, flags
);
564 thi
->t_state
= RUNNING
;
565 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
569 thi
->t_state
= RESTARTING
;
570 conn_info(tconn
, "Restarting %s thread (from %s [%d])\n",
571 thi
->name
, current
->comm
, current
->pid
);
576 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
584 void _drbd_thread_stop(struct drbd_thread
*thi
, int restart
, int wait
)
588 enum drbd_thread_state ns
= restart
? RESTARTING
: EXITING
;
590 /* may be called from state engine, holding the req lock irqsave */
591 spin_lock_irqsave(&thi
->t_lock
, flags
);
593 if (thi
->t_state
== NONE
) {
594 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
596 drbd_thread_start(thi
);
600 if (thi
->t_state
!= ns
) {
601 if (thi
->task
== NULL
) {
602 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
608 init_completion(&thi
->stop
);
609 if (thi
->task
!= current
)
610 force_sig(DRBD_SIGKILL
, thi
->task
);
613 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
616 wait_for_completion(&thi
->stop
);
619 static struct drbd_thread
*drbd_task_to_thread(struct drbd_tconn
*tconn
, struct task_struct
*task
)
621 struct drbd_thread
*thi
=
622 task
== tconn
->receiver
.task
? &tconn
->receiver
:
623 task
== tconn
->asender
.task
? &tconn
->asender
:
624 task
== tconn
->worker
.task
? &tconn
->worker
: NULL
;
629 char *drbd_task_to_thread_name(struct drbd_tconn
*tconn
, struct task_struct
*task
)
631 struct drbd_thread
*thi
= drbd_task_to_thread(tconn
, task
);
632 return thi
? thi
->name
: task
->comm
;
635 int conn_lowest_minor(struct drbd_tconn
*tconn
)
638 struct drbd_conf
*mdev
;
640 mdev
= idr_get_next(&tconn
->volumes
, &vnr
);
643 return mdev_to_minor(mdev
);
648 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
649 * @mdev: DRBD device.
651 * Forces all threads of a device onto the same CPU. This is beneficial for
652 * DRBD's performance. May be overwritten by user's configuration.
654 void drbd_calc_cpu_mask(struct drbd_tconn
*tconn
)
659 if (cpumask_weight(tconn
->cpu_mask
))
662 ord
= conn_lowest_minor(tconn
) % cpumask_weight(cpu_online_mask
);
663 for_each_online_cpu(cpu
) {
665 cpumask_set_cpu(cpu
, tconn
->cpu_mask
);
669 /* should not be reached */
670 cpumask_setall(tconn
->cpu_mask
);
674 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
675 * @mdev: DRBD device.
676 * @thi: drbd_thread object
678 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
681 void drbd_thread_current_set_cpu(struct drbd_thread
*thi
)
683 struct task_struct
*p
= current
;
685 if (!thi
->reset_cpu_mask
)
687 thi
->reset_cpu_mask
= 0;
688 set_cpus_allowed_ptr(p
, thi
->tconn
->cpu_mask
);
693 * drbd_header_size - size of a packet header
695 * The header size is a multiple of 8, so any payload following the header is
696 * word aligned on 64-bit architectures. (The bitmap send and receive code
699 unsigned int drbd_header_size(struct drbd_tconn
*tconn
)
701 BUILD_BUG_ON(sizeof(struct p_header80
) != sizeof(struct p_header95
));
702 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80
), 8));
703 return sizeof(struct p_header80
);
706 static void prepare_header80(struct p_header80
*h
, enum drbd_packet cmd
, int size
)
708 h
->magic
= cpu_to_be32(DRBD_MAGIC
);
709 h
->command
= cpu_to_be16(cmd
);
710 h
->length
= cpu_to_be16(size
);
713 static void prepare_header95(struct p_header95
*h
, enum drbd_packet cmd
, int size
)
715 h
->magic
= cpu_to_be16(DRBD_MAGIC_BIG
);
716 h
->command
= cpu_to_be16(cmd
);
717 h
->length
= cpu_to_be32(size
);
720 static void prepare_header(struct drbd_tconn
*tconn
, int vnr
, struct p_header
*h
,
721 enum drbd_packet cmd
, int size
)
723 if (tconn
->agreed_pro_version
>= 95)
724 prepare_header95(&h
->h95
, cmd
, size
);
726 prepare_header80(&h
->h80
, cmd
, size
);
729 void *conn_prepare_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
)
731 mutex_lock(&sock
->mutex
);
733 mutex_unlock(&sock
->mutex
);
739 void *drbd_prepare_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
)
741 return conn_prepare_command(mdev
->tconn
, sock
);
744 static int __send_command(struct drbd_tconn
*tconn
, int vnr
,
745 struct drbd_socket
*sock
, enum drbd_packet cmd
,
746 unsigned int header_size
, void *data
,
753 * Called with @data == NULL and the size of the data blocks in @size
754 * for commands that send data blocks. For those commands, omit the
755 * MSG_MORE flag: this will increase the likelihood that data blocks
756 * which are page aligned on the sender will end up page aligned on the
759 msg_flags
= data
? MSG_MORE
: 0;
761 prepare_header(tconn
, vnr
, sock
->sbuf
, cmd
,
762 header_size
- sizeof(struct p_header
) + size
);
763 err
= drbd_send_all(tconn
, sock
->socket
, sock
->sbuf
, header_size
,
766 err
= drbd_send_all(tconn
, sock
->socket
, data
, size
, 0);
770 int conn_send_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
,
771 enum drbd_packet cmd
, unsigned int header_size
,
772 void *data
, unsigned int size
)
776 err
= __send_command(tconn
, 0, sock
, cmd
, header_size
, data
, size
);
777 mutex_unlock(&sock
->mutex
);
781 int drbd_send_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
,
782 enum drbd_packet cmd
, unsigned int header_size
,
783 void *data
, unsigned int size
)
787 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, header_size
,
789 mutex_unlock(&sock
->mutex
);
793 int drbd_send_ping(struct drbd_tconn
*tconn
)
795 struct drbd_socket
*sock
;
798 if (!conn_prepare_command(tconn
, sock
))
800 return conn_send_command(tconn
, sock
, P_PING
, sizeof(struct p_header
), NULL
, 0);
803 int drbd_send_ping_ack(struct drbd_tconn
*tconn
)
805 struct drbd_socket
*sock
;
808 if (!conn_prepare_command(tconn
, sock
))
810 return conn_send_command(tconn
, sock
, P_PING_ACK
, sizeof(struct p_header
), NULL
, 0);
813 int drbd_send_sync_param(struct drbd_conf
*mdev
)
815 struct drbd_socket
*sock
;
816 struct p_rs_param_95
*p
;
818 const int apv
= mdev
->tconn
->agreed_pro_version
;
819 enum drbd_packet cmd
;
821 sock
= &mdev
->tconn
->data
;
822 p
= drbd_prepare_command(mdev
, sock
);
826 size
= apv
<= 87 ? sizeof(struct p_rs_param
)
827 : apv
== 88 ? sizeof(struct p_rs_param
)
828 + strlen(mdev
->tconn
->net_conf
->verify_alg
) + 1
829 : apv
<= 94 ? sizeof(struct p_rs_param_89
)
830 : /* apv >= 95 */ sizeof(struct p_rs_param_95
);
832 cmd
= apv
>= 89 ? P_SYNC_PARAM89
: P_SYNC_PARAM
;
834 /* initialize verify_alg and csums_alg */
835 memset(p
->verify_alg
, 0, 2 * SHARED_SECRET_MAX
);
837 if (get_ldev(mdev
)) {
838 p
->rate
= cpu_to_be32(mdev
->ldev
->dc
.resync_rate
);
839 p
->c_plan_ahead
= cpu_to_be32(mdev
->ldev
->dc
.c_plan_ahead
);
840 p
->c_delay_target
= cpu_to_be32(mdev
->ldev
->dc
.c_delay_target
);
841 p
->c_fill_target
= cpu_to_be32(mdev
->ldev
->dc
.c_fill_target
);
842 p
->c_max_rate
= cpu_to_be32(mdev
->ldev
->dc
.c_max_rate
);
845 p
->rate
= cpu_to_be32(DRBD_RATE_DEF
);
846 p
->c_plan_ahead
= cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF
);
847 p
->c_delay_target
= cpu_to_be32(DRBD_C_DELAY_TARGET_DEF
);
848 p
->c_fill_target
= cpu_to_be32(DRBD_C_FILL_TARGET_DEF
);
849 p
->c_max_rate
= cpu_to_be32(DRBD_C_MAX_RATE_DEF
);
853 strcpy(p
->verify_alg
, mdev
->tconn
->net_conf
->verify_alg
);
855 strcpy(p
->csums_alg
, mdev
->tconn
->net_conf
->csums_alg
);
857 return drbd_send_command(mdev
, sock
, cmd
, size
, NULL
, 0);
860 int drbd_send_protocol(struct drbd_tconn
*tconn
)
862 struct drbd_socket
*sock
;
863 struct p_protocol
*p
;
866 if (tconn
->net_conf
->dry_run
&& tconn
->agreed_pro_version
< 92) {
867 conn_err(tconn
, "--dry-run is not supported by peer");
872 p
= conn_prepare_command(tconn
, sock
);
877 if (tconn
->agreed_pro_version
>= 87)
878 size
+= strlen(tconn
->net_conf
->integrity_alg
) + 1;
880 p
->protocol
= cpu_to_be32(tconn
->net_conf
->wire_protocol
);
881 p
->after_sb_0p
= cpu_to_be32(tconn
->net_conf
->after_sb_0p
);
882 p
->after_sb_1p
= cpu_to_be32(tconn
->net_conf
->after_sb_1p
);
883 p
->after_sb_2p
= cpu_to_be32(tconn
->net_conf
->after_sb_2p
);
884 p
->two_primaries
= cpu_to_be32(tconn
->net_conf
->two_primaries
);
886 if (tconn
->net_conf
->want_lose
)
888 if (tconn
->net_conf
->dry_run
)
890 p
->conn_flags
= cpu_to_be32(cf
);
892 if (tconn
->agreed_pro_version
>= 87)
893 strcpy(p
->integrity_alg
, tconn
->net_conf
->integrity_alg
);
894 return conn_send_command(tconn
, sock
, P_PROTOCOL
, size
, NULL
, 0);
897 int _drbd_send_uuids(struct drbd_conf
*mdev
, u64 uuid_flags
)
899 struct drbd_socket
*sock
;
903 if (!get_ldev_if_state(mdev
, D_NEGOTIATING
))
906 sock
= &mdev
->tconn
->data
;
907 p
= drbd_prepare_command(mdev
, sock
);
912 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
913 p
->uuid
[i
] = mdev
->ldev
? cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]) : 0;
915 mdev
->comm_bm_set
= drbd_bm_total_weight(mdev
);
916 p
->uuid
[UI_SIZE
] = cpu_to_be64(mdev
->comm_bm_set
);
917 uuid_flags
|= mdev
->tconn
->net_conf
->want_lose
? 1 : 0;
918 uuid_flags
|= test_bit(CRASHED_PRIMARY
, &mdev
->flags
) ? 2 : 0;
919 uuid_flags
|= mdev
->new_state_tmp
.disk
== D_INCONSISTENT
? 4 : 0;
920 p
->uuid
[UI_FLAGS
] = cpu_to_be64(uuid_flags
);
923 return drbd_send_command(mdev
, sock
, P_UUIDS
, sizeof(*p
), NULL
, 0);
926 int drbd_send_uuids(struct drbd_conf
*mdev
)
928 return _drbd_send_uuids(mdev
, 0);
931 int drbd_send_uuids_skip_initial_sync(struct drbd_conf
*mdev
)
933 return _drbd_send_uuids(mdev
, 8);
936 void drbd_print_uuids(struct drbd_conf
*mdev
, const char *text
)
938 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
939 u64
*uuid
= mdev
->ldev
->md
.uuid
;
940 dev_info(DEV
, "%s %016llX:%016llX:%016llX:%016llX\n",
942 (unsigned long long)uuid
[UI_CURRENT
],
943 (unsigned long long)uuid
[UI_BITMAP
],
944 (unsigned long long)uuid
[UI_HISTORY_START
],
945 (unsigned long long)uuid
[UI_HISTORY_END
]);
948 dev_info(DEV
, "%s effective data uuid: %016llX\n",
950 (unsigned long long)mdev
->ed_uuid
);
954 void drbd_gen_and_send_sync_uuid(struct drbd_conf
*mdev
)
956 struct drbd_socket
*sock
;
960 D_ASSERT(mdev
->state
.disk
== D_UP_TO_DATE
);
962 uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
] + UUID_NEW_BM_OFFSET
;
963 drbd_uuid_set(mdev
, UI_BITMAP
, uuid
);
964 drbd_print_uuids(mdev
, "updated sync UUID");
967 sock
= &mdev
->tconn
->data
;
968 p
= drbd_prepare_command(mdev
, sock
);
970 p
->uuid
= cpu_to_be64(uuid
);
971 drbd_send_command(mdev
, sock
, P_SYNC_UUID
, sizeof(*p
), NULL
, 0);
975 int drbd_send_sizes(struct drbd_conf
*mdev
, int trigger_reply
, enum dds_flags flags
)
977 struct drbd_socket
*sock
;
979 sector_t d_size
, u_size
;
980 int q_order_type
, max_bio_size
;
982 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
983 D_ASSERT(mdev
->ldev
->backing_bdev
);
984 d_size
= drbd_get_max_capacity(mdev
->ldev
);
985 u_size
= mdev
->ldev
->dc
.disk_size
;
986 q_order_type
= drbd_queue_order_type(mdev
);
987 max_bio_size
= queue_max_hw_sectors(mdev
->ldev
->backing_bdev
->bd_disk
->queue
) << 9;
988 max_bio_size
= min_t(int, max_bio_size
, DRBD_MAX_BIO_SIZE
);
993 q_order_type
= QUEUE_ORDERED_NONE
;
994 max_bio_size
= DRBD_MAX_BIO_SIZE
; /* ... multiple BIOs per peer_request */
997 sock
= &mdev
->tconn
->data
;
998 p
= drbd_prepare_command(mdev
, sock
);
1001 p
->d_size
= cpu_to_be64(d_size
);
1002 p
->u_size
= cpu_to_be64(u_size
);
1003 p
->c_size
= cpu_to_be64(trigger_reply
? 0 : drbd_get_capacity(mdev
->this_bdev
));
1004 p
->max_bio_size
= cpu_to_be32(max_bio_size
);
1005 p
->queue_order_type
= cpu_to_be16(q_order_type
);
1006 p
->dds_flags
= cpu_to_be16(flags
);
1007 return drbd_send_command(mdev
, sock
, P_SIZES
, sizeof(*p
), NULL
, 0);
1011 * drbd_send_state() - Sends the drbd state to the peer
1012 * @mdev: DRBD device.
1014 int drbd_send_state(struct drbd_conf
*mdev
)
1016 struct drbd_socket
*sock
;
1019 sock
= &mdev
->tconn
->data
;
1020 p
= drbd_prepare_command(mdev
, sock
);
1023 p
->state
= cpu_to_be32(mdev
->state
.i
); /* Within the send mutex */
1024 return drbd_send_command(mdev
, sock
, P_STATE
, sizeof(*p
), NULL
, 0);
1027 int drbd_send_state_req(struct drbd_conf
*mdev
, union drbd_state mask
, union drbd_state val
)
1029 struct drbd_socket
*sock
;
1030 struct p_req_state
*p
;
1032 sock
= &mdev
->tconn
->data
;
1033 p
= drbd_prepare_command(mdev
, sock
);
1036 p
->mask
= cpu_to_be32(mask
.i
);
1037 p
->val
= cpu_to_be32(val
.i
);
1038 return drbd_send_command(mdev
, sock
, P_STATE_CHG_REQ
, sizeof(*p
), NULL
, 0);
1042 int conn_send_state_req(struct drbd_tconn
*tconn
, union drbd_state mask
, union drbd_state val
)
1044 enum drbd_packet cmd
;
1045 struct drbd_socket
*sock
;
1046 struct p_req_state
*p
;
1048 cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REQ
: P_CONN_ST_CHG_REQ
;
1049 sock
= &tconn
->data
;
1050 p
= conn_prepare_command(tconn
, sock
);
1053 p
->mask
= cpu_to_be32(mask
.i
);
1054 p
->val
= cpu_to_be32(val
.i
);
1055 return conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1058 void drbd_send_sr_reply(struct drbd_conf
*mdev
, enum drbd_state_rv retcode
)
1060 struct drbd_socket
*sock
;
1061 struct p_req_state_reply
*p
;
1063 sock
= &mdev
->tconn
->meta
;
1064 p
= drbd_prepare_command(mdev
, sock
);
1066 p
->retcode
= cpu_to_be32(retcode
);
1067 drbd_send_command(mdev
, sock
, P_STATE_CHG_REPLY
, sizeof(*p
), NULL
, 0);
1071 void conn_send_sr_reply(struct drbd_tconn
*tconn
, enum drbd_state_rv retcode
)
1073 struct drbd_socket
*sock
;
1074 struct p_req_state_reply
*p
;
1075 enum drbd_packet cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REPLY
: P_CONN_ST_CHG_REPLY
;
1077 sock
= &tconn
->meta
;
1078 p
= conn_prepare_command(tconn
, sock
);
1080 p
->retcode
= cpu_to_be32(retcode
);
1081 conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1085 static void dcbp_set_code(struct p_compressed_bm
*p
, enum drbd_bitmap_code code
)
1087 BUG_ON(code
& ~0xf);
1088 p
->encoding
= (p
->encoding
& ~0xf) | code
;
1091 static void dcbp_set_start(struct p_compressed_bm
*p
, int set
)
1093 p
->encoding
= (p
->encoding
& ~0x80) | (set
? 0x80 : 0);
1096 static void dcbp_set_pad_bits(struct p_compressed_bm
*p
, int n
)
1099 p
->encoding
= (p
->encoding
& (~0x7 << 4)) | (n
<< 4);
1102 int fill_bitmap_rle_bits(struct drbd_conf
*mdev
,
1103 struct p_compressed_bm
*p
,
1104 struct bm_xfer_ctx
*c
)
1106 struct bitstream bs
;
1107 unsigned long plain_bits
;
1114 /* may we use this feature? */
1115 if ((mdev
->tconn
->net_conf
->use_rle
== 0) ||
1116 (mdev
->tconn
->agreed_pro_version
< 90))
1119 if (c
->bit_offset
>= c
->bm_bits
)
1120 return 0; /* nothing to do. */
1122 /* use at most thus many bytes */
1123 bitstream_init(&bs
, p
->code
, BM_PACKET_VLI_BYTES_MAX
, 0);
1124 memset(p
->code
, 0, BM_PACKET_VLI_BYTES_MAX
);
1125 /* plain bits covered in this code string */
1128 /* p->encoding & 0x80 stores whether the first run length is set.
1129 * bit offset is implicit.
1130 * start with toggle == 2 to be able to tell the first iteration */
1133 /* see how much plain bits we can stuff into one packet
1134 * using RLE and VLI. */
1136 tmp
= (toggle
== 0) ? _drbd_bm_find_next_zero(mdev
, c
->bit_offset
)
1137 : _drbd_bm_find_next(mdev
, c
->bit_offset
);
1140 rl
= tmp
- c
->bit_offset
;
1142 if (toggle
== 2) { /* first iteration */
1144 /* the first checked bit was set,
1145 * store start value, */
1146 dcbp_set_start(p
, 1);
1147 /* but skip encoding of zero run length */
1151 dcbp_set_start(p
, 0);
1154 /* paranoia: catch zero runlength.
1155 * can only happen if bitmap is modified while we scan it. */
1157 dev_err(DEV
, "unexpected zero runlength while encoding bitmap "
1158 "t:%u bo:%lu\n", toggle
, c
->bit_offset
);
1162 bits
= vli_encode_bits(&bs
, rl
);
1163 if (bits
== -ENOBUFS
) /* buffer full */
1166 dev_err(DEV
, "error while encoding bitmap: %d\n", bits
);
1172 c
->bit_offset
= tmp
;
1173 } while (c
->bit_offset
< c
->bm_bits
);
1175 len
= bs
.cur
.b
- p
->code
+ !!bs
.cur
.bit
;
1177 if (plain_bits
< (len
<< 3)) {
1178 /* incompressible with this method.
1179 * we need to rewind both word and bit position. */
1180 c
->bit_offset
-= plain_bits
;
1181 bm_xfer_ctx_bit_to_word_offset(c
);
1182 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1186 /* RLE + VLI was able to compress it just fine.
1187 * update c->word_offset. */
1188 bm_xfer_ctx_bit_to_word_offset(c
);
1190 /* store pad_bits */
1191 dcbp_set_pad_bits(p
, (8 - bs
.cur
.bit
) & 0x7);
1197 * send_bitmap_rle_or_plain
1199 * Return 0 when done, 1 when another iteration is needed, and a negative error
1200 * code upon failure.
1203 send_bitmap_rle_or_plain(struct drbd_conf
*mdev
, struct bm_xfer_ctx
*c
)
1205 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1206 struct p_compressed_bm
*p
= sock
->sbuf
;
1207 unsigned long num_words
;
1210 len
= fill_bitmap_rle_bits(mdev
, p
, c
);
1215 dcbp_set_code(p
, RLE_VLI_Bits
);
1216 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
,
1217 P_COMPRESSED_BITMAP
, sizeof(*p
) + len
,
1220 c
->bytes
[0] += sizeof(*p
) + len
;
1222 if (c
->bit_offset
>= c
->bm_bits
)
1225 /* was not compressible.
1226 * send a buffer full of plain text bits instead. */
1227 struct p_header
*h
= sock
->sbuf
;
1228 num_words
= min_t(size_t, BM_PACKET_WORDS
, c
->bm_words
- c
->word_offset
);
1229 len
= num_words
* sizeof(long);
1231 drbd_bm_get_lel(mdev
, c
->word_offset
, num_words
,
1232 (unsigned long *)h
->payload
);
1233 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_BITMAP
,
1234 sizeof(*h
) + len
, NULL
, 0);
1235 c
->word_offset
+= num_words
;
1236 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1239 c
->bytes
[1] += sizeof(struct p_header80
) + len
;
1241 if (c
->bit_offset
> c
->bm_bits
)
1242 c
->bit_offset
= c
->bm_bits
;
1246 INFO_bm_xfer_stats(mdev
, "send", c
);
1254 /* See the comment at receive_bitmap() */
1255 static int _drbd_send_bitmap(struct drbd_conf
*mdev
)
1257 struct bm_xfer_ctx c
;
1260 if (!expect(mdev
->bitmap
))
1263 if (get_ldev(mdev
)) {
1264 if (drbd_md_test_flag(mdev
->ldev
, MDF_FULL_SYNC
)) {
1265 dev_info(DEV
, "Writing the whole bitmap, MDF_FullSync was set.\n");
1266 drbd_bm_set_all(mdev
);
1267 if (drbd_bm_write(mdev
)) {
1268 /* write_bm did fail! Leave full sync flag set in Meta P_DATA
1269 * but otherwise process as per normal - need to tell other
1270 * side that a full resync is required! */
1271 dev_err(DEV
, "Failed to write bitmap to disk!\n");
1273 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
1280 c
= (struct bm_xfer_ctx
) {
1281 .bm_bits
= drbd_bm_bits(mdev
),
1282 .bm_words
= drbd_bm_words(mdev
),
1286 err
= send_bitmap_rle_or_plain(mdev
, &c
);
1292 int drbd_send_bitmap(struct drbd_conf
*mdev
)
1294 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1297 mutex_lock(&sock
->mutex
);
1299 err
= !_drbd_send_bitmap(mdev
);
1300 mutex_unlock(&sock
->mutex
);
1304 void drbd_send_b_ack(struct drbd_conf
*mdev
, u32 barrier_nr
, u32 set_size
)
1306 struct drbd_socket
*sock
;
1307 struct p_barrier_ack
*p
;
1309 if (mdev
->state
.conn
< C_CONNECTED
)
1312 sock
= &mdev
->tconn
->meta
;
1313 p
= drbd_prepare_command(mdev
, sock
);
1316 p
->barrier
= barrier_nr
;
1317 p
->set_size
= cpu_to_be32(set_size
);
1318 drbd_send_command(mdev
, sock
, P_BARRIER_ACK
, sizeof(*p
), NULL
, 0);
1322 * _drbd_send_ack() - Sends an ack packet
1323 * @mdev: DRBD device.
1324 * @cmd: Packet command code.
1325 * @sector: sector, needs to be in big endian byte order
1326 * @blksize: size in byte, needs to be in big endian byte order
1327 * @block_id: Id, big endian byte order
1329 static int _drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1330 u64 sector
, u32 blksize
, u64 block_id
)
1332 struct drbd_socket
*sock
;
1333 struct p_block_ack
*p
;
1335 if (mdev
->state
.conn
< C_CONNECTED
)
1338 sock
= &mdev
->tconn
->meta
;
1339 p
= drbd_prepare_command(mdev
, sock
);
1343 p
->block_id
= block_id
;
1344 p
->blksize
= blksize
;
1345 p
->seq_num
= cpu_to_be32(atomic_inc_return(&mdev
->packet_seq
));
1346 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1349 /* dp->sector and dp->block_id already/still in network byte order,
1350 * data_size is payload size according to dp->head,
1351 * and may need to be corrected for digest size. */
1352 void drbd_send_ack_dp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1353 struct p_data
*dp
, int data_size
)
1355 data_size
-= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_r_tfm
) ?
1356 crypto_hash_digestsize(mdev
->tconn
->integrity_r_tfm
) : 0;
1357 _drbd_send_ack(mdev
, cmd
, dp
->sector
, cpu_to_be32(data_size
),
1361 void drbd_send_ack_rp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1362 struct p_block_req
*rp
)
1364 _drbd_send_ack(mdev
, cmd
, rp
->sector
, rp
->blksize
, rp
->block_id
);
1368 * drbd_send_ack() - Sends an ack packet
1369 * @mdev: DRBD device
1370 * @cmd: packet command code
1371 * @peer_req: peer request
1373 int drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1374 struct drbd_peer_request
*peer_req
)
1376 return _drbd_send_ack(mdev
, cmd
,
1377 cpu_to_be64(peer_req
->i
.sector
),
1378 cpu_to_be32(peer_req
->i
.size
),
1379 peer_req
->block_id
);
1382 /* This function misuses the block_id field to signal if the blocks
1383 * are is sync or not. */
1384 int drbd_send_ack_ex(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1385 sector_t sector
, int blksize
, u64 block_id
)
1387 return _drbd_send_ack(mdev
, cmd
,
1388 cpu_to_be64(sector
),
1389 cpu_to_be32(blksize
),
1390 cpu_to_be64(block_id
));
1393 int drbd_send_drequest(struct drbd_conf
*mdev
, int cmd
,
1394 sector_t sector
, int size
, u64 block_id
)
1396 struct drbd_socket
*sock
;
1397 struct p_block_req
*p
;
1399 sock
= &mdev
->tconn
->data
;
1400 p
= drbd_prepare_command(mdev
, sock
);
1403 p
->sector
= cpu_to_be64(sector
);
1404 p
->block_id
= block_id
;
1405 p
->blksize
= cpu_to_be32(size
);
1406 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1409 int drbd_send_drequest_csum(struct drbd_conf
*mdev
, sector_t sector
, int size
,
1410 void *digest
, int digest_size
, enum drbd_packet cmd
)
1412 struct drbd_socket
*sock
;
1413 struct p_block_req
*p
;
1415 /* FIXME: Put the digest into the preallocated socket buffer. */
1417 sock
= &mdev
->tconn
->data
;
1418 p
= drbd_prepare_command(mdev
, sock
);
1421 p
->sector
= cpu_to_be64(sector
);
1422 p
->block_id
= ID_SYNCER
/* unused */;
1423 p
->blksize
= cpu_to_be32(size
);
1424 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
),
1425 digest
, digest_size
);
1428 int drbd_send_ov_request(struct drbd_conf
*mdev
, sector_t sector
, int size
)
1430 struct drbd_socket
*sock
;
1431 struct p_block_req
*p
;
1433 sock
= &mdev
->tconn
->data
;
1434 p
= drbd_prepare_command(mdev
, sock
);
1437 p
->sector
= cpu_to_be64(sector
);
1438 p
->block_id
= ID_SYNCER
/* unused */;
1439 p
->blksize
= cpu_to_be32(size
);
1440 return drbd_send_command(mdev
, sock
, P_OV_REQUEST
, sizeof(*p
), NULL
, 0);
1443 /* called on sndtimeo
1444 * returns false if we should retry,
1445 * true if we think connection is dead
1447 static int we_should_drop_the_connection(struct drbd_tconn
*tconn
, struct socket
*sock
)
1450 /* long elapsed = (long)(jiffies - mdev->last_received); */
1452 drop_it
= tconn
->meta
.socket
== sock
1453 || !tconn
->asender
.task
1454 || get_t_state(&tconn
->asender
) != RUNNING
1455 || tconn
->cstate
< C_WF_REPORT_PARAMS
;
1460 drop_it
= !--tconn
->ko_count
;
1462 conn_err(tconn
, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1463 current
->comm
, current
->pid
, tconn
->ko_count
);
1464 request_ping(tconn
);
1467 return drop_it
; /* && (mdev->state == R_PRIMARY) */;
1470 static void drbd_update_congested(struct drbd_tconn
*tconn
)
1472 struct sock
*sk
= tconn
->data
.socket
->sk
;
1473 if (sk
->sk_wmem_queued
> sk
->sk_sndbuf
* 4 / 5)
1474 set_bit(NET_CONGESTED
, &tconn
->flags
);
1477 /* The idea of sendpage seems to be to put some kind of reference
1478 * to the page into the skb, and to hand it over to the NIC. In
1479 * this process get_page() gets called.
1481 * As soon as the page was really sent over the network put_page()
1482 * gets called by some part of the network layer. [ NIC driver? ]
1484 * [ get_page() / put_page() increment/decrement the count. If count
1485 * reaches 0 the page will be freed. ]
1487 * This works nicely with pages from FSs.
1488 * But this means that in protocol A we might signal IO completion too early!
1490 * In order not to corrupt data during a resync we must make sure
1491 * that we do not reuse our own buffer pages (EEs) to early, therefore
1492 * we have the net_ee list.
1494 * XFS seems to have problems, still, it submits pages with page_count == 0!
1495 * As a workaround, we disable sendpage on pages
1496 * with page_count == 0 or PageSlab.
1498 static int _drbd_no_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1499 int offset
, size_t size
, unsigned msg_flags
)
1501 struct socket
*socket
;
1505 socket
= mdev
->tconn
->data
.socket
;
1506 addr
= kmap(page
) + offset
;
1507 err
= drbd_send_all(mdev
->tconn
, socket
, addr
, size
, msg_flags
);
1510 mdev
->send_cnt
+= size
>> 9;
1514 static int _drbd_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1515 int offset
, size_t size
, unsigned msg_flags
)
1517 struct socket
*socket
= mdev
->tconn
->data
.socket
;
1518 mm_segment_t oldfs
= get_fs();
1522 /* e.g. XFS meta- & log-data is in slab pages, which have a
1523 * page_count of 0 and/or have PageSlab() set.
1524 * we cannot use send_page for those, as that does get_page();
1525 * put_page(); and would cause either a VM_BUG directly, or
1526 * __page_cache_release a page that would actually still be referenced
1527 * by someone, leading to some obscure delayed Oops somewhere else. */
1528 if (disable_sendpage
|| (page_count(page
) < 1) || PageSlab(page
))
1529 return _drbd_no_send_page(mdev
, page
, offset
, size
, msg_flags
);
1531 msg_flags
|= MSG_NOSIGNAL
;
1532 drbd_update_congested(mdev
->tconn
);
1537 sent
= socket
->ops
->sendpage(socket
, page
, offset
, len
, msg_flags
);
1539 if (sent
== -EAGAIN
) {
1540 if (we_should_drop_the_connection(mdev
->tconn
, socket
))
1544 dev_warn(DEV
, "%s: size=%d len=%d sent=%d\n",
1545 __func__
, (int)size
, len
, sent
);
1552 } while (len
> 0 /* THINK && mdev->cstate >= C_CONNECTED*/);
1554 clear_bit(NET_CONGESTED
, &mdev
->tconn
->flags
);
1558 mdev
->send_cnt
+= size
>> 9;
1563 static int _drbd_send_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1565 struct bio_vec
*bvec
;
1567 /* hint all but last page with MSG_MORE */
1568 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1571 err
= _drbd_no_send_page(mdev
, bvec
->bv_page
,
1572 bvec
->bv_offset
, bvec
->bv_len
,
1573 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1580 static int _drbd_send_zc_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1582 struct bio_vec
*bvec
;
1584 /* hint all but last page with MSG_MORE */
1585 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1588 err
= _drbd_send_page(mdev
, bvec
->bv_page
,
1589 bvec
->bv_offset
, bvec
->bv_len
,
1590 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1597 static int _drbd_send_zc_ee(struct drbd_conf
*mdev
,
1598 struct drbd_peer_request
*peer_req
)
1600 struct page
*page
= peer_req
->pages
;
1601 unsigned len
= peer_req
->i
.size
;
1604 /* hint all but last page with MSG_MORE */
1605 page_chain_for_each(page
) {
1606 unsigned l
= min_t(unsigned, len
, PAGE_SIZE
);
1608 err
= _drbd_send_page(mdev
, page
, 0, l
,
1609 page_chain_next(page
) ? MSG_MORE
: 0);
1617 static u32
bio_flags_to_wire(struct drbd_conf
*mdev
, unsigned long bi_rw
)
1619 if (mdev
->tconn
->agreed_pro_version
>= 95)
1620 return (bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0) |
1621 (bi_rw
& REQ_FUA
? DP_FUA
: 0) |
1622 (bi_rw
& REQ_FLUSH
? DP_FLUSH
: 0) |
1623 (bi_rw
& REQ_DISCARD
? DP_DISCARD
: 0);
1625 return bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0;
1628 /* Used to send write requests
1629 * R_PRIMARY -> Peer (P_DATA)
1631 int drbd_send_dblock(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1633 struct drbd_socket
*sock
;
1635 unsigned int dp_flags
= 0;
1639 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1640 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1642 sock
= &mdev
->tconn
->data
;
1643 p
= drbd_prepare_command(mdev
, sock
);
1646 p
->sector
= cpu_to_be64(req
->i
.sector
);
1647 p
->block_id
= (unsigned long)req
;
1648 p
->seq_num
= cpu_to_be32(req
->seq_num
= atomic_inc_return(&mdev
->packet_seq
));
1649 dp_flags
= bio_flags_to_wire(mdev
, req
->master_bio
->bi_rw
);
1650 if (mdev
->state
.conn
>= C_SYNC_SOURCE
&&
1651 mdev
->state
.conn
<= C_PAUSED_SYNC_T
)
1652 dp_flags
|= DP_MAY_SET_IN_SYNC
;
1653 p
->dp_flags
= cpu_to_be32(dp_flags
);
1655 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, p
+ 1);
1656 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_DATA
, sizeof(*p
) + dgs
, NULL
, req
->i
.size
);
1658 /* For protocol A, we have to memcpy the payload into
1659 * socket buffers, as we may complete right away
1660 * as soon as we handed it over to tcp, at which point the data
1661 * pages may become invalid.
1663 * For data-integrity enabled, we copy it as well, so we can be
1664 * sure that even if the bio pages may still be modified, it
1665 * won't change the data on the wire, thus if the digest checks
1666 * out ok after sending on this side, but does not fit on the
1667 * receiving side, we sure have detected corruption elsewhere.
1669 if (mdev
->tconn
->net_conf
->wire_protocol
== DRBD_PROT_A
|| dgs
)
1670 err
= _drbd_send_bio(mdev
, req
->master_bio
);
1672 err
= _drbd_send_zc_bio(mdev
, req
->master_bio
);
1674 /* double check digest, sometimes buffers have been modified in flight. */
1675 if (dgs
> 0 && dgs
<= 64) {
1676 /* 64 byte, 512 bit, is the largest digest size
1677 * currently supported in kernel crypto. */
1678 unsigned char digest
[64];
1679 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, digest
);
1680 if (memcmp(p
+ 1, digest
, dgs
)) {
1682 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1683 (unsigned long long)req
->i
.sector
, req
->i
.size
);
1685 } /* else if (dgs > 64) {
1686 ... Be noisy about digest too large ...
1689 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1694 /* answer packet, used to send data back for read requests:
1695 * Peer -> (diskless) R_PRIMARY (P_DATA_REPLY)
1696 * C_SYNC_SOURCE -> C_SYNC_TARGET (P_RS_DATA_REPLY)
1698 int drbd_send_block(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1699 struct drbd_peer_request
*peer_req
)
1701 struct drbd_socket
*sock
;
1706 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1707 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1709 sock
= &mdev
->tconn
->data
;
1710 p
= drbd_prepare_command(mdev
, sock
);
1713 p
->sector
= cpu_to_be64(peer_req
->i
.sector
);
1714 p
->block_id
= peer_req
->block_id
;
1715 p
->seq_num
= 0; /* unused */
1717 drbd_csum_ee(mdev
, mdev
->tconn
->integrity_w_tfm
, peer_req
, p
+ 1);
1718 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, sizeof(*p
) + dgs
, NULL
, peer_req
->i
.size
);
1720 err
= _drbd_send_zc_ee(mdev
, peer_req
);
1721 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1726 int drbd_send_out_of_sync(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1728 struct drbd_socket
*sock
;
1729 struct p_block_desc
*p
;
1731 sock
= &mdev
->tconn
->data
;
1732 p
= drbd_prepare_command(mdev
, sock
);
1735 p
->sector
= cpu_to_be64(req
->i
.sector
);
1736 p
->blksize
= cpu_to_be32(req
->i
.size
);
1737 return drbd_send_command(mdev
, sock
, P_OUT_OF_SYNC
, sizeof(*p
), NULL
, 0);
1741 drbd_send distinguishes two cases:
1743 Packets sent via the data socket "sock"
1744 and packets sent via the meta data socket "msock"
1747 -----------------+-------------------------+------------------------------
1748 timeout conf.timeout / 2 conf.timeout / 2
1749 timeout action send a ping via msock Abort communication
1750 and close all sockets
1754 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
1756 int drbd_send(struct drbd_tconn
*tconn
, struct socket
*sock
,
1757 void *buf
, size_t size
, unsigned msg_flags
)
1766 /* THINK if (signal_pending) return ... ? */
1771 msg
.msg_name
= NULL
;
1772 msg
.msg_namelen
= 0;
1773 msg
.msg_control
= NULL
;
1774 msg
.msg_controllen
= 0;
1775 msg
.msg_flags
= msg_flags
| MSG_NOSIGNAL
;
1777 if (sock
== tconn
->data
.socket
) {
1778 tconn
->ko_count
= tconn
->net_conf
->ko_count
;
1779 drbd_update_congested(tconn
);
1783 * tcp_sendmsg does _not_ use its size parameter at all ?
1785 * -EAGAIN on timeout, -EINTR on signal.
1788 * do we need to block DRBD_SIG if sock == &meta.socket ??
1789 * otherwise wake_asender() might interrupt some send_*Ack !
1791 rv
= kernel_sendmsg(sock
, &msg
, &iov
, 1, size
);
1792 if (rv
== -EAGAIN
) {
1793 if (we_should_drop_the_connection(tconn
, sock
))
1799 flush_signals(current
);
1807 } while (sent
< size
);
1809 if (sock
== tconn
->data
.socket
)
1810 clear_bit(NET_CONGESTED
, &tconn
->flags
);
1813 if (rv
!= -EAGAIN
) {
1814 conn_err(tconn
, "%s_sendmsg returned %d\n",
1815 sock
== tconn
->meta
.socket
? "msock" : "sock",
1817 conn_request_state(tconn
, NS(conn
, C_BROKEN_PIPE
), CS_HARD
);
1819 conn_request_state(tconn
, NS(conn
, C_TIMEOUT
), CS_HARD
);
1826 * drbd_send_all - Send an entire buffer
1828 * Returns 0 upon success and a negative error value otherwise.
1830 int drbd_send_all(struct drbd_tconn
*tconn
, struct socket
*sock
, void *buffer
,
1831 size_t size
, unsigned msg_flags
)
1835 err
= drbd_send(tconn
, sock
, buffer
, size
, msg_flags
);
1843 static int drbd_open(struct block_device
*bdev
, fmode_t mode
)
1845 struct drbd_conf
*mdev
= bdev
->bd_disk
->private_data
;
1846 unsigned long flags
;
1849 mutex_lock(&drbd_main_mutex
);
1850 spin_lock_irqsave(&mdev
->tconn
->req_lock
, flags
);
1851 /* to have a stable mdev->state.role
1852 * and no race with updating open_cnt */
1854 if (mdev
->state
.role
!= R_PRIMARY
) {
1855 if (mode
& FMODE_WRITE
)
1857 else if (!allow_oos
)
1863 spin_unlock_irqrestore(&mdev
->tconn
->req_lock
, flags
);
1864 mutex_unlock(&drbd_main_mutex
);
1869 static int drbd_release(struct gendisk
*gd
, fmode_t mode
)
1871 struct drbd_conf
*mdev
= gd
->private_data
;
1872 mutex_lock(&drbd_main_mutex
);
1874 mutex_unlock(&drbd_main_mutex
);
1878 static void drbd_set_defaults(struct drbd_conf
*mdev
)
1880 /* Beware! The actual layout differs
1881 * between big endian and little endian */
1882 mdev
->state
= (union drbd_dev_state
) {
1883 { .role
= R_SECONDARY
,
1885 .conn
= C_STANDALONE
,
1891 void drbd_init_set_defaults(struct drbd_conf
*mdev
)
1893 /* the memset(,0,) did most of this.
1894 * note: only assignments, no allocation in here */
1896 drbd_set_defaults(mdev
);
1898 atomic_set(&mdev
->ap_bio_cnt
, 0);
1899 atomic_set(&mdev
->ap_pending_cnt
, 0);
1900 atomic_set(&mdev
->rs_pending_cnt
, 0);
1901 atomic_set(&mdev
->unacked_cnt
, 0);
1902 atomic_set(&mdev
->local_cnt
, 0);
1903 atomic_set(&mdev
->pp_in_use_by_net
, 0);
1904 atomic_set(&mdev
->rs_sect_in
, 0);
1905 atomic_set(&mdev
->rs_sect_ev
, 0);
1906 atomic_set(&mdev
->ap_in_flight
, 0);
1908 mutex_init(&mdev
->md_io_mutex
);
1909 mutex_init(&mdev
->own_state_mutex
);
1910 mdev
->state_mutex
= &mdev
->own_state_mutex
;
1912 spin_lock_init(&mdev
->al_lock
);
1913 spin_lock_init(&mdev
->peer_seq_lock
);
1914 spin_lock_init(&mdev
->epoch_lock
);
1916 INIT_LIST_HEAD(&mdev
->active_ee
);
1917 INIT_LIST_HEAD(&mdev
->sync_ee
);
1918 INIT_LIST_HEAD(&mdev
->done_ee
);
1919 INIT_LIST_HEAD(&mdev
->read_ee
);
1920 INIT_LIST_HEAD(&mdev
->net_ee
);
1921 INIT_LIST_HEAD(&mdev
->resync_reads
);
1922 INIT_LIST_HEAD(&mdev
->resync_work
.list
);
1923 INIT_LIST_HEAD(&mdev
->unplug_work
.list
);
1924 INIT_LIST_HEAD(&mdev
->go_diskless
.list
);
1925 INIT_LIST_HEAD(&mdev
->md_sync_work
.list
);
1926 INIT_LIST_HEAD(&mdev
->start_resync_work
.list
);
1927 INIT_LIST_HEAD(&mdev
->bm_io_work
.w
.list
);
1929 mdev
->resync_work
.cb
= w_resync_timer
;
1930 mdev
->unplug_work
.cb
= w_send_write_hint
;
1931 mdev
->go_diskless
.cb
= w_go_diskless
;
1932 mdev
->md_sync_work
.cb
= w_md_sync
;
1933 mdev
->bm_io_work
.w
.cb
= w_bitmap_io
;
1934 mdev
->start_resync_work
.cb
= w_start_resync
;
1936 mdev
->resync_work
.mdev
= mdev
;
1937 mdev
->unplug_work
.mdev
= mdev
;
1938 mdev
->go_diskless
.mdev
= mdev
;
1939 mdev
->md_sync_work
.mdev
= mdev
;
1940 mdev
->bm_io_work
.w
.mdev
= mdev
;
1941 mdev
->start_resync_work
.mdev
= mdev
;
1943 init_timer(&mdev
->resync_timer
);
1944 init_timer(&mdev
->md_sync_timer
);
1945 init_timer(&mdev
->start_resync_timer
);
1946 init_timer(&mdev
->request_timer
);
1947 mdev
->resync_timer
.function
= resync_timer_fn
;
1948 mdev
->resync_timer
.data
= (unsigned long) mdev
;
1949 mdev
->md_sync_timer
.function
= md_sync_timer_fn
;
1950 mdev
->md_sync_timer
.data
= (unsigned long) mdev
;
1951 mdev
->start_resync_timer
.function
= start_resync_timer_fn
;
1952 mdev
->start_resync_timer
.data
= (unsigned long) mdev
;
1953 mdev
->request_timer
.function
= request_timer_fn
;
1954 mdev
->request_timer
.data
= (unsigned long) mdev
;
1956 init_waitqueue_head(&mdev
->misc_wait
);
1957 init_waitqueue_head(&mdev
->state_wait
);
1958 init_waitqueue_head(&mdev
->ee_wait
);
1959 init_waitqueue_head(&mdev
->al_wait
);
1960 init_waitqueue_head(&mdev
->seq_wait
);
1962 /* mdev->tconn->agreed_pro_version gets initialized in drbd_connect() */
1963 mdev
->write_ordering
= WO_bdev_flush
;
1964 mdev
->resync_wenr
= LC_FREE
;
1965 mdev
->peer_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1966 mdev
->local_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1969 void drbd_mdev_cleanup(struct drbd_conf
*mdev
)
1972 if (mdev
->tconn
->receiver
.t_state
!= NONE
)
1973 dev_err(DEV
, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1974 mdev
->tconn
->receiver
.t_state
);
1976 /* no need to lock it, I'm the only thread alive */
1977 if (atomic_read(&mdev
->current_epoch
->epoch_size
) != 0)
1978 dev_err(DEV
, "epoch_size:%d\n", atomic_read(&mdev
->current_epoch
->epoch_size
));
1988 mdev
->rs_failed
= 0;
1989 mdev
->rs_last_events
= 0;
1990 mdev
->rs_last_sect_ev
= 0;
1991 for (i
= 0; i
< DRBD_SYNC_MARKS
; i
++) {
1992 mdev
->rs_mark_left
[i
] = 0;
1993 mdev
->rs_mark_time
[i
] = 0;
1995 D_ASSERT(mdev
->tconn
->net_conf
== NULL
);
1997 drbd_set_my_capacity(mdev
, 0);
1999 /* maybe never allocated. */
2000 drbd_bm_resize(mdev
, 0, 1);
2001 drbd_bm_cleanup(mdev
);
2004 drbd_free_resources(mdev
);
2005 clear_bit(AL_SUSPENDED
, &mdev
->flags
);
2008 * currently we drbd_init_ee only on module load, so
2009 * we may do drbd_release_ee only on module unload!
2011 D_ASSERT(list_empty(&mdev
->active_ee
));
2012 D_ASSERT(list_empty(&mdev
->sync_ee
));
2013 D_ASSERT(list_empty(&mdev
->done_ee
));
2014 D_ASSERT(list_empty(&mdev
->read_ee
));
2015 D_ASSERT(list_empty(&mdev
->net_ee
));
2016 D_ASSERT(list_empty(&mdev
->resync_reads
));
2017 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2018 D_ASSERT(list_empty(&mdev
->tconn
->meta
.work
.q
));
2019 D_ASSERT(list_empty(&mdev
->resync_work
.list
));
2020 D_ASSERT(list_empty(&mdev
->unplug_work
.list
));
2021 D_ASSERT(list_empty(&mdev
->go_diskless
.list
));
2023 drbd_set_defaults(mdev
);
2027 static void drbd_destroy_mempools(void)
2031 while (drbd_pp_pool
) {
2032 page
= drbd_pp_pool
;
2033 drbd_pp_pool
= (struct page
*)page_private(page
);
2038 /* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */
2040 if (drbd_md_io_bio_set
)
2041 bioset_free(drbd_md_io_bio_set
);
2042 if (drbd_md_io_page_pool
)
2043 mempool_destroy(drbd_md_io_page_pool
);
2044 if (drbd_ee_mempool
)
2045 mempool_destroy(drbd_ee_mempool
);
2046 if (drbd_request_mempool
)
2047 mempool_destroy(drbd_request_mempool
);
2049 kmem_cache_destroy(drbd_ee_cache
);
2050 if (drbd_request_cache
)
2051 kmem_cache_destroy(drbd_request_cache
);
2052 if (drbd_bm_ext_cache
)
2053 kmem_cache_destroy(drbd_bm_ext_cache
);
2054 if (drbd_al_ext_cache
)
2055 kmem_cache_destroy(drbd_al_ext_cache
);
2057 drbd_md_io_bio_set
= NULL
;
2058 drbd_md_io_page_pool
= NULL
;
2059 drbd_ee_mempool
= NULL
;
2060 drbd_request_mempool
= NULL
;
2061 drbd_ee_cache
= NULL
;
2062 drbd_request_cache
= NULL
;
2063 drbd_bm_ext_cache
= NULL
;
2064 drbd_al_ext_cache
= NULL
;
2069 static int drbd_create_mempools(void)
2072 const int number
= (DRBD_MAX_BIO_SIZE
/PAGE_SIZE
) * minor_count
;
2075 /* prepare our caches and mempools */
2076 drbd_request_mempool
= NULL
;
2077 drbd_ee_cache
= NULL
;
2078 drbd_request_cache
= NULL
;
2079 drbd_bm_ext_cache
= NULL
;
2080 drbd_al_ext_cache
= NULL
;
2081 drbd_pp_pool
= NULL
;
2082 drbd_md_io_page_pool
= NULL
;
2083 drbd_md_io_bio_set
= NULL
;
2086 drbd_request_cache
= kmem_cache_create(
2087 "drbd_req", sizeof(struct drbd_request
), 0, 0, NULL
);
2088 if (drbd_request_cache
== NULL
)
2091 drbd_ee_cache
= kmem_cache_create(
2092 "drbd_ee", sizeof(struct drbd_peer_request
), 0, 0, NULL
);
2093 if (drbd_ee_cache
== NULL
)
2096 drbd_bm_ext_cache
= kmem_cache_create(
2097 "drbd_bm", sizeof(struct bm_extent
), 0, 0, NULL
);
2098 if (drbd_bm_ext_cache
== NULL
)
2101 drbd_al_ext_cache
= kmem_cache_create(
2102 "drbd_al", sizeof(struct lc_element
), 0, 0, NULL
);
2103 if (drbd_al_ext_cache
== NULL
)
2107 drbd_md_io_bio_set
= bioset_create(DRBD_MIN_POOL_PAGES
, 0);
2108 if (drbd_md_io_bio_set
== NULL
)
2111 drbd_md_io_page_pool
= mempool_create_page_pool(DRBD_MIN_POOL_PAGES
, 0);
2112 if (drbd_md_io_page_pool
== NULL
)
2115 drbd_request_mempool
= mempool_create(number
,
2116 mempool_alloc_slab
, mempool_free_slab
, drbd_request_cache
);
2117 if (drbd_request_mempool
== NULL
)
2120 drbd_ee_mempool
= mempool_create(number
,
2121 mempool_alloc_slab
, mempool_free_slab
, drbd_ee_cache
);
2122 if (drbd_ee_mempool
== NULL
)
2125 /* drbd's page pool */
2126 spin_lock_init(&drbd_pp_lock
);
2128 for (i
= 0; i
< number
; i
++) {
2129 page
= alloc_page(GFP_HIGHUSER
);
2132 set_page_private(page
, (unsigned long)drbd_pp_pool
);
2133 drbd_pp_pool
= page
;
2135 drbd_pp_vacant
= number
;
2140 drbd_destroy_mempools(); /* in case we allocated some */
2144 static int drbd_notify_sys(struct notifier_block
*this, unsigned long code
,
2147 /* just so we have it. you never know what interesting things we
2148 * might want to do here some day...
2154 static struct notifier_block drbd_notifier
= {
2155 .notifier_call
= drbd_notify_sys
,
2158 static void drbd_release_ee_lists(struct drbd_conf
*mdev
)
2162 rr
= drbd_release_ee(mdev
, &mdev
->active_ee
);
2164 dev_err(DEV
, "%d EEs in active list found!\n", rr
);
2166 rr
= drbd_release_ee(mdev
, &mdev
->sync_ee
);
2168 dev_err(DEV
, "%d EEs in sync list found!\n", rr
);
2170 rr
= drbd_release_ee(mdev
, &mdev
->read_ee
);
2172 dev_err(DEV
, "%d EEs in read list found!\n", rr
);
2174 rr
= drbd_release_ee(mdev
, &mdev
->done_ee
);
2176 dev_err(DEV
, "%d EEs in done list found!\n", rr
);
2178 rr
= drbd_release_ee(mdev
, &mdev
->net_ee
);
2180 dev_err(DEV
, "%d EEs in net list found!\n", rr
);
2183 /* caution. no locking. */
2184 void drbd_delete_device(unsigned int minor
)
2186 struct drbd_conf
*mdev
= minor_to_mdev(minor
);
2191 idr_remove(&mdev
->tconn
->volumes
, mdev
->vnr
);
2192 idr_remove(&minors
, minor
);
2195 /* paranoia asserts */
2196 D_ASSERT(mdev
->open_cnt
== 0);
2197 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2198 /* end paranoia asserts */
2200 del_gendisk(mdev
->vdisk
);
2202 /* cleanup stuff that may have been allocated during
2203 * device (re-)configuration or state changes */
2205 if (mdev
->this_bdev
)
2206 bdput(mdev
->this_bdev
);
2208 drbd_free_resources(mdev
);
2210 drbd_release_ee_lists(mdev
);
2212 lc_destroy(mdev
->act_log
);
2213 lc_destroy(mdev
->resync
);
2215 kfree(mdev
->p_uuid
);
2216 /* mdev->p_uuid = NULL; */
2218 /* cleanup the rest that has been
2219 * allocated from drbd_new_device
2220 * and actually free the mdev itself */
2221 drbd_free_mdev(mdev
);
2224 static void drbd_cleanup(void)
2227 struct drbd_conf
*mdev
;
2229 unregister_reboot_notifier(&drbd_notifier
);
2231 /* first remove proc,
2232 * drbdsetup uses it's presence to detect
2233 * whether DRBD is loaded.
2234 * If we would get stuck in proc removal,
2235 * but have netlink already deregistered,
2236 * some drbdsetup commands may wait forever
2240 remove_proc_entry("drbd", NULL
);
2242 drbd_genl_unregister();
2244 idr_for_each_entry(&minors
, mdev
, i
)
2245 drbd_delete_device(i
);
2246 drbd_destroy_mempools();
2247 unregister_blkdev(DRBD_MAJOR
, "drbd");
2249 idr_destroy(&minors
);
2251 printk(KERN_INFO
"drbd: module cleanup done.\n");
2255 * drbd_congested() - Callback for pdflush
2256 * @congested_data: User data
2257 * @bdi_bits: Bits pdflush is currently interested in
2259 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
2261 static int drbd_congested(void *congested_data
, int bdi_bits
)
2263 struct drbd_conf
*mdev
= congested_data
;
2264 struct request_queue
*q
;
2268 if (!may_inc_ap_bio(mdev
)) {
2269 /* DRBD has frozen IO */
2275 if (get_ldev(mdev
)) {
2276 q
= bdev_get_queue(mdev
->ldev
->backing_bdev
);
2277 r
= bdi_congested(&q
->backing_dev_info
, bdi_bits
);
2283 if (bdi_bits
& (1 << BDI_async_congested
) && test_bit(NET_CONGESTED
, &mdev
->tconn
->flags
)) {
2284 r
|= (1 << BDI_async_congested
);
2285 reason
= reason
== 'b' ? 'a' : 'n';
2289 mdev
->congestion_reason
= reason
;
2293 static void drbd_init_workqueue(struct drbd_work_queue
* wq
)
2295 sema_init(&wq
->s
, 0);
2296 spin_lock_init(&wq
->q_lock
);
2297 INIT_LIST_HEAD(&wq
->q
);
2300 struct drbd_tconn
*conn_by_name(const char *name
)
2302 struct drbd_tconn
*tconn
;
2304 if (!name
|| !name
[0])
2307 mutex_lock(&drbd_cfg_mutex
);
2308 list_for_each_entry(tconn
, &drbd_tconns
, all_tconn
) {
2309 if (!strcmp(tconn
->name
, name
))
2314 mutex_unlock(&drbd_cfg_mutex
);
2318 static int drbd_alloc_socket(struct drbd_socket
*socket
)
2320 socket
->rbuf
= (void *) __get_free_page(GFP_KERNEL
);
2323 socket
->sbuf
= (void *) __get_free_page(GFP_KERNEL
);
2329 static void drbd_free_socket(struct drbd_socket
*socket
)
2331 free_page((unsigned long) socket
->sbuf
);
2332 free_page((unsigned long) socket
->rbuf
);
2335 struct drbd_tconn
*drbd_new_tconn(const char *name
)
2337 struct drbd_tconn
*tconn
;
2339 tconn
= kzalloc(sizeof(struct drbd_tconn
), GFP_KERNEL
);
2343 tconn
->name
= kstrdup(name
, GFP_KERNEL
);
2347 if (drbd_alloc_socket(&tconn
->data
))
2349 if (drbd_alloc_socket(&tconn
->meta
))
2352 if (!zalloc_cpumask_var(&tconn
->cpu_mask
, GFP_KERNEL
))
2355 if (!tl_init(tconn
))
2358 tconn
->cstate
= C_STANDALONE
;
2359 mutex_init(&tconn
->cstate_mutex
);
2360 spin_lock_init(&tconn
->req_lock
);
2361 atomic_set(&tconn
->net_cnt
, 0);
2362 init_waitqueue_head(&tconn
->net_cnt_wait
);
2363 init_waitqueue_head(&tconn
->ping_wait
);
2364 idr_init(&tconn
->volumes
);
2366 drbd_init_workqueue(&tconn
->data
.work
);
2367 mutex_init(&tconn
->data
.mutex
);
2369 drbd_init_workqueue(&tconn
->meta
.work
);
2370 mutex_init(&tconn
->meta
.mutex
);
2372 drbd_thread_init(tconn
, &tconn
->receiver
, drbdd_init
, "receiver");
2373 drbd_thread_init(tconn
, &tconn
->worker
, drbd_worker
, "worker");
2374 drbd_thread_init(tconn
, &tconn
->asender
, drbd_asender
, "asender");
2376 tconn
->res_opts
= (struct res_opts
) {
2377 {}, 0, /* cpu_mask */
2378 DRBD_ON_NO_DATA_DEF
, /* on_no_data */
2381 mutex_lock(&drbd_cfg_mutex
);
2382 list_add_tail(&tconn
->all_tconn
, &drbd_tconns
);
2383 mutex_unlock(&drbd_cfg_mutex
);
2389 free_cpumask_var(tconn
->cpu_mask
);
2390 drbd_free_socket(&tconn
->meta
);
2391 drbd_free_socket(&tconn
->data
);
2398 void drbd_free_tconn(struct drbd_tconn
*tconn
)
2400 list_del(&tconn
->all_tconn
);
2401 idr_destroy(&tconn
->volumes
);
2403 free_cpumask_var(tconn
->cpu_mask
);
2404 drbd_free_socket(&tconn
->meta
);
2405 drbd_free_socket(&tconn
->data
);
2407 kfree(tconn
->int_dig_out
);
2408 kfree(tconn
->int_dig_in
);
2409 kfree(tconn
->int_dig_vv
);
2413 enum drbd_ret_code
conn_new_minor(struct drbd_tconn
*tconn
, unsigned int minor
, int vnr
)
2415 struct drbd_conf
*mdev
;
2416 struct gendisk
*disk
;
2417 struct request_queue
*q
;
2419 int minor_got
= minor
;
2420 enum drbd_ret_code err
= ERR_NOMEM
;
2422 mdev
= minor_to_mdev(minor
);
2424 return ERR_MINOR_EXISTS
;
2426 /* GFP_KERNEL, we are outside of all write-out paths */
2427 mdev
= kzalloc(sizeof(struct drbd_conf
), GFP_KERNEL
);
2431 mdev
->tconn
= tconn
;
2432 mdev
->minor
= minor
;
2435 drbd_init_set_defaults(mdev
);
2437 q
= blk_alloc_queue(GFP_KERNEL
);
2441 q
->queuedata
= mdev
;
2443 disk
= alloc_disk(1);
2448 set_disk_ro(disk
, true);
2451 disk
->major
= DRBD_MAJOR
;
2452 disk
->first_minor
= minor
;
2453 disk
->fops
= &drbd_ops
;
2454 sprintf(disk
->disk_name
, "drbd%d", minor
);
2455 disk
->private_data
= mdev
;
2457 mdev
->this_bdev
= bdget(MKDEV(DRBD_MAJOR
, minor
));
2458 /* we have no partitions. we contain only ourselves. */
2459 mdev
->this_bdev
->bd_contains
= mdev
->this_bdev
;
2461 q
->backing_dev_info
.congested_fn
= drbd_congested
;
2462 q
->backing_dev_info
.congested_data
= mdev
;
2464 blk_queue_make_request(q
, drbd_make_request
);
2465 /* Setting the max_hw_sectors to an odd value of 8kibyte here
2466 This triggers a max_bio_size message upon first attach or connect */
2467 blk_queue_max_hw_sectors(q
, DRBD_MAX_BIO_SIZE_SAFE
>> 8);
2468 blk_queue_bounce_limit(q
, BLK_BOUNCE_ANY
);
2469 blk_queue_merge_bvec(q
, drbd_merge_bvec
);
2470 q
->queue_lock
= &mdev
->tconn
->req_lock
; /* needed since we use */
2472 mdev
->md_io_page
= alloc_page(GFP_KERNEL
);
2473 if (!mdev
->md_io_page
)
2474 goto out_no_io_page
;
2476 if (drbd_bm_init(mdev
))
2478 mdev
->read_requests
= RB_ROOT
;
2479 mdev
->write_requests
= RB_ROOT
;
2481 mdev
->current_epoch
= kzalloc(sizeof(struct drbd_epoch
), GFP_KERNEL
);
2482 if (!mdev
->current_epoch
)
2485 INIT_LIST_HEAD(&mdev
->current_epoch
->list
);
2488 if (!idr_pre_get(&minors
, GFP_KERNEL
))
2489 goto out_no_minor_idr
;
2490 if (idr_get_new_above(&minors
, mdev
, minor
, &minor_got
))
2491 goto out_no_minor_idr
;
2492 if (minor_got
!= minor
) {
2493 err
= ERR_MINOR_EXISTS
;
2494 drbd_msg_put_info("requested minor exists already");
2495 goto out_idr_remove_minor
;
2498 if (!idr_pre_get(&tconn
->volumes
, GFP_KERNEL
))
2499 goto out_idr_remove_minor
;
2500 if (idr_get_new_above(&tconn
->volumes
, mdev
, vnr
, &vnr_got
))
2501 goto out_idr_remove_minor
;
2502 if (vnr_got
!= vnr
) {
2503 err
= ERR_INVALID_REQUEST
;
2504 drbd_msg_put_info("requested volume exists already");
2505 goto out_idr_remove_vol
;
2509 /* inherit the connection state */
2510 mdev
->state
.conn
= tconn
->cstate
;
2511 if (mdev
->state
.conn
== C_WF_REPORT_PARAMS
)
2512 drbd_connected(vnr
, mdev
, tconn
);
2517 idr_remove(&tconn
->volumes
, vnr_got
);
2518 out_idr_remove_minor
:
2519 idr_remove(&minors
, minor_got
);
2522 kfree(mdev
->current_epoch
);
2524 drbd_bm_cleanup(mdev
);
2526 __free_page(mdev
->md_io_page
);
2530 blk_cleanup_queue(q
);
2536 /* counterpart of drbd_new_device.
2537 * last part of drbd_delete_device. */
2538 void drbd_free_mdev(struct drbd_conf
*mdev
)
2540 kfree(mdev
->current_epoch
);
2541 if (mdev
->bitmap
) /* should no longer be there. */
2542 drbd_bm_cleanup(mdev
);
2543 __free_page(mdev
->md_io_page
);
2544 put_disk(mdev
->vdisk
);
2545 blk_cleanup_queue(mdev
->rq_queue
);
2550 int __init
drbd_init(void)
2554 BUILD_BUG_ON(sizeof(struct p_header80
) != sizeof(struct p_header95
));
2555 BUILD_BUG_ON(sizeof(struct p_connection_features
) != 80);
2557 if (minor_count
< DRBD_MINOR_COUNT_MIN
|| minor_count
> DRBD_MINOR_COUNT_MAX
) {
2559 "drbd: invalid minor_count (%d)\n", minor_count
);
2567 err
= register_blkdev(DRBD_MAJOR
, "drbd");
2570 "drbd: unable to register block device major %d\n",
2575 err
= drbd_genl_register();
2577 printk(KERN_ERR
"drbd: unable to register generic netlink family\n");
2582 register_reboot_notifier(&drbd_notifier
);
2585 * allocate all necessary structs
2589 init_waitqueue_head(&drbd_pp_wait
);
2591 drbd_proc
= NULL
; /* play safe for drbd_cleanup */
2594 err
= drbd_create_mempools();
2598 drbd_proc
= proc_create_data("drbd", S_IFREG
| S_IRUGO
, NULL
, &drbd_proc_fops
, NULL
);
2600 printk(KERN_ERR
"drbd: unable to register proc file\n");
2604 rwlock_init(&global_state_lock
);
2605 INIT_LIST_HEAD(&drbd_tconns
);
2607 printk(KERN_INFO
"drbd: initialized. "
2608 "Version: " REL_VERSION
" (api:%d/proto:%d-%d)\n",
2609 API_VERSION
, PRO_VERSION_MIN
, PRO_VERSION_MAX
);
2610 printk(KERN_INFO
"drbd: %s\n", drbd_buildtag());
2611 printk(KERN_INFO
"drbd: registered as block device major %d\n",
2614 return 0; /* Success! */
2619 /* currently always the case */
2620 printk(KERN_ERR
"drbd: ran out of memory\n");
2622 printk(KERN_ERR
"drbd: initialization failure\n");
2626 void drbd_free_bc(struct drbd_backing_dev
*ldev
)
2631 blkdev_put(ldev
->backing_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2632 blkdev_put(ldev
->md_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2637 void drbd_free_sock(struct drbd_tconn
*tconn
)
2639 if (tconn
->data
.socket
) {
2640 mutex_lock(&tconn
->data
.mutex
);
2641 kernel_sock_shutdown(tconn
->data
.socket
, SHUT_RDWR
);
2642 sock_release(tconn
->data
.socket
);
2643 tconn
->data
.socket
= NULL
;
2644 mutex_unlock(&tconn
->data
.mutex
);
2646 if (tconn
->meta
.socket
) {
2647 mutex_lock(&tconn
->meta
.mutex
);
2648 kernel_sock_shutdown(tconn
->meta
.socket
, SHUT_RDWR
);
2649 sock_release(tconn
->meta
.socket
);
2650 tconn
->meta
.socket
= NULL
;
2651 mutex_unlock(&tconn
->meta
.mutex
);
2656 void drbd_free_resources(struct drbd_conf
*mdev
)
2658 crypto_free_hash(mdev
->tconn
->csums_tfm
);
2659 mdev
->tconn
->csums_tfm
= NULL
;
2660 crypto_free_hash(mdev
->tconn
->verify_tfm
);
2661 mdev
->tconn
->verify_tfm
= NULL
;
2662 crypto_free_hash(mdev
->tconn
->cram_hmac_tfm
);
2663 mdev
->tconn
->cram_hmac_tfm
= NULL
;
2664 crypto_free_hash(mdev
->tconn
->integrity_w_tfm
);
2665 mdev
->tconn
->integrity_w_tfm
= NULL
;
2666 crypto_free_hash(mdev
->tconn
->integrity_r_tfm
);
2667 mdev
->tconn
->integrity_r_tfm
= NULL
;
2669 drbd_free_sock(mdev
->tconn
);
2672 drbd_free_bc(mdev
->ldev
);
2673 mdev
->ldev
= NULL
;);
2676 /* meta data management */
2678 struct meta_data_on_disk
{
2679 u64 la_size
; /* last agreed size. */
2680 u64 uuid
[UI_SIZE
]; /* UUIDs. */
2683 u32 flags
; /* MDF */
2686 u32 al_offset
; /* offset to this block */
2687 u32 al_nr_extents
; /* important for restoring the AL */
2688 /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
2689 u32 bm_offset
; /* offset to the bitmap, from here */
2690 u32 bm_bytes_per_bit
; /* BM_BLOCK_SIZE */
2691 u32 la_peer_max_bio_size
; /* last peer max_bio_size */
2692 u32 reserved_u32
[3];
2697 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2698 * @mdev: DRBD device.
2700 void drbd_md_sync(struct drbd_conf
*mdev
)
2702 struct meta_data_on_disk
*buffer
;
2706 del_timer(&mdev
->md_sync_timer
);
2707 /* timer may be rearmed by drbd_md_mark_dirty() now. */
2708 if (!test_and_clear_bit(MD_DIRTY
, &mdev
->flags
))
2711 /* We use here D_FAILED and not D_ATTACHING because we try to write
2712 * metadata even if we detach due to a disk failure! */
2713 if (!get_ldev_if_state(mdev
, D_FAILED
))
2716 mutex_lock(&mdev
->md_io_mutex
);
2717 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2718 memset(buffer
, 0, 512);
2720 buffer
->la_size
= cpu_to_be64(drbd_get_capacity(mdev
->this_bdev
));
2721 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2722 buffer
->uuid
[i
] = cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]);
2723 buffer
->flags
= cpu_to_be32(mdev
->ldev
->md
.flags
);
2724 buffer
->magic
= cpu_to_be32(DRBD_MD_MAGIC
);
2726 buffer
->md_size_sect
= cpu_to_be32(mdev
->ldev
->md
.md_size_sect
);
2727 buffer
->al_offset
= cpu_to_be32(mdev
->ldev
->md
.al_offset
);
2728 buffer
->al_nr_extents
= cpu_to_be32(mdev
->act_log
->nr_elements
);
2729 buffer
->bm_bytes_per_bit
= cpu_to_be32(BM_BLOCK_SIZE
);
2730 buffer
->device_uuid
= cpu_to_be64(mdev
->ldev
->md
.device_uuid
);
2732 buffer
->bm_offset
= cpu_to_be32(mdev
->ldev
->md
.bm_offset
);
2733 buffer
->la_peer_max_bio_size
= cpu_to_be32(mdev
->peer_max_bio_size
);
2735 D_ASSERT(drbd_md_ss__(mdev
, mdev
->ldev
) == mdev
->ldev
->md
.md_offset
);
2736 sector
= mdev
->ldev
->md
.md_offset
;
2738 if (drbd_md_sync_page_io(mdev
, mdev
->ldev
, sector
, WRITE
)) {
2739 /* this was a try anyways ... */
2740 dev_err(DEV
, "meta data update failed!\n");
2741 drbd_chk_io_error(mdev
, 1, true);
2744 /* Update mdev->ldev->md.la_size_sect,
2745 * since we updated it on metadata. */
2746 mdev
->ldev
->md
.la_size_sect
= drbd_get_capacity(mdev
->this_bdev
);
2748 mutex_unlock(&mdev
->md_io_mutex
);
2753 * drbd_md_read() - Reads in the meta data super block
2754 * @mdev: DRBD device.
2755 * @bdev: Device from which the meta data should be read in.
2757 * Return 0 (NO_ERROR) on success, and an enum drbd_ret_code in case
2758 * something goes wrong. Currently only: ERR_IO_MD_DISK, ERR_MD_INVALID.
2760 int drbd_md_read(struct drbd_conf
*mdev
, struct drbd_backing_dev
*bdev
)
2762 struct meta_data_on_disk
*buffer
;
2763 int i
, rv
= NO_ERROR
;
2765 if (!get_ldev_if_state(mdev
, D_ATTACHING
))
2766 return ERR_IO_MD_DISK
;
2768 mutex_lock(&mdev
->md_io_mutex
);
2769 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2771 if (drbd_md_sync_page_io(mdev
, bdev
, bdev
->md
.md_offset
, READ
)) {
2772 /* NOTE: can't do normal error processing here as this is
2773 called BEFORE disk is attached */
2774 dev_err(DEV
, "Error while reading metadata.\n");
2775 rv
= ERR_IO_MD_DISK
;
2779 if (buffer
->magic
!= cpu_to_be32(DRBD_MD_MAGIC
)) {
2780 dev_err(DEV
, "Error while reading metadata, magic not found.\n");
2781 rv
= ERR_MD_INVALID
;
2784 if (be32_to_cpu(buffer
->al_offset
) != bdev
->md
.al_offset
) {
2785 dev_err(DEV
, "unexpected al_offset: %d (expected %d)\n",
2786 be32_to_cpu(buffer
->al_offset
), bdev
->md
.al_offset
);
2787 rv
= ERR_MD_INVALID
;
2790 if (be32_to_cpu(buffer
->bm_offset
) != bdev
->md
.bm_offset
) {
2791 dev_err(DEV
, "unexpected bm_offset: %d (expected %d)\n",
2792 be32_to_cpu(buffer
->bm_offset
), bdev
->md
.bm_offset
);
2793 rv
= ERR_MD_INVALID
;
2796 if (be32_to_cpu(buffer
->md_size_sect
) != bdev
->md
.md_size_sect
) {
2797 dev_err(DEV
, "unexpected md_size: %u (expected %u)\n",
2798 be32_to_cpu(buffer
->md_size_sect
), bdev
->md
.md_size_sect
);
2799 rv
= ERR_MD_INVALID
;
2803 if (be32_to_cpu(buffer
->bm_bytes_per_bit
) != BM_BLOCK_SIZE
) {
2804 dev_err(DEV
, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
2805 be32_to_cpu(buffer
->bm_bytes_per_bit
), BM_BLOCK_SIZE
);
2806 rv
= ERR_MD_INVALID
;
2810 bdev
->md
.la_size_sect
= be64_to_cpu(buffer
->la_size
);
2811 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2812 bdev
->md
.uuid
[i
] = be64_to_cpu(buffer
->uuid
[i
]);
2813 bdev
->md
.flags
= be32_to_cpu(buffer
->flags
);
2814 bdev
->dc
.al_extents
= be32_to_cpu(buffer
->al_nr_extents
);
2815 bdev
->md
.device_uuid
= be64_to_cpu(buffer
->device_uuid
);
2817 spin_lock_irq(&mdev
->tconn
->req_lock
);
2818 if (mdev
->state
.conn
< C_CONNECTED
) {
2820 peer
= be32_to_cpu(buffer
->la_peer_max_bio_size
);
2821 peer
= max_t(int, peer
, DRBD_MAX_BIO_SIZE_SAFE
);
2822 mdev
->peer_max_bio_size
= peer
;
2824 spin_unlock_irq(&mdev
->tconn
->req_lock
);
2826 if (bdev
->dc
.al_extents
< 7)
2827 bdev
->dc
.al_extents
= 127;
2830 mutex_unlock(&mdev
->md_io_mutex
);
2837 * drbd_md_mark_dirty() - Mark meta data super block as dirty
2838 * @mdev: DRBD device.
2840 * Call this function if you change anything that should be written to
2841 * the meta-data super block. This function sets MD_DIRTY, and starts a
2842 * timer that ensures that within five seconds you have to call drbd_md_sync().
2845 void drbd_md_mark_dirty_(struct drbd_conf
*mdev
, unsigned int line
, const char *func
)
2847 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
)) {
2848 mod_timer(&mdev
->md_sync_timer
, jiffies
+ HZ
);
2849 mdev
->last_md_mark_dirty
.line
= line
;
2850 mdev
->last_md_mark_dirty
.func
= func
;
2854 void drbd_md_mark_dirty(struct drbd_conf
*mdev
)
2856 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
))
2857 mod_timer(&mdev
->md_sync_timer
, jiffies
+ 5*HZ
);
2861 static void drbd_uuid_move_history(struct drbd_conf
*mdev
) __must_hold(local
)
2865 for (i
= UI_HISTORY_START
; i
< UI_HISTORY_END
; i
++)
2866 mdev
->ldev
->md
.uuid
[i
+1] = mdev
->ldev
->md
.uuid
[i
];
2869 void _drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2871 if (idx
== UI_CURRENT
) {
2872 if (mdev
->state
.role
== R_PRIMARY
)
2877 drbd_set_ed_uuid(mdev
, val
);
2880 mdev
->ldev
->md
.uuid
[idx
] = val
;
2881 drbd_md_mark_dirty(mdev
);
2885 void drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2887 if (mdev
->ldev
->md
.uuid
[idx
]) {
2888 drbd_uuid_move_history(mdev
);
2889 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[idx
];
2891 _drbd_uuid_set(mdev
, idx
, val
);
2895 * drbd_uuid_new_current() - Creates a new current UUID
2896 * @mdev: DRBD device.
2898 * Creates a new current UUID, and rotates the old current UUID into
2899 * the bitmap slot. Causes an incremental resync upon next connect.
2901 void drbd_uuid_new_current(struct drbd_conf
*mdev
) __must_hold(local
)
2904 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2907 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2909 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = mdev
->ldev
->md
.uuid
[UI_CURRENT
];
2911 get_random_bytes(&val
, sizeof(u64
));
2912 _drbd_uuid_set(mdev
, UI_CURRENT
, val
);
2913 drbd_print_uuids(mdev
, "new current UUID");
2914 /* get it to stable storage _now_ */
2918 void drbd_uuid_set_bm(struct drbd_conf
*mdev
, u64 val
) __must_hold(local
)
2920 if (mdev
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && val
== 0)
2924 drbd_uuid_move_history(mdev
);
2925 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2926 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = 0;
2928 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2930 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2932 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = val
& ~((u64
)1);
2934 drbd_md_mark_dirty(mdev
);
2938 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2939 * @mdev: DRBD device.
2941 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
2943 int drbd_bmio_set_n_write(struct drbd_conf
*mdev
)
2947 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2948 drbd_md_set_flag(mdev
, MDF_FULL_SYNC
);
2950 drbd_bm_set_all(mdev
);
2952 rv
= drbd_bm_write(mdev
);
2955 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
2966 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2967 * @mdev: DRBD device.
2969 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
2971 int drbd_bmio_clear_n_write(struct drbd_conf
*mdev
)
2975 drbd_resume_al(mdev
);
2976 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2977 drbd_bm_clear_all(mdev
);
2978 rv
= drbd_bm_write(mdev
);
2985 static int w_bitmap_io(struct drbd_work
*w
, int unused
)
2987 struct bm_io_work
*work
= container_of(w
, struct bm_io_work
, w
);
2988 struct drbd_conf
*mdev
= w
->mdev
;
2991 D_ASSERT(atomic_read(&mdev
->ap_bio_cnt
) == 0);
2993 if (get_ldev(mdev
)) {
2994 drbd_bm_lock(mdev
, work
->why
, work
->flags
);
2995 rv
= work
->io_fn(mdev
);
2996 drbd_bm_unlock(mdev
);
3000 clear_bit_unlock(BITMAP_IO
, &mdev
->flags
);
3001 wake_up(&mdev
->misc_wait
);
3004 work
->done(mdev
, rv
);
3006 clear_bit(BITMAP_IO_QUEUED
, &mdev
->flags
);
3013 void drbd_ldev_destroy(struct drbd_conf
*mdev
)
3015 lc_destroy(mdev
->resync
);
3016 mdev
->resync
= NULL
;
3017 lc_destroy(mdev
->act_log
);
3018 mdev
->act_log
= NULL
;
3020 drbd_free_bc(mdev
->ldev
);
3021 mdev
->ldev
= NULL
;);
3023 clear_bit(GO_DISKLESS
, &mdev
->flags
);
3026 static int w_go_diskless(struct drbd_work
*w
, int unused
)
3028 struct drbd_conf
*mdev
= w
->mdev
;
3030 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3031 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
3032 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
3033 * the protected members anymore, though, so once put_ldev reaches zero
3034 * again, it will be safe to free them. */
3035 drbd_force_state(mdev
, NS(disk
, D_DISKLESS
));
3039 void drbd_go_diskless(struct drbd_conf
*mdev
)
3041 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3042 if (!test_and_set_bit(GO_DISKLESS
, &mdev
->flags
))
3043 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->go_diskless
);
3047 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3048 * @mdev: DRBD device.
3049 * @io_fn: IO callback to be called when bitmap IO is possible
3050 * @done: callback to be called after the bitmap IO was performed
3051 * @why: Descriptive text of the reason for doing the IO
3053 * While IO on the bitmap happens we freeze application IO thus we ensure
3054 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
3055 * called from worker context. It MUST NOT be used while a previous such
3056 * work is still pending!
3058 void drbd_queue_bitmap_io(struct drbd_conf
*mdev
,
3059 int (*io_fn
)(struct drbd_conf
*),
3060 void (*done
)(struct drbd_conf
*, int),
3061 char *why
, enum bm_flag flags
)
3063 D_ASSERT(current
== mdev
->tconn
->worker
.task
);
3065 D_ASSERT(!test_bit(BITMAP_IO_QUEUED
, &mdev
->flags
));
3066 D_ASSERT(!test_bit(BITMAP_IO
, &mdev
->flags
));
3067 D_ASSERT(list_empty(&mdev
->bm_io_work
.w
.list
));
3068 if (mdev
->bm_io_work
.why
)
3069 dev_err(DEV
, "FIXME going to queue '%s' but '%s' still pending?\n",
3070 why
, mdev
->bm_io_work
.why
);
3072 mdev
->bm_io_work
.io_fn
= io_fn
;
3073 mdev
->bm_io_work
.done
= done
;
3074 mdev
->bm_io_work
.why
= why
;
3075 mdev
->bm_io_work
.flags
= flags
;
3077 spin_lock_irq(&mdev
->tconn
->req_lock
);
3078 set_bit(BITMAP_IO
, &mdev
->flags
);
3079 if (atomic_read(&mdev
->ap_bio_cnt
) == 0) {
3080 if (!test_and_set_bit(BITMAP_IO_QUEUED
, &mdev
->flags
))
3081 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->bm_io_work
.w
);
3083 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3087 * drbd_bitmap_io() - Does an IO operation on the whole bitmap
3088 * @mdev: DRBD device.
3089 * @io_fn: IO callback to be called when bitmap IO is possible
3090 * @why: Descriptive text of the reason for doing the IO
3092 * freezes application IO while that the actual IO operations runs. This
3093 * functions MAY NOT be called from worker context.
3095 int drbd_bitmap_io(struct drbd_conf
*mdev
, int (*io_fn
)(struct drbd_conf
*),
3096 char *why
, enum bm_flag flags
)
3100 D_ASSERT(current
!= mdev
->tconn
->worker
.task
);
3102 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3103 drbd_suspend_io(mdev
);
3105 drbd_bm_lock(mdev
, why
, flags
);
3107 drbd_bm_unlock(mdev
);
3109 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3110 drbd_resume_io(mdev
);
3115 void drbd_md_set_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3117 if ((mdev
->ldev
->md
.flags
& flag
) != flag
) {
3118 drbd_md_mark_dirty(mdev
);
3119 mdev
->ldev
->md
.flags
|= flag
;
3123 void drbd_md_clear_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3125 if ((mdev
->ldev
->md
.flags
& flag
) != 0) {
3126 drbd_md_mark_dirty(mdev
);
3127 mdev
->ldev
->md
.flags
&= ~flag
;
3130 int drbd_md_test_flag(struct drbd_backing_dev
*bdev
, int flag
)
3132 return (bdev
->md
.flags
& flag
) != 0;
3135 static void md_sync_timer_fn(unsigned long data
)
3137 struct drbd_conf
*mdev
= (struct drbd_conf
*) data
;
3139 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &mdev
->md_sync_work
);
3142 static int w_md_sync(struct drbd_work
*w
, int unused
)
3144 struct drbd_conf
*mdev
= w
->mdev
;
3146 dev_warn(DEV
, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3148 dev_warn(DEV
, "last md_mark_dirty: %s:%u\n",
3149 mdev
->last_md_mark_dirty
.func
, mdev
->last_md_mark_dirty
.line
);
3155 const char *cmdname(enum drbd_packet cmd
)
3157 /* THINK may need to become several global tables
3158 * when we want to support more than
3159 * one PRO_VERSION */
3160 static const char *cmdnames
[] = {
3162 [P_DATA_REPLY
] = "DataReply",
3163 [P_RS_DATA_REPLY
] = "RSDataReply",
3164 [P_BARRIER
] = "Barrier",
3165 [P_BITMAP
] = "ReportBitMap",
3166 [P_BECOME_SYNC_TARGET
] = "BecomeSyncTarget",
3167 [P_BECOME_SYNC_SOURCE
] = "BecomeSyncSource",
3168 [P_UNPLUG_REMOTE
] = "UnplugRemote",
3169 [P_DATA_REQUEST
] = "DataRequest",
3170 [P_RS_DATA_REQUEST
] = "RSDataRequest",
3171 [P_SYNC_PARAM
] = "SyncParam",
3172 [P_SYNC_PARAM89
] = "SyncParam89",
3173 [P_PROTOCOL
] = "ReportProtocol",
3174 [P_UUIDS
] = "ReportUUIDs",
3175 [P_SIZES
] = "ReportSizes",
3176 [P_STATE
] = "ReportState",
3177 [P_SYNC_UUID
] = "ReportSyncUUID",
3178 [P_AUTH_CHALLENGE
] = "AuthChallenge",
3179 [P_AUTH_RESPONSE
] = "AuthResponse",
3181 [P_PING_ACK
] = "PingAck",
3182 [P_RECV_ACK
] = "RecvAck",
3183 [P_WRITE_ACK
] = "WriteAck",
3184 [P_RS_WRITE_ACK
] = "RSWriteAck",
3185 [P_DISCARD_WRITE
] = "DiscardWrite",
3186 [P_NEG_ACK
] = "NegAck",
3187 [P_NEG_DREPLY
] = "NegDReply",
3188 [P_NEG_RS_DREPLY
] = "NegRSDReply",
3189 [P_BARRIER_ACK
] = "BarrierAck",
3190 [P_STATE_CHG_REQ
] = "StateChgRequest",
3191 [P_STATE_CHG_REPLY
] = "StateChgReply",
3192 [P_OV_REQUEST
] = "OVRequest",
3193 [P_OV_REPLY
] = "OVReply",
3194 [P_OV_RESULT
] = "OVResult",
3195 [P_CSUM_RS_REQUEST
] = "CsumRSRequest",
3196 [P_RS_IS_IN_SYNC
] = "CsumRSIsInSync",
3197 [P_COMPRESSED_BITMAP
] = "CBitmap",
3198 [P_DELAY_PROBE
] = "DelayProbe",
3199 [P_OUT_OF_SYNC
] = "OutOfSync",
3200 [P_RETRY_WRITE
] = "RetryWrite",
3203 if (cmd
== P_INITIAL_META
)
3204 return "InitialMeta";
3205 if (cmd
== P_INITIAL_DATA
)
3206 return "InitialData";
3207 if (cmd
== P_CONNECTION_FEATURES
)
3208 return "ConnectionFeatures";
3209 if (cmd
>= ARRAY_SIZE(cmdnames
))
3211 return cmdnames
[cmd
];
3215 * drbd_wait_misc - wait for a request to make progress
3216 * @mdev: device associated with the request
3217 * @i: the struct drbd_interval embedded in struct drbd_request or
3218 * struct drbd_peer_request
3220 int drbd_wait_misc(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
3222 struct net_conf
*net_conf
= mdev
->tconn
->net_conf
;
3228 timeout
= MAX_SCHEDULE_TIMEOUT
;
3229 if (net_conf
->ko_count
)
3230 timeout
= net_conf
->timeout
* HZ
/ 10 * net_conf
->ko_count
;
3232 /* Indicate to wake up mdev->misc_wait on progress. */
3234 prepare_to_wait(&mdev
->misc_wait
, &wait
, TASK_INTERRUPTIBLE
);
3235 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3236 timeout
= schedule_timeout(timeout
);
3237 finish_wait(&mdev
->misc_wait
, &wait
);
3238 spin_lock_irq(&mdev
->tconn
->req_lock
);
3239 if (!timeout
|| mdev
->state
.conn
< C_CONNECTED
)
3241 if (signal_pending(current
))
3242 return -ERESTARTSYS
;
3246 #ifdef CONFIG_DRBD_FAULT_INJECTION
3247 /* Fault insertion support including random number generator shamelessly
3248 * stolen from kernel/rcutorture.c */
3249 struct fault_random_state
{
3250 unsigned long state
;
3251 unsigned long count
;
3254 #define FAULT_RANDOM_MULT 39916801 /* prime */
3255 #define FAULT_RANDOM_ADD 479001701 /* prime */
3256 #define FAULT_RANDOM_REFRESH 10000
3259 * Crude but fast random-number generator. Uses a linear congruential
3260 * generator, with occasional help from get_random_bytes().
3262 static unsigned long
3263 _drbd_fault_random(struct fault_random_state
*rsp
)
3267 if (!rsp
->count
--) {
3268 get_random_bytes(&refresh
, sizeof(refresh
));
3269 rsp
->state
+= refresh
;
3270 rsp
->count
= FAULT_RANDOM_REFRESH
;
3272 rsp
->state
= rsp
->state
* FAULT_RANDOM_MULT
+ FAULT_RANDOM_ADD
;
3273 return swahw32(rsp
->state
);
3277 _drbd_fault_str(unsigned int type
) {
3278 static char *_faults
[] = {
3279 [DRBD_FAULT_MD_WR
] = "Meta-data write",
3280 [DRBD_FAULT_MD_RD
] = "Meta-data read",
3281 [DRBD_FAULT_RS_WR
] = "Resync write",
3282 [DRBD_FAULT_RS_RD
] = "Resync read",
3283 [DRBD_FAULT_DT_WR
] = "Data write",
3284 [DRBD_FAULT_DT_RD
] = "Data read",
3285 [DRBD_FAULT_DT_RA
] = "Data read ahead",
3286 [DRBD_FAULT_BM_ALLOC
] = "BM allocation",
3287 [DRBD_FAULT_AL_EE
] = "EE allocation",
3288 [DRBD_FAULT_RECEIVE
] = "receive data corruption",
3291 return (type
< DRBD_FAULT_MAX
) ? _faults
[type
] : "**Unknown**";
3295 _drbd_insert_fault(struct drbd_conf
*mdev
, unsigned int type
)
3297 static struct fault_random_state rrs
= {0, 0};
3299 unsigned int ret
= (
3301 ((1 << mdev_to_minor(mdev
)) & fault_devs
) != 0) &&
3302 (((_drbd_fault_random(&rrs
) % 100) + 1) <= fault_rate
));
3307 if (__ratelimit(&drbd_ratelimit_state
))
3308 dev_warn(DEV
, "***Simulating %s failure\n",
3309 _drbd_fault_str(type
));
3316 const char *drbd_buildtag(void)
3318 /* DRBD built from external sources has here a reference to the
3319 git hash of the source code. */
3321 static char buildtag
[38] = "\0uilt-in";
3323 if (buildtag
[0] == 0) {
3324 #ifdef CONFIG_MODULES
3325 if (THIS_MODULE
!= NULL
)
3326 sprintf(buildtag
, "srcversion: %-24s", THIS_MODULE
->srcversion
);
3335 module_init(drbd_init
)
3336 module_exit(drbd_cleanup
)
3338 EXPORT_SYMBOL(drbd_conn_str
);
3339 EXPORT_SYMBOL(drbd_role_str
);
3340 EXPORT_SYMBOL(drbd_disk_str
);
3341 EXPORT_SYMBOL(drbd_set_st_err_str
);