4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
29 #include <linux/module.h>
30 #include <linux/drbd.h>
31 #include <asm/uaccess.h>
32 #include <asm/types.h>
34 #include <linux/ctype.h>
35 #include <linux/mutex.h>
37 #include <linux/file.h>
38 #include <linux/proc_fs.h>
39 #include <linux/init.h>
41 #include <linux/memcontrol.h>
42 #include <linux/mm_inline.h>
43 #include <linux/slab.h>
44 #include <linux/random.h>
45 #include <linux/reboot.h>
46 #include <linux/notifier.h>
47 #include <linux/kthread.h>
49 #define __KERNEL_SYSCALLS__
50 #include <linux/unistd.h>
51 #include <linux/vmalloc.h>
53 #include <linux/drbd_limits.h>
55 #include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
59 static DEFINE_MUTEX(drbd_main_mutex
);
60 int drbdd_init(struct drbd_thread
*);
61 int drbd_worker(struct drbd_thread
*);
62 int drbd_asender(struct drbd_thread
*);
65 static int drbd_open(struct block_device
*bdev
, fmode_t mode
);
66 static int drbd_release(struct gendisk
*gd
, fmode_t mode
);
67 static int w_md_sync(struct drbd_work
*w
, int unused
);
68 static void md_sync_timer_fn(unsigned long data
);
69 static int w_bitmap_io(struct drbd_work
*w
, int unused
);
70 static int w_go_diskless(struct drbd_work
*w
, int unused
);
72 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
73 "Lars Ellenberg <lars@linbit.com>");
74 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION
);
75 MODULE_VERSION(REL_VERSION
);
76 MODULE_LICENSE("GPL");
77 MODULE_PARM_DESC(minor_count
, "Approximate number of drbd devices ("
78 __stringify(DRBD_MINOR_COUNT_MIN
) "-" __stringify(DRBD_MINOR_COUNT_MAX
) ")");
79 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR
);
81 #include <linux/moduleparam.h>
82 /* allow_open_on_secondary */
83 MODULE_PARM_DESC(allow_oos
, "DONT USE!");
84 /* thanks to these macros, if compiled into the kernel (not-module),
85 * this becomes the boot parameter drbd.minor_count */
86 module_param(minor_count
, uint
, 0444);
87 module_param(disable_sendpage
, bool, 0644);
88 module_param(allow_oos
, bool, 0);
89 module_param(proc_details
, int, 0644);
91 #ifdef CONFIG_DRBD_FAULT_INJECTION
94 static int fault_count
;
96 /* bitmap of enabled faults */
97 module_param(enable_faults
, int, 0664);
98 /* fault rate % value - applies to all enabled faults */
99 module_param(fault_rate
, int, 0664);
100 /* count of faults inserted */
101 module_param(fault_count
, int, 0664);
102 /* bitmap of devices to insert faults on */
103 module_param(fault_devs
, int, 0644);
106 /* module parameter, defined */
107 unsigned int minor_count
= DRBD_MINOR_COUNT_DEF
;
108 int disable_sendpage
;
110 int proc_details
; /* Detail level in proc drbd*/
112 /* Module parameter for setting the user mode helper program
113 * to run. Default is /sbin/drbdadm */
114 char usermode_helper
[80] = "/sbin/drbdadm";
116 module_param_string(usermode_helper
, usermode_helper
, sizeof(usermode_helper
), 0644);
118 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
119 * as member "struct gendisk *vdisk;"
122 struct list_head drbd_tconns
; /* list of struct drbd_tconn */
123 DECLARE_RWSEM(drbd_cfg_rwsem
);
125 struct kmem_cache
*drbd_request_cache
;
126 struct kmem_cache
*drbd_ee_cache
; /* peer requests */
127 struct kmem_cache
*drbd_bm_ext_cache
; /* bitmap extents */
128 struct kmem_cache
*drbd_al_ext_cache
; /* activity log extents */
129 mempool_t
*drbd_request_mempool
;
130 mempool_t
*drbd_ee_mempool
;
131 mempool_t
*drbd_md_io_page_pool
;
132 struct bio_set
*drbd_md_io_bio_set
;
134 /* I do not use a standard mempool, because:
135 1) I want to hand out the pre-allocated objects first.
136 2) I want to be able to interrupt sleeping allocation with a signal.
137 Note: This is a single linked list, the next pointer is the private
138 member of struct page.
140 struct page
*drbd_pp_pool
;
141 spinlock_t drbd_pp_lock
;
143 wait_queue_head_t drbd_pp_wait
;
145 DEFINE_RATELIMIT_STATE(drbd_ratelimit_state
, 5 * HZ
, 5);
147 static const struct block_device_operations drbd_ops
= {
148 .owner
= THIS_MODULE
,
150 .release
= drbd_release
,
153 static void bio_destructor_drbd(struct bio
*bio
)
155 bio_free(bio
, drbd_md_io_bio_set
);
158 struct bio
*bio_alloc_drbd(gfp_t gfp_mask
)
162 if (!drbd_md_io_bio_set
)
163 return bio_alloc(gfp_mask
, 1);
165 bio
= bio_alloc_bioset(gfp_mask
, 1, drbd_md_io_bio_set
);
168 bio
->bi_destructor
= bio_destructor_drbd
;
173 /* When checking with sparse, and this is an inline function, sparse will
174 give tons of false positives. When this is a real functions sparse works.
176 int _get_ldev_if_state(struct drbd_conf
*mdev
, enum drbd_disk_state mins
)
180 atomic_inc(&mdev
->local_cnt
);
181 io_allowed
= (mdev
->state
.disk
>= mins
);
183 if (atomic_dec_and_test(&mdev
->local_cnt
))
184 wake_up(&mdev
->misc_wait
);
192 * DOC: The transfer log
194 * The transfer log is a single linked list of &struct drbd_tl_epoch objects.
195 * mdev->tconn->newest_tle points to the head, mdev->tconn->oldest_tle points to the tail
196 * of the list. There is always at least one &struct drbd_tl_epoch object.
198 * Each &struct drbd_tl_epoch has a circular double linked list of requests
201 static int tl_init(struct drbd_tconn
*tconn
)
203 struct drbd_tl_epoch
*b
;
205 /* during device minor initialization, we may well use GFP_KERNEL */
206 b
= kmalloc(sizeof(struct drbd_tl_epoch
), GFP_KERNEL
);
209 INIT_LIST_HEAD(&b
->requests
);
210 INIT_LIST_HEAD(&b
->w
.list
);
214 b
->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
216 tconn
->oldest_tle
= b
;
217 tconn
->newest_tle
= b
;
218 INIT_LIST_HEAD(&tconn
->out_of_sequence_requests
);
223 static void tl_cleanup(struct drbd_tconn
*tconn
)
225 if (tconn
->oldest_tle
!= tconn
->newest_tle
)
226 conn_err(tconn
, "ASSERT FAILED: oldest_tle == newest_tle\n");
227 if (!list_empty(&tconn
->out_of_sequence_requests
))
228 conn_err(tconn
, "ASSERT FAILED: list_empty(out_of_sequence_requests)\n");
229 kfree(tconn
->oldest_tle
);
230 tconn
->oldest_tle
= NULL
;
231 kfree(tconn
->unused_spare_tle
);
232 tconn
->unused_spare_tle
= NULL
;
236 * _tl_add_barrier() - Adds a barrier to the transfer log
237 * @mdev: DRBD device.
238 * @new: Barrier to be added before the current head of the TL.
240 * The caller must hold the req_lock.
242 void _tl_add_barrier(struct drbd_tconn
*tconn
, struct drbd_tl_epoch
*new)
244 struct drbd_tl_epoch
*newest_before
;
246 INIT_LIST_HEAD(&new->requests
);
247 INIT_LIST_HEAD(&new->w
.list
);
248 new->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
252 newest_before
= tconn
->newest_tle
;
253 /* never send a barrier number == 0, because that is special-cased
254 * when using TCQ for our write ordering code */
255 new->br_number
= (newest_before
->br_number
+1) ?: 1;
256 if (tconn
->newest_tle
!= new) {
257 tconn
->newest_tle
->next
= new;
258 tconn
->newest_tle
= new;
263 * tl_release() - Free or recycle the oldest &struct drbd_tl_epoch object of the TL
264 * @mdev: DRBD device.
265 * @barrier_nr: Expected identifier of the DRBD write barrier packet.
266 * @set_size: Expected number of requests before that barrier.
268 * In case the passed barrier_nr or set_size does not match the oldest
269 * &struct drbd_tl_epoch objects this function will cause a termination
272 void tl_release(struct drbd_tconn
*tconn
, unsigned int barrier_nr
,
273 unsigned int set_size
)
275 struct drbd_conf
*mdev
;
276 struct drbd_tl_epoch
*b
, *nob
; /* next old barrier */
277 struct list_head
*le
, *tle
;
278 struct drbd_request
*r
;
280 spin_lock_irq(&tconn
->req_lock
);
282 b
= tconn
->oldest_tle
;
284 /* first some paranoia code */
286 conn_err(tconn
, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
290 if (b
->br_number
!= barrier_nr
) {
291 conn_err(tconn
, "BAD! BarrierAck #%u received, expected #%u!\n",
292 barrier_nr
, b
->br_number
);
295 if (b
->n_writes
!= set_size
) {
296 conn_err(tconn
, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
297 barrier_nr
, set_size
, b
->n_writes
);
301 /* Clean up list of requests processed during current epoch */
302 list_for_each_safe(le
, tle
, &b
->requests
) {
303 r
= list_entry(le
, struct drbd_request
, tl_requests
);
304 _req_mod(r
, BARRIER_ACKED
);
306 /* There could be requests on the list waiting for completion
307 of the write to the local disk. To avoid corruptions of
308 slab's data structures we have to remove the lists head.
310 Also there could have been a barrier ack out of sequence, overtaking
311 the write acks - which would be a bug and violating write ordering.
312 To not deadlock in case we lose connection while such requests are
313 still pending, we need some way to find them for the
314 _req_mode(CONNECTION_LOST_WHILE_PENDING).
316 These have been list_move'd to the out_of_sequence_requests list in
317 _req_mod(, BARRIER_ACKED) above.
319 list_del_init(&b
->requests
);
323 if (test_and_clear_bit(CREATE_BARRIER
, &mdev
->flags
)) {
324 _tl_add_barrier(tconn
, b
);
326 tconn
->oldest_tle
= nob
;
327 /* if nob == NULL b was the only barrier, and becomes the new
328 barrier. Therefore tconn->oldest_tle points already to b */
330 D_ASSERT(nob
!= NULL
);
331 tconn
->oldest_tle
= nob
;
335 spin_unlock_irq(&tconn
->req_lock
);
336 dec_ap_pending(mdev
);
341 spin_unlock_irq(&tconn
->req_lock
);
342 conn_request_state(tconn
, NS(conn
, C_PROTOCOL_ERROR
), CS_HARD
);
347 * _tl_restart() - Walks the transfer log, and applies an action to all requests
348 * @mdev: DRBD device.
349 * @what: The action/event to perform with all request objects
351 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
352 * RESTART_FROZEN_DISK_IO.
354 void _tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
356 struct drbd_tl_epoch
*b
, *tmp
, **pn
;
357 struct list_head
*le
, *tle
, carry_reads
;
358 struct drbd_request
*req
;
359 int rv
, n_writes
, n_reads
;
361 b
= tconn
->oldest_tle
;
362 pn
= &tconn
->oldest_tle
;
366 INIT_LIST_HEAD(&carry_reads
);
367 list_for_each_safe(le
, tle
, &b
->requests
) {
368 req
= list_entry(le
, struct drbd_request
, tl_requests
);
369 rv
= _req_mod(req
, what
);
371 n_writes
+= (rv
& MR_WRITE
) >> MR_WRITE_SHIFT
;
372 n_reads
+= (rv
& MR_READ
) >> MR_READ_SHIFT
;
377 if (what
== RESEND
) {
378 b
->n_writes
= n_writes
;
379 if (b
->w
.cb
== NULL
) {
380 b
->w
.cb
= w_send_barrier
;
381 inc_ap_pending(b
->w
.mdev
);
382 set_bit(CREATE_BARRIER
, &b
->w
.mdev
->flags
);
385 drbd_queue_work(&tconn
->data
.work
, &b
->w
);
390 list_add(&carry_reads
, &b
->requests
);
391 /* there could still be requests on that ring list,
392 * in case local io is still pending */
393 list_del(&b
->requests
);
395 /* dec_ap_pending corresponding to queue_barrier.
396 * the newest barrier may not have been queued yet,
397 * in which case w.cb is still NULL. */
399 dec_ap_pending(b
->w
.mdev
);
401 if (b
== tconn
->newest_tle
) {
402 /* recycle, but reinit! */
404 conn_err(tconn
, "ASSERT FAILED tmp == NULL");
405 INIT_LIST_HEAD(&b
->requests
);
406 list_splice(&carry_reads
, &b
->requests
);
407 INIT_LIST_HEAD(&b
->w
.list
);
409 b
->br_number
= net_random();
419 list_splice(&carry_reads
, &b
->requests
);
425 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
426 * @mdev: DRBD device.
428 * This is called after the connection to the peer was lost. The storage covered
429 * by the requests on the transfer gets marked as our of sync. Called from the
430 * receiver thread and the worker thread.
432 void tl_clear(struct drbd_tconn
*tconn
)
434 struct drbd_conf
*mdev
;
435 struct list_head
*le
, *tle
;
436 struct drbd_request
*r
;
439 spin_lock_irq(&tconn
->req_lock
);
441 _tl_restart(tconn
, CONNECTION_LOST_WHILE_PENDING
);
443 /* we expect this list to be empty. */
444 if (!list_empty(&tconn
->out_of_sequence_requests
))
445 conn_err(tconn
, "ASSERT FAILED list_empty(&out_of_sequence_requests)\n");
447 /* but just in case, clean it up anyways! */
448 list_for_each_safe(le
, tle
, &tconn
->out_of_sequence_requests
) {
449 r
= list_entry(le
, struct drbd_request
, tl_requests
);
450 /* It would be nice to complete outside of spinlock.
451 * But this is easier for now. */
452 _req_mod(r
, CONNECTION_LOST_WHILE_PENDING
);
455 /* ensure bit indicating barrier is required is clear */
457 idr_for_each_entry(&tconn
->volumes
, mdev
, vnr
)
458 clear_bit(CREATE_BARRIER
, &mdev
->flags
);
461 spin_unlock_irq(&tconn
->req_lock
);
464 void tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
466 spin_lock_irq(&tconn
->req_lock
);
467 _tl_restart(tconn
, what
);
468 spin_unlock_irq(&tconn
->req_lock
);
471 static int drbd_thread_setup(void *arg
)
473 struct drbd_thread
*thi
= (struct drbd_thread
*) arg
;
474 struct drbd_tconn
*tconn
= thi
->tconn
;
478 snprintf(current
->comm
, sizeof(current
->comm
), "drbd_%c_%s",
479 thi
->name
[0], thi
->tconn
->name
);
482 retval
= thi
->function(thi
);
484 spin_lock_irqsave(&thi
->t_lock
, flags
);
486 /* if the receiver has been "EXITING", the last thing it did
487 * was set the conn state to "StandAlone",
488 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
489 * and receiver thread will be "started".
490 * drbd_thread_start needs to set "RESTARTING" in that case.
491 * t_state check and assignment needs to be within the same spinlock,
492 * so either thread_start sees EXITING, and can remap to RESTARTING,
493 * or thread_start see NONE, and can proceed as normal.
496 if (thi
->t_state
== RESTARTING
) {
497 conn_info(tconn
, "Restarting %s thread\n", thi
->name
);
498 thi
->t_state
= RUNNING
;
499 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
506 complete(&thi
->stop
);
507 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
509 conn_info(tconn
, "Terminating %s\n", current
->comm
);
511 /* Release mod reference taken when thread was started */
512 module_put(THIS_MODULE
);
516 static void drbd_thread_init(struct drbd_tconn
*tconn
, struct drbd_thread
*thi
,
517 int (*func
) (struct drbd_thread
*), char *name
)
519 spin_lock_init(&thi
->t_lock
);
522 thi
->function
= func
;
524 strncpy(thi
->name
, name
, ARRAY_SIZE(thi
->name
));
527 int drbd_thread_start(struct drbd_thread
*thi
)
529 struct drbd_tconn
*tconn
= thi
->tconn
;
530 struct task_struct
*nt
;
533 /* is used from state engine doing drbd_thread_stop_nowait,
534 * while holding the req lock irqsave */
535 spin_lock_irqsave(&thi
->t_lock
, flags
);
537 switch (thi
->t_state
) {
539 conn_info(tconn
, "Starting %s thread (from %s [%d])\n",
540 thi
->name
, current
->comm
, current
->pid
);
542 /* Get ref on module for thread - this is released when thread exits */
543 if (!try_module_get(THIS_MODULE
)) {
544 conn_err(tconn
, "Failed to get module reference in drbd_thread_start\n");
545 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
549 init_completion(&thi
->stop
);
550 thi
->reset_cpu_mask
= 1;
551 thi
->t_state
= RUNNING
;
552 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
553 flush_signals(current
); /* otherw. may get -ERESTARTNOINTR */
555 nt
= kthread_create(drbd_thread_setup
, (void *) thi
,
556 "drbd_%c_%s", thi
->name
[0], thi
->tconn
->name
);
559 conn_err(tconn
, "Couldn't start thread\n");
561 module_put(THIS_MODULE
);
564 spin_lock_irqsave(&thi
->t_lock
, flags
);
566 thi
->t_state
= RUNNING
;
567 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
571 thi
->t_state
= RESTARTING
;
572 conn_info(tconn
, "Restarting %s thread (from %s [%d])\n",
573 thi
->name
, current
->comm
, current
->pid
);
578 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
586 void _drbd_thread_stop(struct drbd_thread
*thi
, int restart
, int wait
)
590 enum drbd_thread_state ns
= restart
? RESTARTING
: EXITING
;
592 /* may be called from state engine, holding the req lock irqsave */
593 spin_lock_irqsave(&thi
->t_lock
, flags
);
595 if (thi
->t_state
== NONE
) {
596 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
598 drbd_thread_start(thi
);
602 if (thi
->t_state
!= ns
) {
603 if (thi
->task
== NULL
) {
604 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
610 init_completion(&thi
->stop
);
611 if (thi
->task
!= current
)
612 force_sig(DRBD_SIGKILL
, thi
->task
);
615 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
618 wait_for_completion(&thi
->stop
);
621 static struct drbd_thread
*drbd_task_to_thread(struct drbd_tconn
*tconn
, struct task_struct
*task
)
623 struct drbd_thread
*thi
=
624 task
== tconn
->receiver
.task
? &tconn
->receiver
:
625 task
== tconn
->asender
.task
? &tconn
->asender
:
626 task
== tconn
->worker
.task
? &tconn
->worker
: NULL
;
631 char *drbd_task_to_thread_name(struct drbd_tconn
*tconn
, struct task_struct
*task
)
633 struct drbd_thread
*thi
= drbd_task_to_thread(tconn
, task
);
634 return thi
? thi
->name
: task
->comm
;
637 int conn_lowest_minor(struct drbd_tconn
*tconn
)
639 struct drbd_conf
*mdev
;
643 mdev
= idr_get_next(&tconn
->volumes
, &vnr
);
644 m
= mdev
? mdev_to_minor(mdev
) : -1;
652 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
653 * @mdev: DRBD device.
655 * Forces all threads of a device onto the same CPU. This is beneficial for
656 * DRBD's performance. May be overwritten by user's configuration.
658 void drbd_calc_cpu_mask(struct drbd_tconn
*tconn
)
663 if (cpumask_weight(tconn
->cpu_mask
))
666 ord
= conn_lowest_minor(tconn
) % cpumask_weight(cpu_online_mask
);
667 for_each_online_cpu(cpu
) {
669 cpumask_set_cpu(cpu
, tconn
->cpu_mask
);
673 /* should not be reached */
674 cpumask_setall(tconn
->cpu_mask
);
678 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
679 * @mdev: DRBD device.
680 * @thi: drbd_thread object
682 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
685 void drbd_thread_current_set_cpu(struct drbd_thread
*thi
)
687 struct task_struct
*p
= current
;
689 if (!thi
->reset_cpu_mask
)
691 thi
->reset_cpu_mask
= 0;
692 set_cpus_allowed_ptr(p
, thi
->tconn
->cpu_mask
);
697 * drbd_header_size - size of a packet header
699 * The header size is a multiple of 8, so any payload following the header is
700 * word aligned on 64-bit architectures. (The bitmap send and receive code
703 unsigned int drbd_header_size(struct drbd_tconn
*tconn
)
705 if (tconn
->agreed_pro_version
>= 100) {
706 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header100
), 8));
707 return sizeof(struct p_header100
);
709 BUILD_BUG_ON(sizeof(struct p_header80
) !=
710 sizeof(struct p_header95
));
711 BUILD_BUG_ON(!IS_ALIGNED(sizeof(struct p_header80
), 8));
712 return sizeof(struct p_header80
);
716 static unsigned int prepare_header80(struct p_header80
*h
, enum drbd_packet cmd
, int size
)
718 h
->magic
= cpu_to_be32(DRBD_MAGIC
);
719 h
->command
= cpu_to_be16(cmd
);
720 h
->length
= cpu_to_be16(size
);
721 return sizeof(struct p_header80
);
724 static unsigned int prepare_header95(struct p_header95
*h
, enum drbd_packet cmd
, int size
)
726 h
->magic
= cpu_to_be16(DRBD_MAGIC_BIG
);
727 h
->command
= cpu_to_be16(cmd
);
728 h
->length
= cpu_to_be32(size
);
729 return sizeof(struct p_header95
);
732 static unsigned int prepare_header100(struct p_header100
*h
, enum drbd_packet cmd
,
735 h
->magic
= cpu_to_be32(DRBD_MAGIC_100
);
736 h
->volume
= cpu_to_be16(vnr
);
737 h
->command
= cpu_to_be16(cmd
);
738 h
->length
= cpu_to_be32(size
);
740 return sizeof(struct p_header100
);
743 static unsigned int prepare_header(struct drbd_tconn
*tconn
, int vnr
,
744 void *buffer
, enum drbd_packet cmd
, int size
)
746 if (tconn
->agreed_pro_version
>= 100)
747 return prepare_header100(buffer
, cmd
, size
, vnr
);
748 else if (tconn
->agreed_pro_version
>= 95 &&
749 size
> DRBD_MAX_SIZE_H80_PACKET
)
750 return prepare_header95(buffer
, cmd
, size
);
752 return prepare_header80(buffer
, cmd
, size
);
755 void *conn_prepare_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
)
757 mutex_lock(&sock
->mutex
);
759 mutex_unlock(&sock
->mutex
);
762 return sock
->sbuf
+ drbd_header_size(tconn
);
765 void *drbd_prepare_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
)
767 return conn_prepare_command(mdev
->tconn
, sock
);
770 static int __send_command(struct drbd_tconn
*tconn
, int vnr
,
771 struct drbd_socket
*sock
, enum drbd_packet cmd
,
772 unsigned int header_size
, void *data
,
779 * Called with @data == NULL and the size of the data blocks in @size
780 * for commands that send data blocks. For those commands, omit the
781 * MSG_MORE flag: this will increase the likelihood that data blocks
782 * which are page aligned on the sender will end up page aligned on the
785 msg_flags
= data
? MSG_MORE
: 0;
787 header_size
+= prepare_header(tconn
, vnr
, sock
->sbuf
, cmd
,
789 err
= drbd_send_all(tconn
, sock
->socket
, sock
->sbuf
, header_size
,
792 err
= drbd_send_all(tconn
, sock
->socket
, data
, size
, 0);
796 int conn_send_command(struct drbd_tconn
*tconn
, struct drbd_socket
*sock
,
797 enum drbd_packet cmd
, unsigned int header_size
,
798 void *data
, unsigned int size
)
802 err
= __send_command(tconn
, 0, sock
, cmd
, header_size
, data
, size
);
803 mutex_unlock(&sock
->mutex
);
807 int drbd_send_command(struct drbd_conf
*mdev
, struct drbd_socket
*sock
,
808 enum drbd_packet cmd
, unsigned int header_size
,
809 void *data
, unsigned int size
)
813 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, header_size
,
815 mutex_unlock(&sock
->mutex
);
819 int drbd_send_ping(struct drbd_tconn
*tconn
)
821 struct drbd_socket
*sock
;
824 if (!conn_prepare_command(tconn
, sock
))
826 return conn_send_command(tconn
, sock
, P_PING
, 0, NULL
, 0);
829 int drbd_send_ping_ack(struct drbd_tconn
*tconn
)
831 struct drbd_socket
*sock
;
834 if (!conn_prepare_command(tconn
, sock
))
836 return conn_send_command(tconn
, sock
, P_PING_ACK
, 0, NULL
, 0);
839 int drbd_send_sync_param(struct drbd_conf
*mdev
)
841 struct drbd_socket
*sock
;
842 struct p_rs_param_95
*p
;
844 const int apv
= mdev
->tconn
->agreed_pro_version
;
845 enum drbd_packet cmd
;
848 sock
= &mdev
->tconn
->data
;
849 p
= drbd_prepare_command(mdev
, sock
);
854 nc
= rcu_dereference(mdev
->tconn
->net_conf
);
856 size
= apv
<= 87 ? sizeof(struct p_rs_param
)
857 : apv
== 88 ? sizeof(struct p_rs_param
)
858 + strlen(nc
->verify_alg
) + 1
859 : apv
<= 94 ? sizeof(struct p_rs_param_89
)
860 : /* apv >= 95 */ sizeof(struct p_rs_param_95
);
862 cmd
= apv
>= 89 ? P_SYNC_PARAM89
: P_SYNC_PARAM
;
864 /* initialize verify_alg and csums_alg */
865 memset(p
->verify_alg
, 0, 2 * SHARED_SECRET_MAX
);
867 if (get_ldev(mdev
)) {
868 p
->rate
= cpu_to_be32(mdev
->ldev
->dc
.resync_rate
);
869 p
->c_plan_ahead
= cpu_to_be32(mdev
->ldev
->dc
.c_plan_ahead
);
870 p
->c_delay_target
= cpu_to_be32(mdev
->ldev
->dc
.c_delay_target
);
871 p
->c_fill_target
= cpu_to_be32(mdev
->ldev
->dc
.c_fill_target
);
872 p
->c_max_rate
= cpu_to_be32(mdev
->ldev
->dc
.c_max_rate
);
875 p
->rate
= cpu_to_be32(DRBD_RATE_DEF
);
876 p
->c_plan_ahead
= cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF
);
877 p
->c_delay_target
= cpu_to_be32(DRBD_C_DELAY_TARGET_DEF
);
878 p
->c_fill_target
= cpu_to_be32(DRBD_C_FILL_TARGET_DEF
);
879 p
->c_max_rate
= cpu_to_be32(DRBD_C_MAX_RATE_DEF
);
883 strcpy(p
->verify_alg
, nc
->verify_alg
);
885 strcpy(p
->csums_alg
, nc
->csums_alg
);
888 return drbd_send_command(mdev
, sock
, cmd
, size
, NULL
, 0);
891 int drbd_send_protocol(struct drbd_tconn
*tconn
)
893 struct drbd_socket
*sock
;
894 struct p_protocol
*p
;
899 p
= conn_prepare_command(tconn
, sock
);
904 nc
= rcu_dereference(tconn
->net_conf
);
906 if (nc
->dry_run
&& tconn
->agreed_pro_version
< 92) {
908 mutex_unlock(&sock
->mutex
);
909 conn_err(tconn
, "--dry-run is not supported by peer");
914 if (tconn
->agreed_pro_version
>= 87)
915 size
+= strlen(nc
->integrity_alg
) + 1;
917 p
->protocol
= cpu_to_be32(nc
->wire_protocol
);
918 p
->after_sb_0p
= cpu_to_be32(nc
->after_sb_0p
);
919 p
->after_sb_1p
= cpu_to_be32(nc
->after_sb_1p
);
920 p
->after_sb_2p
= cpu_to_be32(nc
->after_sb_2p
);
921 p
->two_primaries
= cpu_to_be32(nc
->two_primaries
);
927 p
->conn_flags
= cpu_to_be32(cf
);
929 if (tconn
->agreed_pro_version
>= 87)
930 strcpy(p
->integrity_alg
, nc
->integrity_alg
);
933 return conn_send_command(tconn
, sock
, P_PROTOCOL
, size
, NULL
, 0);
936 int _drbd_send_uuids(struct drbd_conf
*mdev
, u64 uuid_flags
)
938 struct drbd_socket
*sock
;
942 if (!get_ldev_if_state(mdev
, D_NEGOTIATING
))
945 sock
= &mdev
->tconn
->data
;
946 p
= drbd_prepare_command(mdev
, sock
);
951 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
952 p
->uuid
[i
] = mdev
->ldev
? cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]) : 0;
954 mdev
->comm_bm_set
= drbd_bm_total_weight(mdev
);
955 p
->uuid
[UI_SIZE
] = cpu_to_be64(mdev
->comm_bm_set
);
957 uuid_flags
|= rcu_dereference(mdev
->tconn
->net_conf
)->want_lose
? 1 : 0;
959 uuid_flags
|= test_bit(CRASHED_PRIMARY
, &mdev
->flags
) ? 2 : 0;
960 uuid_flags
|= mdev
->new_state_tmp
.disk
== D_INCONSISTENT
? 4 : 0;
961 p
->uuid
[UI_FLAGS
] = cpu_to_be64(uuid_flags
);
964 return drbd_send_command(mdev
, sock
, P_UUIDS
, sizeof(*p
), NULL
, 0);
967 int drbd_send_uuids(struct drbd_conf
*mdev
)
969 return _drbd_send_uuids(mdev
, 0);
972 int drbd_send_uuids_skip_initial_sync(struct drbd_conf
*mdev
)
974 return _drbd_send_uuids(mdev
, 8);
977 void drbd_print_uuids(struct drbd_conf
*mdev
, const char *text
)
979 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
980 u64
*uuid
= mdev
->ldev
->md
.uuid
;
981 dev_info(DEV
, "%s %016llX:%016llX:%016llX:%016llX\n",
983 (unsigned long long)uuid
[UI_CURRENT
],
984 (unsigned long long)uuid
[UI_BITMAP
],
985 (unsigned long long)uuid
[UI_HISTORY_START
],
986 (unsigned long long)uuid
[UI_HISTORY_END
]);
989 dev_info(DEV
, "%s effective data uuid: %016llX\n",
991 (unsigned long long)mdev
->ed_uuid
);
995 void drbd_gen_and_send_sync_uuid(struct drbd_conf
*mdev
)
997 struct drbd_socket
*sock
;
1001 D_ASSERT(mdev
->state
.disk
== D_UP_TO_DATE
);
1003 uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
] + UUID_NEW_BM_OFFSET
;
1004 drbd_uuid_set(mdev
, UI_BITMAP
, uuid
);
1005 drbd_print_uuids(mdev
, "updated sync UUID");
1008 sock
= &mdev
->tconn
->data
;
1009 p
= drbd_prepare_command(mdev
, sock
);
1011 p
->uuid
= cpu_to_be64(uuid
);
1012 drbd_send_command(mdev
, sock
, P_SYNC_UUID
, sizeof(*p
), NULL
, 0);
1016 int drbd_send_sizes(struct drbd_conf
*mdev
, int trigger_reply
, enum dds_flags flags
)
1018 struct drbd_socket
*sock
;
1020 sector_t d_size
, u_size
;
1021 int q_order_type
, max_bio_size
;
1023 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
1024 D_ASSERT(mdev
->ldev
->backing_bdev
);
1025 d_size
= drbd_get_max_capacity(mdev
->ldev
);
1026 u_size
= mdev
->ldev
->dc
.disk_size
;
1027 q_order_type
= drbd_queue_order_type(mdev
);
1028 max_bio_size
= queue_max_hw_sectors(mdev
->ldev
->backing_bdev
->bd_disk
->queue
) << 9;
1029 max_bio_size
= min_t(int, max_bio_size
, DRBD_MAX_BIO_SIZE
);
1034 q_order_type
= QUEUE_ORDERED_NONE
;
1035 max_bio_size
= DRBD_MAX_BIO_SIZE
; /* ... multiple BIOs per peer_request */
1038 sock
= &mdev
->tconn
->data
;
1039 p
= drbd_prepare_command(mdev
, sock
);
1042 p
->d_size
= cpu_to_be64(d_size
);
1043 p
->u_size
= cpu_to_be64(u_size
);
1044 p
->c_size
= cpu_to_be64(trigger_reply
? 0 : drbd_get_capacity(mdev
->this_bdev
));
1045 p
->max_bio_size
= cpu_to_be32(max_bio_size
);
1046 p
->queue_order_type
= cpu_to_be16(q_order_type
);
1047 p
->dds_flags
= cpu_to_be16(flags
);
1048 return drbd_send_command(mdev
, sock
, P_SIZES
, sizeof(*p
), NULL
, 0);
1052 * drbd_send_state() - Sends the drbd state to the peer
1053 * @mdev: DRBD device.
1055 int drbd_send_state(struct drbd_conf
*mdev
)
1057 struct drbd_socket
*sock
;
1060 sock
= &mdev
->tconn
->data
;
1061 p
= drbd_prepare_command(mdev
, sock
);
1064 p
->state
= cpu_to_be32(mdev
->state
.i
); /* Within the send mutex */
1065 return drbd_send_command(mdev
, sock
, P_STATE
, sizeof(*p
), NULL
, 0);
1068 int drbd_send_state_req(struct drbd_conf
*mdev
, union drbd_state mask
, union drbd_state val
)
1070 struct drbd_socket
*sock
;
1071 struct p_req_state
*p
;
1073 sock
= &mdev
->tconn
->data
;
1074 p
= drbd_prepare_command(mdev
, sock
);
1077 p
->mask
= cpu_to_be32(mask
.i
);
1078 p
->val
= cpu_to_be32(val
.i
);
1079 return drbd_send_command(mdev
, sock
, P_STATE_CHG_REQ
, sizeof(*p
), NULL
, 0);
1083 int conn_send_state_req(struct drbd_tconn
*tconn
, union drbd_state mask
, union drbd_state val
)
1085 enum drbd_packet cmd
;
1086 struct drbd_socket
*sock
;
1087 struct p_req_state
*p
;
1089 cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REQ
: P_CONN_ST_CHG_REQ
;
1090 sock
= &tconn
->data
;
1091 p
= conn_prepare_command(tconn
, sock
);
1094 p
->mask
= cpu_to_be32(mask
.i
);
1095 p
->val
= cpu_to_be32(val
.i
);
1096 return conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1099 void drbd_send_sr_reply(struct drbd_conf
*mdev
, enum drbd_state_rv retcode
)
1101 struct drbd_socket
*sock
;
1102 struct p_req_state_reply
*p
;
1104 sock
= &mdev
->tconn
->meta
;
1105 p
= drbd_prepare_command(mdev
, sock
);
1107 p
->retcode
= cpu_to_be32(retcode
);
1108 drbd_send_command(mdev
, sock
, P_STATE_CHG_REPLY
, sizeof(*p
), NULL
, 0);
1112 void conn_send_sr_reply(struct drbd_tconn
*tconn
, enum drbd_state_rv retcode
)
1114 struct drbd_socket
*sock
;
1115 struct p_req_state_reply
*p
;
1116 enum drbd_packet cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REPLY
: P_CONN_ST_CHG_REPLY
;
1118 sock
= &tconn
->meta
;
1119 p
= conn_prepare_command(tconn
, sock
);
1121 p
->retcode
= cpu_to_be32(retcode
);
1122 conn_send_command(tconn
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1126 static void dcbp_set_code(struct p_compressed_bm
*p
, enum drbd_bitmap_code code
)
1128 BUG_ON(code
& ~0xf);
1129 p
->encoding
= (p
->encoding
& ~0xf) | code
;
1132 static void dcbp_set_start(struct p_compressed_bm
*p
, int set
)
1134 p
->encoding
= (p
->encoding
& ~0x80) | (set
? 0x80 : 0);
1137 static void dcbp_set_pad_bits(struct p_compressed_bm
*p
, int n
)
1140 p
->encoding
= (p
->encoding
& (~0x7 << 4)) | (n
<< 4);
1143 int fill_bitmap_rle_bits(struct drbd_conf
*mdev
,
1144 struct p_compressed_bm
*p
,
1146 struct bm_xfer_ctx
*c
)
1148 struct bitstream bs
;
1149 unsigned long plain_bits
;
1156 /* may we use this feature? */
1158 use_rle
= rcu_dereference(mdev
->tconn
->net_conf
)->use_rle
;
1160 if (!use_rle
|| mdev
->tconn
->agreed_pro_version
< 90)
1163 if (c
->bit_offset
>= c
->bm_bits
)
1164 return 0; /* nothing to do. */
1166 /* use at most thus many bytes */
1167 bitstream_init(&bs
, p
->code
, size
, 0);
1168 memset(p
->code
, 0, size
);
1169 /* plain bits covered in this code string */
1172 /* p->encoding & 0x80 stores whether the first run length is set.
1173 * bit offset is implicit.
1174 * start with toggle == 2 to be able to tell the first iteration */
1177 /* see how much plain bits we can stuff into one packet
1178 * using RLE and VLI. */
1180 tmp
= (toggle
== 0) ? _drbd_bm_find_next_zero(mdev
, c
->bit_offset
)
1181 : _drbd_bm_find_next(mdev
, c
->bit_offset
);
1184 rl
= tmp
- c
->bit_offset
;
1186 if (toggle
== 2) { /* first iteration */
1188 /* the first checked bit was set,
1189 * store start value, */
1190 dcbp_set_start(p
, 1);
1191 /* but skip encoding of zero run length */
1195 dcbp_set_start(p
, 0);
1198 /* paranoia: catch zero runlength.
1199 * can only happen if bitmap is modified while we scan it. */
1201 dev_err(DEV
, "unexpected zero runlength while encoding bitmap "
1202 "t:%u bo:%lu\n", toggle
, c
->bit_offset
);
1206 bits
= vli_encode_bits(&bs
, rl
);
1207 if (bits
== -ENOBUFS
) /* buffer full */
1210 dev_err(DEV
, "error while encoding bitmap: %d\n", bits
);
1216 c
->bit_offset
= tmp
;
1217 } while (c
->bit_offset
< c
->bm_bits
);
1219 len
= bs
.cur
.b
- p
->code
+ !!bs
.cur
.bit
;
1221 if (plain_bits
< (len
<< 3)) {
1222 /* incompressible with this method.
1223 * we need to rewind both word and bit position. */
1224 c
->bit_offset
-= plain_bits
;
1225 bm_xfer_ctx_bit_to_word_offset(c
);
1226 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1230 /* RLE + VLI was able to compress it just fine.
1231 * update c->word_offset. */
1232 bm_xfer_ctx_bit_to_word_offset(c
);
1234 /* store pad_bits */
1235 dcbp_set_pad_bits(p
, (8 - bs
.cur
.bit
) & 0x7);
1241 * send_bitmap_rle_or_plain
1243 * Return 0 when done, 1 when another iteration is needed, and a negative error
1244 * code upon failure.
1247 send_bitmap_rle_or_plain(struct drbd_conf
*mdev
, struct bm_xfer_ctx
*c
)
1249 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1250 unsigned int header_size
= drbd_header_size(mdev
->tconn
);
1251 struct p_compressed_bm
*p
= sock
->sbuf
+ header_size
;
1254 len
= fill_bitmap_rle_bits(mdev
, p
,
1255 DRBD_SOCKET_BUFFER_SIZE
- header_size
- sizeof(*p
), c
);
1260 dcbp_set_code(p
, RLE_VLI_Bits
);
1261 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
,
1262 P_COMPRESSED_BITMAP
, sizeof(*p
) + len
,
1265 c
->bytes
[0] += header_size
+ sizeof(*p
) + len
;
1267 if (c
->bit_offset
>= c
->bm_bits
)
1270 /* was not compressible.
1271 * send a buffer full of plain text bits instead. */
1272 unsigned int data_size
;
1273 unsigned long num_words
;
1274 unsigned long *p
= sock
->sbuf
+ header_size
;
1276 data_size
= DRBD_SOCKET_BUFFER_SIZE
- header_size
;
1277 num_words
= min_t(size_t, data_size
/ sizeof(*p
),
1278 c
->bm_words
- c
->word_offset
);
1279 len
= num_words
* sizeof(*p
);
1281 drbd_bm_get_lel(mdev
, c
->word_offset
, num_words
, p
);
1282 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_BITMAP
, len
, NULL
, 0);
1283 c
->word_offset
+= num_words
;
1284 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1287 c
->bytes
[1] += header_size
+ len
;
1289 if (c
->bit_offset
> c
->bm_bits
)
1290 c
->bit_offset
= c
->bm_bits
;
1294 INFO_bm_xfer_stats(mdev
, "send", c
);
1302 /* See the comment at receive_bitmap() */
1303 static int _drbd_send_bitmap(struct drbd_conf
*mdev
)
1305 struct bm_xfer_ctx c
;
1308 if (!expect(mdev
->bitmap
))
1311 if (get_ldev(mdev
)) {
1312 if (drbd_md_test_flag(mdev
->ldev
, MDF_FULL_SYNC
)) {
1313 dev_info(DEV
, "Writing the whole bitmap, MDF_FullSync was set.\n");
1314 drbd_bm_set_all(mdev
);
1315 if (drbd_bm_write(mdev
)) {
1316 /* write_bm did fail! Leave full sync flag set in Meta P_DATA
1317 * but otherwise process as per normal - need to tell other
1318 * side that a full resync is required! */
1319 dev_err(DEV
, "Failed to write bitmap to disk!\n");
1321 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
1328 c
= (struct bm_xfer_ctx
) {
1329 .bm_bits
= drbd_bm_bits(mdev
),
1330 .bm_words
= drbd_bm_words(mdev
),
1334 err
= send_bitmap_rle_or_plain(mdev
, &c
);
1340 int drbd_send_bitmap(struct drbd_conf
*mdev
)
1342 struct drbd_socket
*sock
= &mdev
->tconn
->data
;
1345 mutex_lock(&sock
->mutex
);
1347 err
= !_drbd_send_bitmap(mdev
);
1348 mutex_unlock(&sock
->mutex
);
1352 void drbd_send_b_ack(struct drbd_conf
*mdev
, u32 barrier_nr
, u32 set_size
)
1354 struct drbd_socket
*sock
;
1355 struct p_barrier_ack
*p
;
1357 if (mdev
->state
.conn
< C_CONNECTED
)
1360 sock
= &mdev
->tconn
->meta
;
1361 p
= drbd_prepare_command(mdev
, sock
);
1364 p
->barrier
= barrier_nr
;
1365 p
->set_size
= cpu_to_be32(set_size
);
1366 drbd_send_command(mdev
, sock
, P_BARRIER_ACK
, sizeof(*p
), NULL
, 0);
1370 * _drbd_send_ack() - Sends an ack packet
1371 * @mdev: DRBD device.
1372 * @cmd: Packet command code.
1373 * @sector: sector, needs to be in big endian byte order
1374 * @blksize: size in byte, needs to be in big endian byte order
1375 * @block_id: Id, big endian byte order
1377 static int _drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1378 u64 sector
, u32 blksize
, u64 block_id
)
1380 struct drbd_socket
*sock
;
1381 struct p_block_ack
*p
;
1383 if (mdev
->state
.conn
< C_CONNECTED
)
1386 sock
= &mdev
->tconn
->meta
;
1387 p
= drbd_prepare_command(mdev
, sock
);
1391 p
->block_id
= block_id
;
1392 p
->blksize
= blksize
;
1393 p
->seq_num
= cpu_to_be32(atomic_inc_return(&mdev
->packet_seq
));
1394 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1397 /* dp->sector and dp->block_id already/still in network byte order,
1398 * data_size is payload size according to dp->head,
1399 * and may need to be corrected for digest size. */
1400 void drbd_send_ack_dp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1401 struct p_data
*dp
, int data_size
)
1403 data_size
-= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_r_tfm
) ?
1404 crypto_hash_digestsize(mdev
->tconn
->integrity_r_tfm
) : 0;
1405 _drbd_send_ack(mdev
, cmd
, dp
->sector
, cpu_to_be32(data_size
),
1409 void drbd_send_ack_rp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1410 struct p_block_req
*rp
)
1412 _drbd_send_ack(mdev
, cmd
, rp
->sector
, rp
->blksize
, rp
->block_id
);
1416 * drbd_send_ack() - Sends an ack packet
1417 * @mdev: DRBD device
1418 * @cmd: packet command code
1419 * @peer_req: peer request
1421 int drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1422 struct drbd_peer_request
*peer_req
)
1424 return _drbd_send_ack(mdev
, cmd
,
1425 cpu_to_be64(peer_req
->i
.sector
),
1426 cpu_to_be32(peer_req
->i
.size
),
1427 peer_req
->block_id
);
1430 /* This function misuses the block_id field to signal if the blocks
1431 * are is sync or not. */
1432 int drbd_send_ack_ex(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1433 sector_t sector
, int blksize
, u64 block_id
)
1435 return _drbd_send_ack(mdev
, cmd
,
1436 cpu_to_be64(sector
),
1437 cpu_to_be32(blksize
),
1438 cpu_to_be64(block_id
));
1441 int drbd_send_drequest(struct drbd_conf
*mdev
, int cmd
,
1442 sector_t sector
, int size
, u64 block_id
)
1444 struct drbd_socket
*sock
;
1445 struct p_block_req
*p
;
1447 sock
= &mdev
->tconn
->data
;
1448 p
= drbd_prepare_command(mdev
, sock
);
1451 p
->sector
= cpu_to_be64(sector
);
1452 p
->block_id
= block_id
;
1453 p
->blksize
= cpu_to_be32(size
);
1454 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
), NULL
, 0);
1457 int drbd_send_drequest_csum(struct drbd_conf
*mdev
, sector_t sector
, int size
,
1458 void *digest
, int digest_size
, enum drbd_packet cmd
)
1460 struct drbd_socket
*sock
;
1461 struct p_block_req
*p
;
1463 /* FIXME: Put the digest into the preallocated socket buffer. */
1465 sock
= &mdev
->tconn
->data
;
1466 p
= drbd_prepare_command(mdev
, sock
);
1469 p
->sector
= cpu_to_be64(sector
);
1470 p
->block_id
= ID_SYNCER
/* unused */;
1471 p
->blksize
= cpu_to_be32(size
);
1472 return drbd_send_command(mdev
, sock
, cmd
, sizeof(*p
),
1473 digest
, digest_size
);
1476 int drbd_send_ov_request(struct drbd_conf
*mdev
, sector_t sector
, int size
)
1478 struct drbd_socket
*sock
;
1479 struct p_block_req
*p
;
1481 sock
= &mdev
->tconn
->data
;
1482 p
= drbd_prepare_command(mdev
, sock
);
1485 p
->sector
= cpu_to_be64(sector
);
1486 p
->block_id
= ID_SYNCER
/* unused */;
1487 p
->blksize
= cpu_to_be32(size
);
1488 return drbd_send_command(mdev
, sock
, P_OV_REQUEST
, sizeof(*p
), NULL
, 0);
1491 /* called on sndtimeo
1492 * returns false if we should retry,
1493 * true if we think connection is dead
1495 static int we_should_drop_the_connection(struct drbd_tconn
*tconn
, struct socket
*sock
)
1498 /* long elapsed = (long)(jiffies - mdev->last_received); */
1500 drop_it
= tconn
->meta
.socket
== sock
1501 || !tconn
->asender
.task
1502 || get_t_state(&tconn
->asender
) != RUNNING
1503 || tconn
->cstate
< C_WF_REPORT_PARAMS
;
1508 drop_it
= !--tconn
->ko_count
;
1510 conn_err(tconn
, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1511 current
->comm
, current
->pid
, tconn
->ko_count
);
1512 request_ping(tconn
);
1515 return drop_it
; /* && (mdev->state == R_PRIMARY) */;
1518 static void drbd_update_congested(struct drbd_tconn
*tconn
)
1520 struct sock
*sk
= tconn
->data
.socket
->sk
;
1521 if (sk
->sk_wmem_queued
> sk
->sk_sndbuf
* 4 / 5)
1522 set_bit(NET_CONGESTED
, &tconn
->flags
);
1525 /* The idea of sendpage seems to be to put some kind of reference
1526 * to the page into the skb, and to hand it over to the NIC. In
1527 * this process get_page() gets called.
1529 * As soon as the page was really sent over the network put_page()
1530 * gets called by some part of the network layer. [ NIC driver? ]
1532 * [ get_page() / put_page() increment/decrement the count. If count
1533 * reaches 0 the page will be freed. ]
1535 * This works nicely with pages from FSs.
1536 * But this means that in protocol A we might signal IO completion too early!
1538 * In order not to corrupt data during a resync we must make sure
1539 * that we do not reuse our own buffer pages (EEs) to early, therefore
1540 * we have the net_ee list.
1542 * XFS seems to have problems, still, it submits pages with page_count == 0!
1543 * As a workaround, we disable sendpage on pages
1544 * with page_count == 0 or PageSlab.
1546 static int _drbd_no_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1547 int offset
, size_t size
, unsigned msg_flags
)
1549 struct socket
*socket
;
1553 socket
= mdev
->tconn
->data
.socket
;
1554 addr
= kmap(page
) + offset
;
1555 err
= drbd_send_all(mdev
->tconn
, socket
, addr
, size
, msg_flags
);
1558 mdev
->send_cnt
+= size
>> 9;
1562 static int _drbd_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1563 int offset
, size_t size
, unsigned msg_flags
)
1565 struct socket
*socket
= mdev
->tconn
->data
.socket
;
1566 mm_segment_t oldfs
= get_fs();
1570 /* e.g. XFS meta- & log-data is in slab pages, which have a
1571 * page_count of 0 and/or have PageSlab() set.
1572 * we cannot use send_page for those, as that does get_page();
1573 * put_page(); and would cause either a VM_BUG directly, or
1574 * __page_cache_release a page that would actually still be referenced
1575 * by someone, leading to some obscure delayed Oops somewhere else. */
1576 if (disable_sendpage
|| (page_count(page
) < 1) || PageSlab(page
))
1577 return _drbd_no_send_page(mdev
, page
, offset
, size
, msg_flags
);
1579 msg_flags
|= MSG_NOSIGNAL
;
1580 drbd_update_congested(mdev
->tconn
);
1585 sent
= socket
->ops
->sendpage(socket
, page
, offset
, len
, msg_flags
);
1587 if (sent
== -EAGAIN
) {
1588 if (we_should_drop_the_connection(mdev
->tconn
, socket
))
1592 dev_warn(DEV
, "%s: size=%d len=%d sent=%d\n",
1593 __func__
, (int)size
, len
, sent
);
1600 } while (len
> 0 /* THINK && mdev->cstate >= C_CONNECTED*/);
1602 clear_bit(NET_CONGESTED
, &mdev
->tconn
->flags
);
1606 mdev
->send_cnt
+= size
>> 9;
1611 static int _drbd_send_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1613 struct bio_vec
*bvec
;
1615 /* hint all but last page with MSG_MORE */
1616 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1619 err
= _drbd_no_send_page(mdev
, bvec
->bv_page
,
1620 bvec
->bv_offset
, bvec
->bv_len
,
1621 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1628 static int _drbd_send_zc_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1630 struct bio_vec
*bvec
;
1632 /* hint all but last page with MSG_MORE */
1633 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1636 err
= _drbd_send_page(mdev
, bvec
->bv_page
,
1637 bvec
->bv_offset
, bvec
->bv_len
,
1638 i
== bio
->bi_vcnt
- 1 ? 0 : MSG_MORE
);
1645 static int _drbd_send_zc_ee(struct drbd_conf
*mdev
,
1646 struct drbd_peer_request
*peer_req
)
1648 struct page
*page
= peer_req
->pages
;
1649 unsigned len
= peer_req
->i
.size
;
1652 /* hint all but last page with MSG_MORE */
1653 page_chain_for_each(page
) {
1654 unsigned l
= min_t(unsigned, len
, PAGE_SIZE
);
1656 err
= _drbd_send_page(mdev
, page
, 0, l
,
1657 page_chain_next(page
) ? MSG_MORE
: 0);
1665 static u32
bio_flags_to_wire(struct drbd_conf
*mdev
, unsigned long bi_rw
)
1667 if (mdev
->tconn
->agreed_pro_version
>= 95)
1668 return (bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0) |
1669 (bi_rw
& REQ_FUA
? DP_FUA
: 0) |
1670 (bi_rw
& REQ_FLUSH
? DP_FLUSH
: 0) |
1671 (bi_rw
& REQ_DISCARD
? DP_DISCARD
: 0);
1673 return bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0;
1676 /* Used to send write requests
1677 * R_PRIMARY -> Peer (P_DATA)
1679 int drbd_send_dblock(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1681 struct drbd_socket
*sock
;
1683 unsigned int dp_flags
= 0;
1687 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1688 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1690 sock
= &mdev
->tconn
->data
;
1691 p
= drbd_prepare_command(mdev
, sock
);
1694 p
->sector
= cpu_to_be64(req
->i
.sector
);
1695 p
->block_id
= (unsigned long)req
;
1696 p
->seq_num
= cpu_to_be32(req
->seq_num
= atomic_inc_return(&mdev
->packet_seq
));
1697 dp_flags
= bio_flags_to_wire(mdev
, req
->master_bio
->bi_rw
);
1698 if (mdev
->state
.conn
>= C_SYNC_SOURCE
&&
1699 mdev
->state
.conn
<= C_PAUSED_SYNC_T
)
1700 dp_flags
|= DP_MAY_SET_IN_SYNC
;
1701 if (mdev
->tconn
->agreed_pro_version
>= 100) {
1702 if (req
->rq_state
& RQ_EXP_RECEIVE_ACK
)
1703 dp_flags
|= DP_SEND_RECEIVE_ACK
;
1704 if (req
->rq_state
& RQ_EXP_WRITE_ACK
)
1705 dp_flags
|= DP_SEND_WRITE_ACK
;
1707 p
->dp_flags
= cpu_to_be32(dp_flags
);
1709 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, p
+ 1);
1710 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, P_DATA
, sizeof(*p
) + dgs
, NULL
, req
->i
.size
);
1712 /* For protocol A, we have to memcpy the payload into
1713 * socket buffers, as we may complete right away
1714 * as soon as we handed it over to tcp, at which point the data
1715 * pages may become invalid.
1717 * For data-integrity enabled, we copy it as well, so we can be
1718 * sure that even if the bio pages may still be modified, it
1719 * won't change the data on the wire, thus if the digest checks
1720 * out ok after sending on this side, but does not fit on the
1721 * receiving side, we sure have detected corruption elsewhere.
1723 if (!(req
->rq_state
& (RQ_EXP_RECEIVE_ACK
| RQ_EXP_WRITE_ACK
)) || dgs
)
1724 err
= _drbd_send_bio(mdev
, req
->master_bio
);
1726 err
= _drbd_send_zc_bio(mdev
, req
->master_bio
);
1728 /* double check digest, sometimes buffers have been modified in flight. */
1729 if (dgs
> 0 && dgs
<= 64) {
1730 /* 64 byte, 512 bit, is the largest digest size
1731 * currently supported in kernel crypto. */
1732 unsigned char digest
[64];
1733 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, digest
);
1734 if (memcmp(p
+ 1, digest
, dgs
)) {
1736 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1737 (unsigned long long)req
->i
.sector
, req
->i
.size
);
1739 } /* else if (dgs > 64) {
1740 ... Be noisy about digest too large ...
1743 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1748 /* answer packet, used to send data back for read requests:
1749 * Peer -> (diskless) R_PRIMARY (P_DATA_REPLY)
1750 * C_SYNC_SOURCE -> C_SYNC_TARGET (P_RS_DATA_REPLY)
1752 int drbd_send_block(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1753 struct drbd_peer_request
*peer_req
)
1755 struct drbd_socket
*sock
;
1760 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1761 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1763 sock
= &mdev
->tconn
->data
;
1764 p
= drbd_prepare_command(mdev
, sock
);
1767 p
->sector
= cpu_to_be64(peer_req
->i
.sector
);
1768 p
->block_id
= peer_req
->block_id
;
1769 p
->seq_num
= 0; /* unused */
1771 drbd_csum_ee(mdev
, mdev
->tconn
->integrity_w_tfm
, peer_req
, p
+ 1);
1772 err
= __send_command(mdev
->tconn
, mdev
->vnr
, sock
, cmd
, sizeof(*p
) + dgs
, NULL
, peer_req
->i
.size
);
1774 err
= _drbd_send_zc_ee(mdev
, peer_req
);
1775 mutex_unlock(&sock
->mutex
); /* locked by drbd_prepare_command() */
1780 int drbd_send_out_of_sync(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1782 struct drbd_socket
*sock
;
1783 struct p_block_desc
*p
;
1785 sock
= &mdev
->tconn
->data
;
1786 p
= drbd_prepare_command(mdev
, sock
);
1789 p
->sector
= cpu_to_be64(req
->i
.sector
);
1790 p
->blksize
= cpu_to_be32(req
->i
.size
);
1791 return drbd_send_command(mdev
, sock
, P_OUT_OF_SYNC
, sizeof(*p
), NULL
, 0);
1795 drbd_send distinguishes two cases:
1797 Packets sent via the data socket "sock"
1798 and packets sent via the meta data socket "msock"
1801 -----------------+-------------------------+------------------------------
1802 timeout conf.timeout / 2 conf.timeout / 2
1803 timeout action send a ping via msock Abort communication
1804 and close all sockets
1808 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
1810 int drbd_send(struct drbd_tconn
*tconn
, struct socket
*sock
,
1811 void *buf
, size_t size
, unsigned msg_flags
)
1820 /* THINK if (signal_pending) return ... ? */
1825 msg
.msg_name
= NULL
;
1826 msg
.msg_namelen
= 0;
1827 msg
.msg_control
= NULL
;
1828 msg
.msg_controllen
= 0;
1829 msg
.msg_flags
= msg_flags
| MSG_NOSIGNAL
;
1831 if (sock
== tconn
->data
.socket
) {
1833 tconn
->ko_count
= rcu_dereference(tconn
->net_conf
)->ko_count
;
1835 drbd_update_congested(tconn
);
1839 * tcp_sendmsg does _not_ use its size parameter at all ?
1841 * -EAGAIN on timeout, -EINTR on signal.
1844 * do we need to block DRBD_SIG if sock == &meta.socket ??
1845 * otherwise wake_asender() might interrupt some send_*Ack !
1847 rv
= kernel_sendmsg(sock
, &msg
, &iov
, 1, size
);
1848 if (rv
== -EAGAIN
) {
1849 if (we_should_drop_the_connection(tconn
, sock
))
1855 flush_signals(current
);
1863 } while (sent
< size
);
1865 if (sock
== tconn
->data
.socket
)
1866 clear_bit(NET_CONGESTED
, &tconn
->flags
);
1869 if (rv
!= -EAGAIN
) {
1870 conn_err(tconn
, "%s_sendmsg returned %d\n",
1871 sock
== tconn
->meta
.socket
? "msock" : "sock",
1873 conn_request_state(tconn
, NS(conn
, C_BROKEN_PIPE
), CS_HARD
);
1875 conn_request_state(tconn
, NS(conn
, C_TIMEOUT
), CS_HARD
);
1882 * drbd_send_all - Send an entire buffer
1884 * Returns 0 upon success and a negative error value otherwise.
1886 int drbd_send_all(struct drbd_tconn
*tconn
, struct socket
*sock
, void *buffer
,
1887 size_t size
, unsigned msg_flags
)
1891 err
= drbd_send(tconn
, sock
, buffer
, size
, msg_flags
);
1899 static int drbd_open(struct block_device
*bdev
, fmode_t mode
)
1901 struct drbd_conf
*mdev
= bdev
->bd_disk
->private_data
;
1902 unsigned long flags
;
1905 mutex_lock(&drbd_main_mutex
);
1906 spin_lock_irqsave(&mdev
->tconn
->req_lock
, flags
);
1907 /* to have a stable mdev->state.role
1908 * and no race with updating open_cnt */
1910 if (mdev
->state
.role
!= R_PRIMARY
) {
1911 if (mode
& FMODE_WRITE
)
1913 else if (!allow_oos
)
1919 spin_unlock_irqrestore(&mdev
->tconn
->req_lock
, flags
);
1920 mutex_unlock(&drbd_main_mutex
);
1925 static int drbd_release(struct gendisk
*gd
, fmode_t mode
)
1927 struct drbd_conf
*mdev
= gd
->private_data
;
1928 mutex_lock(&drbd_main_mutex
);
1930 mutex_unlock(&drbd_main_mutex
);
1934 static void drbd_set_defaults(struct drbd_conf
*mdev
)
1936 /* Beware! The actual layout differs
1937 * between big endian and little endian */
1938 mdev
->state
= (union drbd_dev_state
) {
1939 { .role
= R_SECONDARY
,
1941 .conn
= C_STANDALONE
,
1947 void drbd_init_set_defaults(struct drbd_conf
*mdev
)
1949 /* the memset(,0,) did most of this.
1950 * note: only assignments, no allocation in here */
1952 drbd_set_defaults(mdev
);
1954 atomic_set(&mdev
->ap_bio_cnt
, 0);
1955 atomic_set(&mdev
->ap_pending_cnt
, 0);
1956 atomic_set(&mdev
->rs_pending_cnt
, 0);
1957 atomic_set(&mdev
->unacked_cnt
, 0);
1958 atomic_set(&mdev
->local_cnt
, 0);
1959 atomic_set(&mdev
->pp_in_use_by_net
, 0);
1960 atomic_set(&mdev
->rs_sect_in
, 0);
1961 atomic_set(&mdev
->rs_sect_ev
, 0);
1962 atomic_set(&mdev
->ap_in_flight
, 0);
1964 mutex_init(&mdev
->md_io_mutex
);
1965 mutex_init(&mdev
->own_state_mutex
);
1966 mdev
->state_mutex
= &mdev
->own_state_mutex
;
1968 spin_lock_init(&mdev
->al_lock
);
1969 spin_lock_init(&mdev
->peer_seq_lock
);
1970 spin_lock_init(&mdev
->epoch_lock
);
1972 INIT_LIST_HEAD(&mdev
->active_ee
);
1973 INIT_LIST_HEAD(&mdev
->sync_ee
);
1974 INIT_LIST_HEAD(&mdev
->done_ee
);
1975 INIT_LIST_HEAD(&mdev
->read_ee
);
1976 INIT_LIST_HEAD(&mdev
->net_ee
);
1977 INIT_LIST_HEAD(&mdev
->resync_reads
);
1978 INIT_LIST_HEAD(&mdev
->resync_work
.list
);
1979 INIT_LIST_HEAD(&mdev
->unplug_work
.list
);
1980 INIT_LIST_HEAD(&mdev
->go_diskless
.list
);
1981 INIT_LIST_HEAD(&mdev
->md_sync_work
.list
);
1982 INIT_LIST_HEAD(&mdev
->start_resync_work
.list
);
1983 INIT_LIST_HEAD(&mdev
->bm_io_work
.w
.list
);
1985 mdev
->resync_work
.cb
= w_resync_timer
;
1986 mdev
->unplug_work
.cb
= w_send_write_hint
;
1987 mdev
->go_diskless
.cb
= w_go_diskless
;
1988 mdev
->md_sync_work
.cb
= w_md_sync
;
1989 mdev
->bm_io_work
.w
.cb
= w_bitmap_io
;
1990 mdev
->start_resync_work
.cb
= w_start_resync
;
1992 mdev
->resync_work
.mdev
= mdev
;
1993 mdev
->unplug_work
.mdev
= mdev
;
1994 mdev
->go_diskless
.mdev
= mdev
;
1995 mdev
->md_sync_work
.mdev
= mdev
;
1996 mdev
->bm_io_work
.w
.mdev
= mdev
;
1997 mdev
->start_resync_work
.mdev
= mdev
;
1999 init_timer(&mdev
->resync_timer
);
2000 init_timer(&mdev
->md_sync_timer
);
2001 init_timer(&mdev
->start_resync_timer
);
2002 init_timer(&mdev
->request_timer
);
2003 mdev
->resync_timer
.function
= resync_timer_fn
;
2004 mdev
->resync_timer
.data
= (unsigned long) mdev
;
2005 mdev
->md_sync_timer
.function
= md_sync_timer_fn
;
2006 mdev
->md_sync_timer
.data
= (unsigned long) mdev
;
2007 mdev
->start_resync_timer
.function
= start_resync_timer_fn
;
2008 mdev
->start_resync_timer
.data
= (unsigned long) mdev
;
2009 mdev
->request_timer
.function
= request_timer_fn
;
2010 mdev
->request_timer
.data
= (unsigned long) mdev
;
2012 init_waitqueue_head(&mdev
->misc_wait
);
2013 init_waitqueue_head(&mdev
->state_wait
);
2014 init_waitqueue_head(&mdev
->ee_wait
);
2015 init_waitqueue_head(&mdev
->al_wait
);
2016 init_waitqueue_head(&mdev
->seq_wait
);
2018 /* mdev->tconn->agreed_pro_version gets initialized in drbd_connect() */
2019 mdev
->write_ordering
= WO_bdev_flush
;
2020 mdev
->resync_wenr
= LC_FREE
;
2021 mdev
->peer_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
2022 mdev
->local_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
2025 void drbd_mdev_cleanup(struct drbd_conf
*mdev
)
2028 if (mdev
->tconn
->receiver
.t_state
!= NONE
)
2029 dev_err(DEV
, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
2030 mdev
->tconn
->receiver
.t_state
);
2032 /* no need to lock it, I'm the only thread alive */
2033 if (atomic_read(&mdev
->current_epoch
->epoch_size
) != 0)
2034 dev_err(DEV
, "epoch_size:%d\n", atomic_read(&mdev
->current_epoch
->epoch_size
));
2044 mdev
->rs_failed
= 0;
2045 mdev
->rs_last_events
= 0;
2046 mdev
->rs_last_sect_ev
= 0;
2047 for (i
= 0; i
< DRBD_SYNC_MARKS
; i
++) {
2048 mdev
->rs_mark_left
[i
] = 0;
2049 mdev
->rs_mark_time
[i
] = 0;
2051 D_ASSERT(mdev
->tconn
->net_conf
== NULL
);
2053 drbd_set_my_capacity(mdev
, 0);
2055 /* maybe never allocated. */
2056 drbd_bm_resize(mdev
, 0, 1);
2057 drbd_bm_cleanup(mdev
);
2060 drbd_free_resources(mdev
);
2061 clear_bit(AL_SUSPENDED
, &mdev
->flags
);
2063 D_ASSERT(list_empty(&mdev
->active_ee
));
2064 D_ASSERT(list_empty(&mdev
->sync_ee
));
2065 D_ASSERT(list_empty(&mdev
->done_ee
));
2066 D_ASSERT(list_empty(&mdev
->read_ee
));
2067 D_ASSERT(list_empty(&mdev
->net_ee
));
2068 D_ASSERT(list_empty(&mdev
->resync_reads
));
2069 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2070 D_ASSERT(list_empty(&mdev
->tconn
->meta
.work
.q
));
2071 D_ASSERT(list_empty(&mdev
->resync_work
.list
));
2072 D_ASSERT(list_empty(&mdev
->unplug_work
.list
));
2073 D_ASSERT(list_empty(&mdev
->go_diskless
.list
));
2075 drbd_set_defaults(mdev
);
2079 static void drbd_destroy_mempools(void)
2083 while (drbd_pp_pool
) {
2084 page
= drbd_pp_pool
;
2085 drbd_pp_pool
= (struct page
*)page_private(page
);
2090 /* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */
2092 if (drbd_md_io_bio_set
)
2093 bioset_free(drbd_md_io_bio_set
);
2094 if (drbd_md_io_page_pool
)
2095 mempool_destroy(drbd_md_io_page_pool
);
2096 if (drbd_ee_mempool
)
2097 mempool_destroy(drbd_ee_mempool
);
2098 if (drbd_request_mempool
)
2099 mempool_destroy(drbd_request_mempool
);
2101 kmem_cache_destroy(drbd_ee_cache
);
2102 if (drbd_request_cache
)
2103 kmem_cache_destroy(drbd_request_cache
);
2104 if (drbd_bm_ext_cache
)
2105 kmem_cache_destroy(drbd_bm_ext_cache
);
2106 if (drbd_al_ext_cache
)
2107 kmem_cache_destroy(drbd_al_ext_cache
);
2109 drbd_md_io_bio_set
= NULL
;
2110 drbd_md_io_page_pool
= NULL
;
2111 drbd_ee_mempool
= NULL
;
2112 drbd_request_mempool
= NULL
;
2113 drbd_ee_cache
= NULL
;
2114 drbd_request_cache
= NULL
;
2115 drbd_bm_ext_cache
= NULL
;
2116 drbd_al_ext_cache
= NULL
;
2121 static int drbd_create_mempools(void)
2124 const int number
= (DRBD_MAX_BIO_SIZE
/PAGE_SIZE
) * minor_count
;
2127 /* prepare our caches and mempools */
2128 drbd_request_mempool
= NULL
;
2129 drbd_ee_cache
= NULL
;
2130 drbd_request_cache
= NULL
;
2131 drbd_bm_ext_cache
= NULL
;
2132 drbd_al_ext_cache
= NULL
;
2133 drbd_pp_pool
= NULL
;
2134 drbd_md_io_page_pool
= NULL
;
2135 drbd_md_io_bio_set
= NULL
;
2138 drbd_request_cache
= kmem_cache_create(
2139 "drbd_req", sizeof(struct drbd_request
), 0, 0, NULL
);
2140 if (drbd_request_cache
== NULL
)
2143 drbd_ee_cache
= kmem_cache_create(
2144 "drbd_ee", sizeof(struct drbd_peer_request
), 0, 0, NULL
);
2145 if (drbd_ee_cache
== NULL
)
2148 drbd_bm_ext_cache
= kmem_cache_create(
2149 "drbd_bm", sizeof(struct bm_extent
), 0, 0, NULL
);
2150 if (drbd_bm_ext_cache
== NULL
)
2153 drbd_al_ext_cache
= kmem_cache_create(
2154 "drbd_al", sizeof(struct lc_element
), 0, 0, NULL
);
2155 if (drbd_al_ext_cache
== NULL
)
2159 drbd_md_io_bio_set
= bioset_create(DRBD_MIN_POOL_PAGES
, 0);
2160 if (drbd_md_io_bio_set
== NULL
)
2163 drbd_md_io_page_pool
= mempool_create_page_pool(DRBD_MIN_POOL_PAGES
, 0);
2164 if (drbd_md_io_page_pool
== NULL
)
2167 drbd_request_mempool
= mempool_create(number
,
2168 mempool_alloc_slab
, mempool_free_slab
, drbd_request_cache
);
2169 if (drbd_request_mempool
== NULL
)
2172 drbd_ee_mempool
= mempool_create(number
,
2173 mempool_alloc_slab
, mempool_free_slab
, drbd_ee_cache
);
2174 if (drbd_ee_mempool
== NULL
)
2177 /* drbd's page pool */
2178 spin_lock_init(&drbd_pp_lock
);
2180 for (i
= 0; i
< number
; i
++) {
2181 page
= alloc_page(GFP_HIGHUSER
);
2184 set_page_private(page
, (unsigned long)drbd_pp_pool
);
2185 drbd_pp_pool
= page
;
2187 drbd_pp_vacant
= number
;
2192 drbd_destroy_mempools(); /* in case we allocated some */
2196 static int drbd_notify_sys(struct notifier_block
*this, unsigned long code
,
2199 /* just so we have it. you never know what interesting things we
2200 * might want to do here some day...
2206 static struct notifier_block drbd_notifier
= {
2207 .notifier_call
= drbd_notify_sys
,
2210 static void drbd_release_all_peer_reqs(struct drbd_conf
*mdev
)
2214 rr
= drbd_free_peer_reqs(mdev
, &mdev
->active_ee
);
2216 dev_err(DEV
, "%d EEs in active list found!\n", rr
);
2218 rr
= drbd_free_peer_reqs(mdev
, &mdev
->sync_ee
);
2220 dev_err(DEV
, "%d EEs in sync list found!\n", rr
);
2222 rr
= drbd_free_peer_reqs(mdev
, &mdev
->read_ee
);
2224 dev_err(DEV
, "%d EEs in read list found!\n", rr
);
2226 rr
= drbd_free_peer_reqs(mdev
, &mdev
->done_ee
);
2228 dev_err(DEV
, "%d EEs in done list found!\n", rr
);
2230 rr
= drbd_free_peer_reqs(mdev
, &mdev
->net_ee
);
2232 dev_err(DEV
, "%d EEs in net list found!\n", rr
);
2235 /* caution. no locking. */
2236 void drbd_delete_device(struct drbd_conf
*mdev
)
2238 idr_remove(&mdev
->tconn
->volumes
, mdev
->vnr
);
2239 idr_remove(&minors
, mdev_to_minor(mdev
));
2242 /* paranoia asserts */
2243 D_ASSERT(mdev
->open_cnt
== 0);
2244 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2245 /* end paranoia asserts */
2247 del_gendisk(mdev
->vdisk
);
2249 /* cleanup stuff that may have been allocated during
2250 * device (re-)configuration or state changes */
2252 if (mdev
->this_bdev
)
2253 bdput(mdev
->this_bdev
);
2255 drbd_free_resources(mdev
);
2257 drbd_release_all_peer_reqs(mdev
);
2259 lc_destroy(mdev
->act_log
);
2260 lc_destroy(mdev
->resync
);
2262 kfree(mdev
->p_uuid
);
2263 /* mdev->p_uuid = NULL; */
2265 kfree(mdev
->current_epoch
);
2266 if (mdev
->bitmap
) /* should no longer be there. */
2267 drbd_bm_cleanup(mdev
);
2268 __free_page(mdev
->md_io_page
);
2269 put_disk(mdev
->vdisk
);
2270 blk_cleanup_queue(mdev
->rq_queue
);
2274 static void drbd_cleanup(void)
2277 struct drbd_conf
*mdev
;
2279 unregister_reboot_notifier(&drbd_notifier
);
2281 /* first remove proc,
2282 * drbdsetup uses it's presence to detect
2283 * whether DRBD is loaded.
2284 * If we would get stuck in proc removal,
2285 * but have netlink already deregistered,
2286 * some drbdsetup commands may wait forever
2290 remove_proc_entry("drbd", NULL
);
2292 drbd_genl_unregister();
2294 down_write(&drbd_cfg_rwsem
);
2295 idr_for_each_entry(&minors
, mdev
, i
)
2296 drbd_delete_device(mdev
);
2297 up_write(&drbd_cfg_rwsem
);
2299 drbd_destroy_mempools();
2300 unregister_blkdev(DRBD_MAJOR
, "drbd");
2302 idr_destroy(&minors
);
2304 printk(KERN_INFO
"drbd: module cleanup done.\n");
2308 * drbd_congested() - Callback for pdflush
2309 * @congested_data: User data
2310 * @bdi_bits: Bits pdflush is currently interested in
2312 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
2314 static int drbd_congested(void *congested_data
, int bdi_bits
)
2316 struct drbd_conf
*mdev
= congested_data
;
2317 struct request_queue
*q
;
2321 if (!may_inc_ap_bio(mdev
)) {
2322 /* DRBD has frozen IO */
2328 if (get_ldev(mdev
)) {
2329 q
= bdev_get_queue(mdev
->ldev
->backing_bdev
);
2330 r
= bdi_congested(&q
->backing_dev_info
, bdi_bits
);
2336 if (bdi_bits
& (1 << BDI_async_congested
) && test_bit(NET_CONGESTED
, &mdev
->tconn
->flags
)) {
2337 r
|= (1 << BDI_async_congested
);
2338 reason
= reason
== 'b' ? 'a' : 'n';
2342 mdev
->congestion_reason
= reason
;
2346 static void drbd_init_workqueue(struct drbd_work_queue
* wq
)
2348 sema_init(&wq
->s
, 0);
2349 spin_lock_init(&wq
->q_lock
);
2350 INIT_LIST_HEAD(&wq
->q
);
2353 struct drbd_tconn
*conn_by_name(const char *name
)
2355 struct drbd_tconn
*tconn
;
2357 if (!name
|| !name
[0])
2360 down_read(&drbd_cfg_rwsem
);
2361 list_for_each_entry(tconn
, &drbd_tconns
, all_tconn
) {
2362 if (!strcmp(tconn
->name
, name
))
2367 up_read(&drbd_cfg_rwsem
);
2371 static int drbd_alloc_socket(struct drbd_socket
*socket
)
2373 socket
->rbuf
= (void *) __get_free_page(GFP_KERNEL
);
2376 socket
->sbuf
= (void *) __get_free_page(GFP_KERNEL
);
2382 static void drbd_free_socket(struct drbd_socket
*socket
)
2384 free_page((unsigned long) socket
->sbuf
);
2385 free_page((unsigned long) socket
->rbuf
);
2388 void conn_free_crypto(struct drbd_tconn
*tconn
)
2390 crypto_free_hash(tconn
->cram_hmac_tfm
);
2391 crypto_free_hash(tconn
->integrity_w_tfm
);
2392 crypto_free_hash(tconn
->integrity_r_tfm
);
2393 kfree(tconn
->int_dig_in
);
2394 kfree(tconn
->int_dig_vv
);
2395 tconn
->cram_hmac_tfm
= NULL
;
2396 tconn
->integrity_w_tfm
= NULL
;
2397 tconn
->integrity_r_tfm
= NULL
;
2398 tconn
->int_dig_in
= NULL
;
2399 tconn
->int_dig_vv
= NULL
;
2402 struct drbd_tconn
*drbd_new_tconn(const char *name
)
2404 struct drbd_tconn
*tconn
;
2406 tconn
= kzalloc(sizeof(struct drbd_tconn
), GFP_KERNEL
);
2410 tconn
->name
= kstrdup(name
, GFP_KERNEL
);
2414 if (drbd_alloc_socket(&tconn
->data
))
2416 if (drbd_alloc_socket(&tconn
->meta
))
2419 if (!zalloc_cpumask_var(&tconn
->cpu_mask
, GFP_KERNEL
))
2422 if (!tl_init(tconn
))
2425 tconn
->cstate
= C_STANDALONE
;
2426 mutex_init(&tconn
->cstate_mutex
);
2427 spin_lock_init(&tconn
->req_lock
);
2428 mutex_init(&tconn
->net_conf_update
);
2429 init_waitqueue_head(&tconn
->ping_wait
);
2430 idr_init(&tconn
->volumes
);
2432 drbd_init_workqueue(&tconn
->data
.work
);
2433 mutex_init(&tconn
->data
.mutex
);
2435 drbd_init_workqueue(&tconn
->meta
.work
);
2436 mutex_init(&tconn
->meta
.mutex
);
2438 drbd_thread_init(tconn
, &tconn
->receiver
, drbdd_init
, "receiver");
2439 drbd_thread_init(tconn
, &tconn
->worker
, drbd_worker
, "worker");
2440 drbd_thread_init(tconn
, &tconn
->asender
, drbd_asender
, "asender");
2442 tconn
->res_opts
= (struct res_opts
) {
2443 {}, 0, /* cpu_mask */
2444 DRBD_ON_NO_DATA_DEF
, /* on_no_data */
2447 down_write(&drbd_cfg_rwsem
);
2448 list_add_tail(&tconn
->all_tconn
, &drbd_tconns
);
2449 up_write(&drbd_cfg_rwsem
);
2455 free_cpumask_var(tconn
->cpu_mask
);
2456 drbd_free_socket(&tconn
->meta
);
2457 drbd_free_socket(&tconn
->data
);
2464 void drbd_free_tconn(struct drbd_tconn
*tconn
)
2466 list_del(&tconn
->all_tconn
);
2467 idr_destroy(&tconn
->volumes
);
2469 free_cpumask_var(tconn
->cpu_mask
);
2470 drbd_free_socket(&tconn
->meta
);
2471 drbd_free_socket(&tconn
->data
);
2473 kfree(tconn
->int_dig_in
);
2474 kfree(tconn
->int_dig_vv
);
2478 enum drbd_ret_code
conn_new_minor(struct drbd_tconn
*tconn
, unsigned int minor
, int vnr
)
2480 struct drbd_conf
*mdev
;
2481 struct gendisk
*disk
;
2482 struct request_queue
*q
;
2484 int minor_got
= minor
;
2485 enum drbd_ret_code err
= ERR_NOMEM
;
2487 mdev
= minor_to_mdev(minor
);
2489 return ERR_MINOR_EXISTS
;
2491 /* GFP_KERNEL, we are outside of all write-out paths */
2492 mdev
= kzalloc(sizeof(struct drbd_conf
), GFP_KERNEL
);
2496 mdev
->tconn
= tconn
;
2497 mdev
->minor
= minor
;
2500 drbd_init_set_defaults(mdev
);
2502 q
= blk_alloc_queue(GFP_KERNEL
);
2506 q
->queuedata
= mdev
;
2508 disk
= alloc_disk(1);
2513 set_disk_ro(disk
, true);
2516 disk
->major
= DRBD_MAJOR
;
2517 disk
->first_minor
= minor
;
2518 disk
->fops
= &drbd_ops
;
2519 sprintf(disk
->disk_name
, "drbd%d", minor
);
2520 disk
->private_data
= mdev
;
2522 mdev
->this_bdev
= bdget(MKDEV(DRBD_MAJOR
, minor
));
2523 /* we have no partitions. we contain only ourselves. */
2524 mdev
->this_bdev
->bd_contains
= mdev
->this_bdev
;
2526 q
->backing_dev_info
.congested_fn
= drbd_congested
;
2527 q
->backing_dev_info
.congested_data
= mdev
;
2529 blk_queue_make_request(q
, drbd_make_request
);
2530 /* Setting the max_hw_sectors to an odd value of 8kibyte here
2531 This triggers a max_bio_size message upon first attach or connect */
2532 blk_queue_max_hw_sectors(q
, DRBD_MAX_BIO_SIZE_SAFE
>> 8);
2533 blk_queue_bounce_limit(q
, BLK_BOUNCE_ANY
);
2534 blk_queue_merge_bvec(q
, drbd_merge_bvec
);
2535 q
->queue_lock
= &mdev
->tconn
->req_lock
; /* needed since we use */
2537 mdev
->md_io_page
= alloc_page(GFP_KERNEL
);
2538 if (!mdev
->md_io_page
)
2539 goto out_no_io_page
;
2541 if (drbd_bm_init(mdev
))
2543 mdev
->read_requests
= RB_ROOT
;
2544 mdev
->write_requests
= RB_ROOT
;
2546 mdev
->current_epoch
= kzalloc(sizeof(struct drbd_epoch
), GFP_KERNEL
);
2547 if (!mdev
->current_epoch
)
2550 INIT_LIST_HEAD(&mdev
->current_epoch
->list
);
2553 if (!idr_pre_get(&minors
, GFP_KERNEL
))
2554 goto out_no_minor_idr
;
2555 if (idr_get_new_above(&minors
, mdev
, minor
, &minor_got
))
2556 goto out_no_minor_idr
;
2557 if (minor_got
!= minor
) {
2558 err
= ERR_MINOR_EXISTS
;
2559 drbd_msg_put_info("requested minor exists already");
2560 goto out_idr_remove_minor
;
2563 if (!idr_pre_get(&tconn
->volumes
, GFP_KERNEL
))
2564 goto out_idr_remove_minor
;
2565 if (idr_get_new_above(&tconn
->volumes
, mdev
, vnr
, &vnr_got
))
2566 goto out_idr_remove_minor
;
2567 if (vnr_got
!= vnr
) {
2568 err
= ERR_INVALID_REQUEST
;
2569 drbd_msg_put_info("requested volume exists already");
2570 goto out_idr_remove_vol
;
2574 /* inherit the connection state */
2575 mdev
->state
.conn
= tconn
->cstate
;
2576 if (mdev
->state
.conn
== C_WF_REPORT_PARAMS
)
2577 drbd_connected(vnr
, mdev
, tconn
);
2582 idr_remove(&tconn
->volumes
, vnr_got
);
2583 out_idr_remove_minor
:
2584 idr_remove(&minors
, minor_got
);
2587 kfree(mdev
->current_epoch
);
2589 drbd_bm_cleanup(mdev
);
2591 __free_page(mdev
->md_io_page
);
2595 blk_cleanup_queue(q
);
2601 int __init
drbd_init(void)
2605 if (minor_count
< DRBD_MINOR_COUNT_MIN
|| minor_count
> DRBD_MINOR_COUNT_MAX
) {
2607 "drbd: invalid minor_count (%d)\n", minor_count
);
2615 err
= register_blkdev(DRBD_MAJOR
, "drbd");
2618 "drbd: unable to register block device major %d\n",
2623 err
= drbd_genl_register();
2625 printk(KERN_ERR
"drbd: unable to register generic netlink family\n");
2630 register_reboot_notifier(&drbd_notifier
);
2633 * allocate all necessary structs
2637 init_waitqueue_head(&drbd_pp_wait
);
2639 drbd_proc
= NULL
; /* play safe for drbd_cleanup */
2642 err
= drbd_create_mempools();
2646 drbd_proc
= proc_create_data("drbd", S_IFREG
| S_IRUGO
, NULL
, &drbd_proc_fops
, NULL
);
2648 printk(KERN_ERR
"drbd: unable to register proc file\n");
2652 rwlock_init(&global_state_lock
);
2653 INIT_LIST_HEAD(&drbd_tconns
);
2655 printk(KERN_INFO
"drbd: initialized. "
2656 "Version: " REL_VERSION
" (api:%d/proto:%d-%d)\n",
2657 API_VERSION
, PRO_VERSION_MIN
, PRO_VERSION_MAX
);
2658 printk(KERN_INFO
"drbd: %s\n", drbd_buildtag());
2659 printk(KERN_INFO
"drbd: registered as block device major %d\n",
2662 return 0; /* Success! */
2667 /* currently always the case */
2668 printk(KERN_ERR
"drbd: ran out of memory\n");
2670 printk(KERN_ERR
"drbd: initialization failure\n");
2674 void drbd_free_bc(struct drbd_backing_dev
*ldev
)
2679 blkdev_put(ldev
->backing_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2680 blkdev_put(ldev
->md_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2685 void drbd_free_sock(struct drbd_tconn
*tconn
)
2687 if (tconn
->data
.socket
) {
2688 mutex_lock(&tconn
->data
.mutex
);
2689 kernel_sock_shutdown(tconn
->data
.socket
, SHUT_RDWR
);
2690 sock_release(tconn
->data
.socket
);
2691 tconn
->data
.socket
= NULL
;
2692 mutex_unlock(&tconn
->data
.mutex
);
2694 if (tconn
->meta
.socket
) {
2695 mutex_lock(&tconn
->meta
.mutex
);
2696 kernel_sock_shutdown(tconn
->meta
.socket
, SHUT_RDWR
);
2697 sock_release(tconn
->meta
.socket
);
2698 tconn
->meta
.socket
= NULL
;
2699 mutex_unlock(&tconn
->meta
.mutex
);
2704 void drbd_free_resources(struct drbd_conf
*mdev
)
2706 crypto_free_hash(mdev
->tconn
->csums_tfm
);
2707 mdev
->tconn
->csums_tfm
= NULL
;
2708 crypto_free_hash(mdev
->tconn
->verify_tfm
);
2709 mdev
->tconn
->verify_tfm
= NULL
;
2710 crypto_free_hash(mdev
->tconn
->cram_hmac_tfm
);
2711 mdev
->tconn
->cram_hmac_tfm
= NULL
;
2712 crypto_free_hash(mdev
->tconn
->integrity_w_tfm
);
2713 mdev
->tconn
->integrity_w_tfm
= NULL
;
2714 crypto_free_hash(mdev
->tconn
->integrity_r_tfm
);
2715 mdev
->tconn
->integrity_r_tfm
= NULL
;
2717 drbd_free_sock(mdev
->tconn
);
2720 drbd_free_bc(mdev
->ldev
);
2721 mdev
->ldev
= NULL
;);
2724 /* meta data management */
2726 struct meta_data_on_disk
{
2727 u64 la_size
; /* last agreed size. */
2728 u64 uuid
[UI_SIZE
]; /* UUIDs. */
2731 u32 flags
; /* MDF */
2734 u32 al_offset
; /* offset to this block */
2735 u32 al_nr_extents
; /* important for restoring the AL */
2736 /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
2737 u32 bm_offset
; /* offset to the bitmap, from here */
2738 u32 bm_bytes_per_bit
; /* BM_BLOCK_SIZE */
2739 u32 la_peer_max_bio_size
; /* last peer max_bio_size */
2740 u32 reserved_u32
[3];
2745 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2746 * @mdev: DRBD device.
2748 void drbd_md_sync(struct drbd_conf
*mdev
)
2750 struct meta_data_on_disk
*buffer
;
2754 del_timer(&mdev
->md_sync_timer
);
2755 /* timer may be rearmed by drbd_md_mark_dirty() now. */
2756 if (!test_and_clear_bit(MD_DIRTY
, &mdev
->flags
))
2759 /* We use here D_FAILED and not D_ATTACHING because we try to write
2760 * metadata even if we detach due to a disk failure! */
2761 if (!get_ldev_if_state(mdev
, D_FAILED
))
2764 mutex_lock(&mdev
->md_io_mutex
);
2765 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2766 memset(buffer
, 0, 512);
2768 buffer
->la_size
= cpu_to_be64(drbd_get_capacity(mdev
->this_bdev
));
2769 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2770 buffer
->uuid
[i
] = cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]);
2771 buffer
->flags
= cpu_to_be32(mdev
->ldev
->md
.flags
);
2772 buffer
->magic
= cpu_to_be32(DRBD_MD_MAGIC
);
2774 buffer
->md_size_sect
= cpu_to_be32(mdev
->ldev
->md
.md_size_sect
);
2775 buffer
->al_offset
= cpu_to_be32(mdev
->ldev
->md
.al_offset
);
2776 buffer
->al_nr_extents
= cpu_to_be32(mdev
->act_log
->nr_elements
);
2777 buffer
->bm_bytes_per_bit
= cpu_to_be32(BM_BLOCK_SIZE
);
2778 buffer
->device_uuid
= cpu_to_be64(mdev
->ldev
->md
.device_uuid
);
2780 buffer
->bm_offset
= cpu_to_be32(mdev
->ldev
->md
.bm_offset
);
2781 buffer
->la_peer_max_bio_size
= cpu_to_be32(mdev
->peer_max_bio_size
);
2783 D_ASSERT(drbd_md_ss__(mdev
, mdev
->ldev
) == mdev
->ldev
->md
.md_offset
);
2784 sector
= mdev
->ldev
->md
.md_offset
;
2786 if (drbd_md_sync_page_io(mdev
, mdev
->ldev
, sector
, WRITE
)) {
2787 /* this was a try anyways ... */
2788 dev_err(DEV
, "meta data update failed!\n");
2789 drbd_chk_io_error(mdev
, 1, true);
2792 /* Update mdev->ldev->md.la_size_sect,
2793 * since we updated it on metadata. */
2794 mdev
->ldev
->md
.la_size_sect
= drbd_get_capacity(mdev
->this_bdev
);
2796 mutex_unlock(&mdev
->md_io_mutex
);
2801 * drbd_md_read() - Reads in the meta data super block
2802 * @mdev: DRBD device.
2803 * @bdev: Device from which the meta data should be read in.
2805 * Return 0 (NO_ERROR) on success, and an enum drbd_ret_code in case
2806 * something goes wrong. Currently only: ERR_IO_MD_DISK, ERR_MD_INVALID.
2808 int drbd_md_read(struct drbd_conf
*mdev
, struct drbd_backing_dev
*bdev
)
2810 struct meta_data_on_disk
*buffer
;
2811 int i
, rv
= NO_ERROR
;
2813 if (!get_ldev_if_state(mdev
, D_ATTACHING
))
2814 return ERR_IO_MD_DISK
;
2816 mutex_lock(&mdev
->md_io_mutex
);
2817 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2819 if (drbd_md_sync_page_io(mdev
, bdev
, bdev
->md
.md_offset
, READ
)) {
2820 /* NOTE: can't do normal error processing here as this is
2821 called BEFORE disk is attached */
2822 dev_err(DEV
, "Error while reading metadata.\n");
2823 rv
= ERR_IO_MD_DISK
;
2827 if (buffer
->magic
!= cpu_to_be32(DRBD_MD_MAGIC
)) {
2828 dev_err(DEV
, "Error while reading metadata, magic not found.\n");
2829 rv
= ERR_MD_INVALID
;
2832 if (be32_to_cpu(buffer
->al_offset
) != bdev
->md
.al_offset
) {
2833 dev_err(DEV
, "unexpected al_offset: %d (expected %d)\n",
2834 be32_to_cpu(buffer
->al_offset
), bdev
->md
.al_offset
);
2835 rv
= ERR_MD_INVALID
;
2838 if (be32_to_cpu(buffer
->bm_offset
) != bdev
->md
.bm_offset
) {
2839 dev_err(DEV
, "unexpected bm_offset: %d (expected %d)\n",
2840 be32_to_cpu(buffer
->bm_offset
), bdev
->md
.bm_offset
);
2841 rv
= ERR_MD_INVALID
;
2844 if (be32_to_cpu(buffer
->md_size_sect
) != bdev
->md
.md_size_sect
) {
2845 dev_err(DEV
, "unexpected md_size: %u (expected %u)\n",
2846 be32_to_cpu(buffer
->md_size_sect
), bdev
->md
.md_size_sect
);
2847 rv
= ERR_MD_INVALID
;
2851 if (be32_to_cpu(buffer
->bm_bytes_per_bit
) != BM_BLOCK_SIZE
) {
2852 dev_err(DEV
, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
2853 be32_to_cpu(buffer
->bm_bytes_per_bit
), BM_BLOCK_SIZE
);
2854 rv
= ERR_MD_INVALID
;
2858 bdev
->md
.la_size_sect
= be64_to_cpu(buffer
->la_size
);
2859 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2860 bdev
->md
.uuid
[i
] = be64_to_cpu(buffer
->uuid
[i
]);
2861 bdev
->md
.flags
= be32_to_cpu(buffer
->flags
);
2862 bdev
->dc
.al_extents
= be32_to_cpu(buffer
->al_nr_extents
);
2863 bdev
->md
.device_uuid
= be64_to_cpu(buffer
->device_uuid
);
2865 spin_lock_irq(&mdev
->tconn
->req_lock
);
2866 if (mdev
->state
.conn
< C_CONNECTED
) {
2868 peer
= be32_to_cpu(buffer
->la_peer_max_bio_size
);
2869 peer
= max_t(int, peer
, DRBD_MAX_BIO_SIZE_SAFE
);
2870 mdev
->peer_max_bio_size
= peer
;
2872 spin_unlock_irq(&mdev
->tconn
->req_lock
);
2874 if (bdev
->dc
.al_extents
< 7)
2875 bdev
->dc
.al_extents
= 127;
2878 mutex_unlock(&mdev
->md_io_mutex
);
2885 * drbd_md_mark_dirty() - Mark meta data super block as dirty
2886 * @mdev: DRBD device.
2888 * Call this function if you change anything that should be written to
2889 * the meta-data super block. This function sets MD_DIRTY, and starts a
2890 * timer that ensures that within five seconds you have to call drbd_md_sync().
2893 void drbd_md_mark_dirty_(struct drbd_conf
*mdev
, unsigned int line
, const char *func
)
2895 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
)) {
2896 mod_timer(&mdev
->md_sync_timer
, jiffies
+ HZ
);
2897 mdev
->last_md_mark_dirty
.line
= line
;
2898 mdev
->last_md_mark_dirty
.func
= func
;
2902 void drbd_md_mark_dirty(struct drbd_conf
*mdev
)
2904 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
))
2905 mod_timer(&mdev
->md_sync_timer
, jiffies
+ 5*HZ
);
2909 static void drbd_uuid_move_history(struct drbd_conf
*mdev
) __must_hold(local
)
2913 for (i
= UI_HISTORY_START
; i
< UI_HISTORY_END
; i
++)
2914 mdev
->ldev
->md
.uuid
[i
+1] = mdev
->ldev
->md
.uuid
[i
];
2917 void _drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2919 if (idx
== UI_CURRENT
) {
2920 if (mdev
->state
.role
== R_PRIMARY
)
2925 drbd_set_ed_uuid(mdev
, val
);
2928 mdev
->ldev
->md
.uuid
[idx
] = val
;
2929 drbd_md_mark_dirty(mdev
);
2933 void drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2935 if (mdev
->ldev
->md
.uuid
[idx
]) {
2936 drbd_uuid_move_history(mdev
);
2937 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[idx
];
2939 _drbd_uuid_set(mdev
, idx
, val
);
2943 * drbd_uuid_new_current() - Creates a new current UUID
2944 * @mdev: DRBD device.
2946 * Creates a new current UUID, and rotates the old current UUID into
2947 * the bitmap slot. Causes an incremental resync upon next connect.
2949 void drbd_uuid_new_current(struct drbd_conf
*mdev
) __must_hold(local
)
2952 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2955 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2957 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = mdev
->ldev
->md
.uuid
[UI_CURRENT
];
2959 get_random_bytes(&val
, sizeof(u64
));
2960 _drbd_uuid_set(mdev
, UI_CURRENT
, val
);
2961 drbd_print_uuids(mdev
, "new current UUID");
2962 /* get it to stable storage _now_ */
2966 void drbd_uuid_set_bm(struct drbd_conf
*mdev
, u64 val
) __must_hold(local
)
2968 if (mdev
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && val
== 0)
2972 drbd_uuid_move_history(mdev
);
2973 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2974 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = 0;
2976 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2978 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2980 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = val
& ~((u64
)1);
2982 drbd_md_mark_dirty(mdev
);
2986 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2987 * @mdev: DRBD device.
2989 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
2991 int drbd_bmio_set_n_write(struct drbd_conf
*mdev
)
2995 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2996 drbd_md_set_flag(mdev
, MDF_FULL_SYNC
);
2998 drbd_bm_set_all(mdev
);
3000 rv
= drbd_bm_write(mdev
);
3003 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
3014 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
3015 * @mdev: DRBD device.
3017 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
3019 int drbd_bmio_clear_n_write(struct drbd_conf
*mdev
)
3023 drbd_resume_al(mdev
);
3024 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
3025 drbd_bm_clear_all(mdev
);
3026 rv
= drbd_bm_write(mdev
);
3033 static int w_bitmap_io(struct drbd_work
*w
, int unused
)
3035 struct bm_io_work
*work
= container_of(w
, struct bm_io_work
, w
);
3036 struct drbd_conf
*mdev
= w
->mdev
;
3039 D_ASSERT(atomic_read(&mdev
->ap_bio_cnt
) == 0);
3041 if (get_ldev(mdev
)) {
3042 drbd_bm_lock(mdev
, work
->why
, work
->flags
);
3043 rv
= work
->io_fn(mdev
);
3044 drbd_bm_unlock(mdev
);
3048 clear_bit_unlock(BITMAP_IO
, &mdev
->flags
);
3049 wake_up(&mdev
->misc_wait
);
3052 work
->done(mdev
, rv
);
3054 clear_bit(BITMAP_IO_QUEUED
, &mdev
->flags
);
3061 void drbd_ldev_destroy(struct drbd_conf
*mdev
)
3063 lc_destroy(mdev
->resync
);
3064 mdev
->resync
= NULL
;
3065 lc_destroy(mdev
->act_log
);
3066 mdev
->act_log
= NULL
;
3068 drbd_free_bc(mdev
->ldev
);
3069 mdev
->ldev
= NULL
;);
3071 clear_bit(GO_DISKLESS
, &mdev
->flags
);
3074 static int w_go_diskless(struct drbd_work
*w
, int unused
)
3076 struct drbd_conf
*mdev
= w
->mdev
;
3078 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3079 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
3080 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
3081 * the protected members anymore, though, so once put_ldev reaches zero
3082 * again, it will be safe to free them. */
3083 drbd_force_state(mdev
, NS(disk
, D_DISKLESS
));
3087 void drbd_go_diskless(struct drbd_conf
*mdev
)
3089 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
3090 if (!test_and_set_bit(GO_DISKLESS
, &mdev
->flags
))
3091 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->go_diskless
);
3095 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
3096 * @mdev: DRBD device.
3097 * @io_fn: IO callback to be called when bitmap IO is possible
3098 * @done: callback to be called after the bitmap IO was performed
3099 * @why: Descriptive text of the reason for doing the IO
3101 * While IO on the bitmap happens we freeze application IO thus we ensure
3102 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
3103 * called from worker context. It MUST NOT be used while a previous such
3104 * work is still pending!
3106 void drbd_queue_bitmap_io(struct drbd_conf
*mdev
,
3107 int (*io_fn
)(struct drbd_conf
*),
3108 void (*done
)(struct drbd_conf
*, int),
3109 char *why
, enum bm_flag flags
)
3111 D_ASSERT(current
== mdev
->tconn
->worker
.task
);
3113 D_ASSERT(!test_bit(BITMAP_IO_QUEUED
, &mdev
->flags
));
3114 D_ASSERT(!test_bit(BITMAP_IO
, &mdev
->flags
));
3115 D_ASSERT(list_empty(&mdev
->bm_io_work
.w
.list
));
3116 if (mdev
->bm_io_work
.why
)
3117 dev_err(DEV
, "FIXME going to queue '%s' but '%s' still pending?\n",
3118 why
, mdev
->bm_io_work
.why
);
3120 mdev
->bm_io_work
.io_fn
= io_fn
;
3121 mdev
->bm_io_work
.done
= done
;
3122 mdev
->bm_io_work
.why
= why
;
3123 mdev
->bm_io_work
.flags
= flags
;
3125 spin_lock_irq(&mdev
->tconn
->req_lock
);
3126 set_bit(BITMAP_IO
, &mdev
->flags
);
3127 if (atomic_read(&mdev
->ap_bio_cnt
) == 0) {
3128 if (!test_and_set_bit(BITMAP_IO_QUEUED
, &mdev
->flags
))
3129 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->bm_io_work
.w
);
3131 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3135 * drbd_bitmap_io() - Does an IO operation on the whole bitmap
3136 * @mdev: DRBD device.
3137 * @io_fn: IO callback to be called when bitmap IO is possible
3138 * @why: Descriptive text of the reason for doing the IO
3140 * freezes application IO while that the actual IO operations runs. This
3141 * functions MAY NOT be called from worker context.
3143 int drbd_bitmap_io(struct drbd_conf
*mdev
, int (*io_fn
)(struct drbd_conf
*),
3144 char *why
, enum bm_flag flags
)
3148 D_ASSERT(current
!= mdev
->tconn
->worker
.task
);
3150 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3151 drbd_suspend_io(mdev
);
3153 drbd_bm_lock(mdev
, why
, flags
);
3155 drbd_bm_unlock(mdev
);
3157 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3158 drbd_resume_io(mdev
);
3163 void drbd_md_set_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3165 if ((mdev
->ldev
->md
.flags
& flag
) != flag
) {
3166 drbd_md_mark_dirty(mdev
);
3167 mdev
->ldev
->md
.flags
|= flag
;
3171 void drbd_md_clear_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3173 if ((mdev
->ldev
->md
.flags
& flag
) != 0) {
3174 drbd_md_mark_dirty(mdev
);
3175 mdev
->ldev
->md
.flags
&= ~flag
;
3178 int drbd_md_test_flag(struct drbd_backing_dev
*bdev
, int flag
)
3180 return (bdev
->md
.flags
& flag
) != 0;
3183 static void md_sync_timer_fn(unsigned long data
)
3185 struct drbd_conf
*mdev
= (struct drbd_conf
*) data
;
3187 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &mdev
->md_sync_work
);
3190 static int w_md_sync(struct drbd_work
*w
, int unused
)
3192 struct drbd_conf
*mdev
= w
->mdev
;
3194 dev_warn(DEV
, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3196 dev_warn(DEV
, "last md_mark_dirty: %s:%u\n",
3197 mdev
->last_md_mark_dirty
.func
, mdev
->last_md_mark_dirty
.line
);
3203 const char *cmdname(enum drbd_packet cmd
)
3205 /* THINK may need to become several global tables
3206 * when we want to support more than
3207 * one PRO_VERSION */
3208 static const char *cmdnames
[] = {
3210 [P_DATA_REPLY
] = "DataReply",
3211 [P_RS_DATA_REPLY
] = "RSDataReply",
3212 [P_BARRIER
] = "Barrier",
3213 [P_BITMAP
] = "ReportBitMap",
3214 [P_BECOME_SYNC_TARGET
] = "BecomeSyncTarget",
3215 [P_BECOME_SYNC_SOURCE
] = "BecomeSyncSource",
3216 [P_UNPLUG_REMOTE
] = "UnplugRemote",
3217 [P_DATA_REQUEST
] = "DataRequest",
3218 [P_RS_DATA_REQUEST
] = "RSDataRequest",
3219 [P_SYNC_PARAM
] = "SyncParam",
3220 [P_SYNC_PARAM89
] = "SyncParam89",
3221 [P_PROTOCOL
] = "ReportProtocol",
3222 [P_UUIDS
] = "ReportUUIDs",
3223 [P_SIZES
] = "ReportSizes",
3224 [P_STATE
] = "ReportState",
3225 [P_SYNC_UUID
] = "ReportSyncUUID",
3226 [P_AUTH_CHALLENGE
] = "AuthChallenge",
3227 [P_AUTH_RESPONSE
] = "AuthResponse",
3229 [P_PING_ACK
] = "PingAck",
3230 [P_RECV_ACK
] = "RecvAck",
3231 [P_WRITE_ACK
] = "WriteAck",
3232 [P_RS_WRITE_ACK
] = "RSWriteAck",
3233 [P_DISCARD_WRITE
] = "DiscardWrite",
3234 [P_NEG_ACK
] = "NegAck",
3235 [P_NEG_DREPLY
] = "NegDReply",
3236 [P_NEG_RS_DREPLY
] = "NegRSDReply",
3237 [P_BARRIER_ACK
] = "BarrierAck",
3238 [P_STATE_CHG_REQ
] = "StateChgRequest",
3239 [P_STATE_CHG_REPLY
] = "StateChgReply",
3240 [P_OV_REQUEST
] = "OVRequest",
3241 [P_OV_REPLY
] = "OVReply",
3242 [P_OV_RESULT
] = "OVResult",
3243 [P_CSUM_RS_REQUEST
] = "CsumRSRequest",
3244 [P_RS_IS_IN_SYNC
] = "CsumRSIsInSync",
3245 [P_COMPRESSED_BITMAP
] = "CBitmap",
3246 [P_DELAY_PROBE
] = "DelayProbe",
3247 [P_OUT_OF_SYNC
] = "OutOfSync",
3248 [P_RETRY_WRITE
] = "RetryWrite",
3251 if (cmd
== P_INITIAL_META
)
3252 return "InitialMeta";
3253 if (cmd
== P_INITIAL_DATA
)
3254 return "InitialData";
3255 if (cmd
== P_CONNECTION_FEATURES
)
3256 return "ConnectionFeatures";
3257 if (cmd
>= ARRAY_SIZE(cmdnames
))
3259 return cmdnames
[cmd
];
3263 * drbd_wait_misc - wait for a request to make progress
3264 * @mdev: device associated with the request
3265 * @i: the struct drbd_interval embedded in struct drbd_request or
3266 * struct drbd_peer_request
3268 int drbd_wait_misc(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
3270 struct net_conf
*nc
;
3275 nc
= rcu_dereference(mdev
->tconn
->net_conf
);
3280 timeout
= nc
->ko_count
? nc
->timeout
* HZ
/ 10 * nc
->ko_count
: MAX_SCHEDULE_TIMEOUT
;
3283 /* Indicate to wake up mdev->misc_wait on progress. */
3285 prepare_to_wait(&mdev
->misc_wait
, &wait
, TASK_INTERRUPTIBLE
);
3286 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3287 timeout
= schedule_timeout(timeout
);
3288 finish_wait(&mdev
->misc_wait
, &wait
);
3289 spin_lock_irq(&mdev
->tconn
->req_lock
);
3290 if (!timeout
|| mdev
->state
.conn
< C_CONNECTED
)
3292 if (signal_pending(current
))
3293 return -ERESTARTSYS
;
3297 #ifdef CONFIG_DRBD_FAULT_INJECTION
3298 /* Fault insertion support including random number generator shamelessly
3299 * stolen from kernel/rcutorture.c */
3300 struct fault_random_state
{
3301 unsigned long state
;
3302 unsigned long count
;
3305 #define FAULT_RANDOM_MULT 39916801 /* prime */
3306 #define FAULT_RANDOM_ADD 479001701 /* prime */
3307 #define FAULT_RANDOM_REFRESH 10000
3310 * Crude but fast random-number generator. Uses a linear congruential
3311 * generator, with occasional help from get_random_bytes().
3313 static unsigned long
3314 _drbd_fault_random(struct fault_random_state
*rsp
)
3318 if (!rsp
->count
--) {
3319 get_random_bytes(&refresh
, sizeof(refresh
));
3320 rsp
->state
+= refresh
;
3321 rsp
->count
= FAULT_RANDOM_REFRESH
;
3323 rsp
->state
= rsp
->state
* FAULT_RANDOM_MULT
+ FAULT_RANDOM_ADD
;
3324 return swahw32(rsp
->state
);
3328 _drbd_fault_str(unsigned int type
) {
3329 static char *_faults
[] = {
3330 [DRBD_FAULT_MD_WR
] = "Meta-data write",
3331 [DRBD_FAULT_MD_RD
] = "Meta-data read",
3332 [DRBD_FAULT_RS_WR
] = "Resync write",
3333 [DRBD_FAULT_RS_RD
] = "Resync read",
3334 [DRBD_FAULT_DT_WR
] = "Data write",
3335 [DRBD_FAULT_DT_RD
] = "Data read",
3336 [DRBD_FAULT_DT_RA
] = "Data read ahead",
3337 [DRBD_FAULT_BM_ALLOC
] = "BM allocation",
3338 [DRBD_FAULT_AL_EE
] = "EE allocation",
3339 [DRBD_FAULT_RECEIVE
] = "receive data corruption",
3342 return (type
< DRBD_FAULT_MAX
) ? _faults
[type
] : "**Unknown**";
3346 _drbd_insert_fault(struct drbd_conf
*mdev
, unsigned int type
)
3348 static struct fault_random_state rrs
= {0, 0};
3350 unsigned int ret
= (
3352 ((1 << mdev_to_minor(mdev
)) & fault_devs
) != 0) &&
3353 (((_drbd_fault_random(&rrs
) % 100) + 1) <= fault_rate
));
3358 if (__ratelimit(&drbd_ratelimit_state
))
3359 dev_warn(DEV
, "***Simulating %s failure\n",
3360 _drbd_fault_str(type
));
3367 const char *drbd_buildtag(void)
3369 /* DRBD built from external sources has here a reference to the
3370 git hash of the source code. */
3372 static char buildtag
[38] = "\0uilt-in";
3374 if (buildtag
[0] == 0) {
3375 #ifdef CONFIG_MODULES
3376 if (THIS_MODULE
!= NULL
)
3377 sprintf(buildtag
, "srcversion: %-24s", THIS_MODULE
->srcversion
);
3386 module_init(drbd_init
)
3387 module_exit(drbd_cleanup
)
3389 EXPORT_SYMBOL(drbd_conn_str
);
3390 EXPORT_SYMBOL(drbd_role_str
);
3391 EXPORT_SYMBOL(drbd_disk_str
);
3392 EXPORT_SYMBOL(drbd_set_st_err_str
);