4 This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
6 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
7 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
8 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
10 Thanks to Carter Burden, Bart Grantham and Gennadiy Nerubayev
11 from Logicworks, Inc. for making SDP replication support possible.
13 drbd is free software; you can redistribute it and/or modify
14 it under the terms of the GNU General Public License as published by
15 the Free Software Foundation; either version 2, or (at your option)
18 drbd is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
23 You should have received a copy of the GNU General Public License
24 along with drbd; see the file COPYING. If not, write to
25 the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA.
29 #include <linux/module.h>
30 #include <linux/drbd.h>
31 #include <asm/uaccess.h>
32 #include <asm/types.h>
34 #include <linux/ctype.h>
35 #include <linux/mutex.h>
37 #include <linux/file.h>
38 #include <linux/proc_fs.h>
39 #include <linux/init.h>
41 #include <linux/memcontrol.h>
42 #include <linux/mm_inline.h>
43 #include <linux/slab.h>
44 #include <linux/random.h>
45 #include <linux/reboot.h>
46 #include <linux/notifier.h>
47 #include <linux/kthread.h>
49 #define __KERNEL_SYSCALLS__
50 #include <linux/unistd.h>
51 #include <linux/vmalloc.h>
53 #include <linux/drbd_limits.h>
55 #include "drbd_req.h" /* only for _req_mod in tl_release and tl_clear */
59 static DEFINE_MUTEX(drbd_main_mutex
);
60 int drbdd_init(struct drbd_thread
*);
61 int drbd_worker(struct drbd_thread
*);
62 int drbd_asender(struct drbd_thread
*);
65 static int drbd_open(struct block_device
*bdev
, fmode_t mode
);
66 static int drbd_release(struct gendisk
*gd
, fmode_t mode
);
67 static int w_md_sync(struct drbd_work
*w
, int unused
);
68 static void md_sync_timer_fn(unsigned long data
);
69 static int w_bitmap_io(struct drbd_work
*w
, int unused
);
70 static int w_go_diskless(struct drbd_work
*w
, int unused
);
72 MODULE_AUTHOR("Philipp Reisner <phil@linbit.com>, "
73 "Lars Ellenberg <lars@linbit.com>");
74 MODULE_DESCRIPTION("drbd - Distributed Replicated Block Device v" REL_VERSION
);
75 MODULE_VERSION(REL_VERSION
);
76 MODULE_LICENSE("GPL");
77 MODULE_PARM_DESC(minor_count
, "Approximate number of drbd devices ("
78 __stringify(DRBD_MINOR_COUNT_MIN
) "-" __stringify(DRBD_MINOR_COUNT_MAX
) ")");
79 MODULE_ALIAS_BLOCKDEV_MAJOR(DRBD_MAJOR
);
81 #include <linux/moduleparam.h>
82 /* allow_open_on_secondary */
83 MODULE_PARM_DESC(allow_oos
, "DONT USE!");
84 /* thanks to these macros, if compiled into the kernel (not-module),
85 * this becomes the boot parameter drbd.minor_count */
86 module_param(minor_count
, uint
, 0444);
87 module_param(disable_sendpage
, bool, 0644);
88 module_param(allow_oos
, bool, 0);
89 module_param(proc_details
, int, 0644);
91 #ifdef CONFIG_DRBD_FAULT_INJECTION
94 static int fault_count
;
96 /* bitmap of enabled faults */
97 module_param(enable_faults
, int, 0664);
98 /* fault rate % value - applies to all enabled faults */
99 module_param(fault_rate
, int, 0664);
100 /* count of faults inserted */
101 module_param(fault_count
, int, 0664);
102 /* bitmap of devices to insert faults on */
103 module_param(fault_devs
, int, 0644);
106 /* module parameter, defined */
107 unsigned int minor_count
= DRBD_MINOR_COUNT_DEF
;
108 int disable_sendpage
;
110 int proc_details
; /* Detail level in proc drbd*/
112 /* Module parameter for setting the user mode helper program
113 * to run. Default is /sbin/drbdadm */
114 char usermode_helper
[80] = "/sbin/drbdadm";
116 module_param_string(usermode_helper
, usermode_helper
, sizeof(usermode_helper
), 0644);
118 /* in 2.6.x, our device mapping and config info contains our virtual gendisks
119 * as member "struct gendisk *vdisk;"
122 struct list_head drbd_tconns
; /* list of struct drbd_tconn */
123 DEFINE_MUTEX(drbd_cfg_mutex
);
125 struct kmem_cache
*drbd_request_cache
;
126 struct kmem_cache
*drbd_ee_cache
; /* peer requests */
127 struct kmem_cache
*drbd_bm_ext_cache
; /* bitmap extents */
128 struct kmem_cache
*drbd_al_ext_cache
; /* activity log extents */
129 mempool_t
*drbd_request_mempool
;
130 mempool_t
*drbd_ee_mempool
;
131 mempool_t
*drbd_md_io_page_pool
;
132 struct bio_set
*drbd_md_io_bio_set
;
134 /* I do not use a standard mempool, because:
135 1) I want to hand out the pre-allocated objects first.
136 2) I want to be able to interrupt sleeping allocation with a signal.
137 Note: This is a single linked list, the next pointer is the private
138 member of struct page.
140 struct page
*drbd_pp_pool
;
141 spinlock_t drbd_pp_lock
;
143 wait_queue_head_t drbd_pp_wait
;
145 DEFINE_RATELIMIT_STATE(drbd_ratelimit_state
, 5 * HZ
, 5);
147 static const struct block_device_operations drbd_ops
= {
148 .owner
= THIS_MODULE
,
150 .release
= drbd_release
,
153 static void bio_destructor_drbd(struct bio
*bio
)
155 bio_free(bio
, drbd_md_io_bio_set
);
158 struct bio
*bio_alloc_drbd(gfp_t gfp_mask
)
162 if (!drbd_md_io_bio_set
)
163 return bio_alloc(gfp_mask
, 1);
165 bio
= bio_alloc_bioset(gfp_mask
, 1, drbd_md_io_bio_set
);
168 bio
->bi_destructor
= bio_destructor_drbd
;
173 /* When checking with sparse, and this is an inline function, sparse will
174 give tons of false positives. When this is a real functions sparse works.
176 int _get_ldev_if_state(struct drbd_conf
*mdev
, enum drbd_disk_state mins
)
180 atomic_inc(&mdev
->local_cnt
);
181 io_allowed
= (mdev
->state
.disk
>= mins
);
183 if (atomic_dec_and_test(&mdev
->local_cnt
))
184 wake_up(&mdev
->misc_wait
);
192 * DOC: The transfer log
194 * The transfer log is a single linked list of &struct drbd_tl_epoch objects.
195 * mdev->tconn->newest_tle points to the head, mdev->tconn->oldest_tle points to the tail
196 * of the list. There is always at least one &struct drbd_tl_epoch object.
198 * Each &struct drbd_tl_epoch has a circular double linked list of requests
201 static int tl_init(struct drbd_tconn
*tconn
)
203 struct drbd_tl_epoch
*b
;
205 /* during device minor initialization, we may well use GFP_KERNEL */
206 b
= kmalloc(sizeof(struct drbd_tl_epoch
), GFP_KERNEL
);
209 INIT_LIST_HEAD(&b
->requests
);
210 INIT_LIST_HEAD(&b
->w
.list
);
214 b
->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
216 tconn
->oldest_tle
= b
;
217 tconn
->newest_tle
= b
;
218 INIT_LIST_HEAD(&tconn
->out_of_sequence_requests
);
223 static void tl_cleanup(struct drbd_tconn
*tconn
)
225 if (tconn
->oldest_tle
!= tconn
->newest_tle
)
226 conn_err(tconn
, "ASSERT FAILED: oldest_tle == newest_tle\n");
227 if (!list_empty(&tconn
->out_of_sequence_requests
))
228 conn_err(tconn
, "ASSERT FAILED: list_empty(out_of_sequence_requests)\n");
229 kfree(tconn
->oldest_tle
);
230 tconn
->oldest_tle
= NULL
;
231 kfree(tconn
->unused_spare_tle
);
232 tconn
->unused_spare_tle
= NULL
;
236 * _tl_add_barrier() - Adds a barrier to the transfer log
237 * @mdev: DRBD device.
238 * @new: Barrier to be added before the current head of the TL.
240 * The caller must hold the req_lock.
242 void _tl_add_barrier(struct drbd_tconn
*tconn
, struct drbd_tl_epoch
*new)
244 struct drbd_tl_epoch
*newest_before
;
246 INIT_LIST_HEAD(&new->requests
);
247 INIT_LIST_HEAD(&new->w
.list
);
248 new->w
.cb
= NULL
; /* if this is != NULL, we need to dec_ap_pending in tl_clear */
252 newest_before
= tconn
->newest_tle
;
253 /* never send a barrier number == 0, because that is special-cased
254 * when using TCQ for our write ordering code */
255 new->br_number
= (newest_before
->br_number
+1) ?: 1;
256 if (tconn
->newest_tle
!= new) {
257 tconn
->newest_tle
->next
= new;
258 tconn
->newest_tle
= new;
263 * tl_release() - Free or recycle the oldest &struct drbd_tl_epoch object of the TL
264 * @mdev: DRBD device.
265 * @barrier_nr: Expected identifier of the DRBD write barrier packet.
266 * @set_size: Expected number of requests before that barrier.
268 * In case the passed barrier_nr or set_size does not match the oldest
269 * &struct drbd_tl_epoch objects this function will cause a termination
272 void tl_release(struct drbd_tconn
*tconn
, unsigned int barrier_nr
,
273 unsigned int set_size
)
275 struct drbd_conf
*mdev
;
276 struct drbd_tl_epoch
*b
, *nob
; /* next old barrier */
277 struct list_head
*le
, *tle
;
278 struct drbd_request
*r
;
280 spin_lock_irq(&tconn
->req_lock
);
282 b
= tconn
->oldest_tle
;
284 /* first some paranoia code */
286 conn_err(tconn
, "BAD! BarrierAck #%u received, but no epoch in tl!?\n",
290 if (b
->br_number
!= barrier_nr
) {
291 conn_err(tconn
, "BAD! BarrierAck #%u received, expected #%u!\n",
292 barrier_nr
, b
->br_number
);
295 if (b
->n_writes
!= set_size
) {
296 conn_err(tconn
, "BAD! BarrierAck #%u received with n_writes=%u, expected n_writes=%u!\n",
297 barrier_nr
, set_size
, b
->n_writes
);
301 /* Clean up list of requests processed during current epoch */
302 list_for_each_safe(le
, tle
, &b
->requests
) {
303 r
= list_entry(le
, struct drbd_request
, tl_requests
);
304 _req_mod(r
, BARRIER_ACKED
);
306 /* There could be requests on the list waiting for completion
307 of the write to the local disk. To avoid corruptions of
308 slab's data structures we have to remove the lists head.
310 Also there could have been a barrier ack out of sequence, overtaking
311 the write acks - which would be a bug and violating write ordering.
312 To not deadlock in case we lose connection while such requests are
313 still pending, we need some way to find them for the
314 _req_mode(CONNECTION_LOST_WHILE_PENDING).
316 These have been list_move'd to the out_of_sequence_requests list in
317 _req_mod(, BARRIER_ACKED) above.
319 list_del_init(&b
->requests
);
323 if (test_and_clear_bit(CREATE_BARRIER
, &mdev
->flags
)) {
324 _tl_add_barrier(tconn
, b
);
326 tconn
->oldest_tle
= nob
;
327 /* if nob == NULL b was the only barrier, and becomes the new
328 barrier. Therefore tconn->oldest_tle points already to b */
330 D_ASSERT(nob
!= NULL
);
331 tconn
->oldest_tle
= nob
;
335 spin_unlock_irq(&tconn
->req_lock
);
336 dec_ap_pending(mdev
);
341 spin_unlock_irq(&tconn
->req_lock
);
342 conn_request_state(tconn
, NS(conn
, C_PROTOCOL_ERROR
), CS_HARD
);
347 * _tl_restart() - Walks the transfer log, and applies an action to all requests
348 * @mdev: DRBD device.
349 * @what: The action/event to perform with all request objects
351 * @what might be one of CONNECTION_LOST_WHILE_PENDING, RESEND, FAIL_FROZEN_DISK_IO,
352 * RESTART_FROZEN_DISK_IO.
354 void _tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
356 struct drbd_tl_epoch
*b
, *tmp
, **pn
;
357 struct list_head
*le
, *tle
, carry_reads
;
358 struct drbd_request
*req
;
359 int rv
, n_writes
, n_reads
;
361 b
= tconn
->oldest_tle
;
362 pn
= &tconn
->oldest_tle
;
366 INIT_LIST_HEAD(&carry_reads
);
367 list_for_each_safe(le
, tle
, &b
->requests
) {
368 req
= list_entry(le
, struct drbd_request
, tl_requests
);
369 rv
= _req_mod(req
, what
);
371 n_writes
+= (rv
& MR_WRITE
) >> MR_WRITE_SHIFT
;
372 n_reads
+= (rv
& MR_READ
) >> MR_READ_SHIFT
;
377 if (what
== RESEND
) {
378 b
->n_writes
= n_writes
;
379 if (b
->w
.cb
== NULL
) {
380 b
->w
.cb
= w_send_barrier
;
381 inc_ap_pending(b
->w
.mdev
);
382 set_bit(CREATE_BARRIER
, &b
->w
.mdev
->flags
);
385 drbd_queue_work(&tconn
->data
.work
, &b
->w
);
390 list_add(&carry_reads
, &b
->requests
);
391 /* there could still be requests on that ring list,
392 * in case local io is still pending */
393 list_del(&b
->requests
);
395 /* dec_ap_pending corresponding to queue_barrier.
396 * the newest barrier may not have been queued yet,
397 * in which case w.cb is still NULL. */
399 dec_ap_pending(b
->w
.mdev
);
401 if (b
== tconn
->newest_tle
) {
402 /* recycle, but reinit! */
404 conn_err(tconn
, "ASSERT FAILED tmp == NULL");
405 INIT_LIST_HEAD(&b
->requests
);
406 list_splice(&carry_reads
, &b
->requests
);
407 INIT_LIST_HEAD(&b
->w
.list
);
409 b
->br_number
= net_random();
419 list_splice(&carry_reads
, &b
->requests
);
425 * tl_clear() - Clears all requests and &struct drbd_tl_epoch objects out of the TL
426 * @mdev: DRBD device.
428 * This is called after the connection to the peer was lost. The storage covered
429 * by the requests on the transfer gets marked as our of sync. Called from the
430 * receiver thread and the worker thread.
432 void tl_clear(struct drbd_tconn
*tconn
)
434 struct drbd_conf
*mdev
;
435 struct list_head
*le
, *tle
;
436 struct drbd_request
*r
;
439 spin_lock_irq(&tconn
->req_lock
);
441 _tl_restart(tconn
, CONNECTION_LOST_WHILE_PENDING
);
443 /* we expect this list to be empty. */
444 if (!list_empty(&tconn
->out_of_sequence_requests
))
445 conn_err(tconn
, "ASSERT FAILED list_empty(&out_of_sequence_requests)\n");
447 /* but just in case, clean it up anyways! */
448 list_for_each_safe(le
, tle
, &tconn
->out_of_sequence_requests
) {
449 r
= list_entry(le
, struct drbd_request
, tl_requests
);
450 /* It would be nice to complete outside of spinlock.
451 * But this is easier for now. */
452 _req_mod(r
, CONNECTION_LOST_WHILE_PENDING
);
455 /* ensure bit indicating barrier is required is clear */
456 idr_for_each_entry(&tconn
->volumes
, mdev
, vnr
)
457 clear_bit(CREATE_BARRIER
, &mdev
->flags
);
459 spin_unlock_irq(&tconn
->req_lock
);
462 void tl_restart(struct drbd_tconn
*tconn
, enum drbd_req_event what
)
464 spin_lock_irq(&tconn
->req_lock
);
465 _tl_restart(tconn
, what
);
466 spin_unlock_irq(&tconn
->req_lock
);
469 static int drbd_thread_setup(void *arg
)
471 struct drbd_thread
*thi
= (struct drbd_thread
*) arg
;
472 struct drbd_tconn
*tconn
= thi
->tconn
;
476 snprintf(current
->comm
, sizeof(current
->comm
), "drbd_%c_%s",
477 thi
->name
[0], thi
->tconn
->name
);
480 retval
= thi
->function(thi
);
482 spin_lock_irqsave(&thi
->t_lock
, flags
);
484 /* if the receiver has been "EXITING", the last thing it did
485 * was set the conn state to "StandAlone",
486 * if now a re-connect request comes in, conn state goes C_UNCONNECTED,
487 * and receiver thread will be "started".
488 * drbd_thread_start needs to set "RESTARTING" in that case.
489 * t_state check and assignment needs to be within the same spinlock,
490 * so either thread_start sees EXITING, and can remap to RESTARTING,
491 * or thread_start see NONE, and can proceed as normal.
494 if (thi
->t_state
== RESTARTING
) {
495 conn_info(tconn
, "Restarting %s thread\n", thi
->name
);
496 thi
->t_state
= RUNNING
;
497 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
504 complete(&thi
->stop
);
505 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
507 conn_info(tconn
, "Terminating %s\n", current
->comm
);
509 /* Release mod reference taken when thread was started */
510 module_put(THIS_MODULE
);
514 static void drbd_thread_init(struct drbd_tconn
*tconn
, struct drbd_thread
*thi
,
515 int (*func
) (struct drbd_thread
*), char *name
)
517 spin_lock_init(&thi
->t_lock
);
520 thi
->function
= func
;
522 strncpy(thi
->name
, name
, ARRAY_SIZE(thi
->name
));
525 int drbd_thread_start(struct drbd_thread
*thi
)
527 struct drbd_tconn
*tconn
= thi
->tconn
;
528 struct task_struct
*nt
;
531 /* is used from state engine doing drbd_thread_stop_nowait,
532 * while holding the req lock irqsave */
533 spin_lock_irqsave(&thi
->t_lock
, flags
);
535 switch (thi
->t_state
) {
537 conn_info(tconn
, "Starting %s thread (from %s [%d])\n",
538 thi
->name
, current
->comm
, current
->pid
);
540 /* Get ref on module for thread - this is released when thread exits */
541 if (!try_module_get(THIS_MODULE
)) {
542 conn_err(tconn
, "Failed to get module reference in drbd_thread_start\n");
543 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
547 init_completion(&thi
->stop
);
548 thi
->reset_cpu_mask
= 1;
549 thi
->t_state
= RUNNING
;
550 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
551 flush_signals(current
); /* otherw. may get -ERESTARTNOINTR */
553 nt
= kthread_create(drbd_thread_setup
, (void *) thi
,
554 "drbd_%c_%s", thi
->name
[0], thi
->tconn
->name
);
557 conn_err(tconn
, "Couldn't start thread\n");
559 module_put(THIS_MODULE
);
562 spin_lock_irqsave(&thi
->t_lock
, flags
);
564 thi
->t_state
= RUNNING
;
565 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
569 thi
->t_state
= RESTARTING
;
570 conn_info(tconn
, "Restarting %s thread (from %s [%d])\n",
571 thi
->name
, current
->comm
, current
->pid
);
576 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
584 void _drbd_thread_stop(struct drbd_thread
*thi
, int restart
, int wait
)
588 enum drbd_thread_state ns
= restart
? RESTARTING
: EXITING
;
590 /* may be called from state engine, holding the req lock irqsave */
591 spin_lock_irqsave(&thi
->t_lock
, flags
);
593 if (thi
->t_state
== NONE
) {
594 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
596 drbd_thread_start(thi
);
600 if (thi
->t_state
!= ns
) {
601 if (thi
->task
== NULL
) {
602 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
608 init_completion(&thi
->stop
);
609 if (thi
->task
!= current
)
610 force_sig(DRBD_SIGKILL
, thi
->task
);
613 spin_unlock_irqrestore(&thi
->t_lock
, flags
);
616 wait_for_completion(&thi
->stop
);
619 static struct drbd_thread
*drbd_task_to_thread(struct drbd_tconn
*tconn
, struct task_struct
*task
)
621 struct drbd_thread
*thi
=
622 task
== tconn
->receiver
.task
? &tconn
->receiver
:
623 task
== tconn
->asender
.task
? &tconn
->asender
:
624 task
== tconn
->worker
.task
? &tconn
->worker
: NULL
;
629 char *drbd_task_to_thread_name(struct drbd_tconn
*tconn
, struct task_struct
*task
)
631 struct drbd_thread
*thi
= drbd_task_to_thread(tconn
, task
);
632 return thi
? thi
->name
: task
->comm
;
635 int conn_lowest_minor(struct drbd_tconn
*tconn
)
638 struct drbd_conf
*mdev
;
640 mdev
= idr_get_next(&tconn
->volumes
, &vnr
);
643 return mdev_to_minor(mdev
);
648 * drbd_calc_cpu_mask() - Generate CPU masks, spread over all CPUs
649 * @mdev: DRBD device.
651 * Forces all threads of a device onto the same CPU. This is beneficial for
652 * DRBD's performance. May be overwritten by user's configuration.
654 void drbd_calc_cpu_mask(struct drbd_tconn
*tconn
)
659 if (cpumask_weight(tconn
->cpu_mask
))
662 ord
= conn_lowest_minor(tconn
) % cpumask_weight(cpu_online_mask
);
663 for_each_online_cpu(cpu
) {
665 cpumask_set_cpu(cpu
, tconn
->cpu_mask
);
669 /* should not be reached */
670 cpumask_setall(tconn
->cpu_mask
);
674 * drbd_thread_current_set_cpu() - modifies the cpu mask of the _current_ thread
675 * @mdev: DRBD device.
676 * @thi: drbd_thread object
678 * call in the "main loop" of _all_ threads, no need for any mutex, current won't die
681 void drbd_thread_current_set_cpu(struct drbd_thread
*thi
)
683 struct task_struct
*p
= current
;
685 if (!thi
->reset_cpu_mask
)
687 thi
->reset_cpu_mask
= 0;
688 set_cpus_allowed_ptr(p
, thi
->tconn
->cpu_mask
);
692 static void prepare_header80(struct p_header80
*h
, enum drbd_packet cmd
, int size
)
694 h
->magic
= cpu_to_be32(DRBD_MAGIC
);
695 h
->command
= cpu_to_be16(cmd
);
696 h
->length
= cpu_to_be16(size
);
699 static void prepare_header95(struct p_header95
*h
, enum drbd_packet cmd
, int size
)
701 h
->magic
= cpu_to_be16(DRBD_MAGIC_BIG
);
702 h
->command
= cpu_to_be16(cmd
);
703 h
->length
= cpu_to_be32(size
);
706 static void _prepare_header(struct drbd_tconn
*tconn
, int vnr
, struct p_header
*h
,
707 enum drbd_packet cmd
, int size
)
709 if (tconn
->agreed_pro_version
>= 100 || size
> DRBD_MAX_SIZE_H80_PACKET
)
710 prepare_header95(&h
->h95
, cmd
, size
);
712 prepare_header80(&h
->h80
, cmd
, size
);
715 static void prepare_header(struct drbd_conf
*mdev
, struct p_header
*h
,
716 enum drbd_packet cmd
, int size
)
718 _prepare_header(mdev
->tconn
, mdev
->vnr
, h
, cmd
, size
);
721 /* the appropriate socket mutex must be held already */
722 int _conn_send_cmd(struct drbd_tconn
*tconn
, int vnr
, struct socket
*sock
,
723 enum drbd_packet cmd
, struct p_header
*h
, size_t size
,
728 _prepare_header(tconn
, vnr
, h
, cmd
, size
- sizeof(struct p_header
));
729 err
= drbd_send_all(tconn
, sock
, h
, size
, msg_flags
);
730 if (err
&& !signal_pending(current
))
731 conn_warn(tconn
, "short send %s size=%d\n",
732 cmdname(cmd
), (int)size
);
736 /* don't pass the socket. we may only look at it
737 * when we hold the appropriate socket mutex.
739 int conn_send_cmd(struct drbd_tconn
*tconn
, int vnr
, struct drbd_socket
*sock
,
740 enum drbd_packet cmd
, struct p_header
*h
, size_t size
)
744 mutex_lock(&sock
->mutex
);
746 err
= _conn_send_cmd(tconn
, vnr
, sock
->socket
, cmd
, h
, size
, 0);
747 mutex_unlock(&sock
->mutex
);
751 int conn_send_cmd2(struct drbd_tconn
*tconn
, enum drbd_packet cmd
, char *data
,
757 prepare_header80(&h
, cmd
, size
);
758 err
= drbd_get_data_sock(tconn
);
760 err
= drbd_send_all(tconn
, tconn
->data
.socket
, &h
, sizeof(h
), 0);
762 err
= drbd_send_all(tconn
, tconn
->data
.socket
, data
, size
, 0);
763 drbd_put_data_sock(tconn
);
768 int drbd_send_sync_param(struct drbd_conf
*mdev
)
770 struct p_rs_param_95
*p
;
773 const int apv
= mdev
->tconn
->agreed_pro_version
;
775 size
= apv
<= 87 ? sizeof(struct p_rs_param
)
776 : apv
== 88 ? sizeof(struct p_rs_param
)
777 + strlen(mdev
->tconn
->net_conf
->verify_alg
) + 1
778 : apv
<= 94 ? sizeof(struct p_rs_param_89
)
779 : /* apv >= 95 */ sizeof(struct p_rs_param_95
);
781 /* used from admin command context and receiver/worker context.
782 * to avoid kmalloc, grab the socket right here,
783 * then use the pre-allocated sbuf there */
784 mutex_lock(&mdev
->tconn
->data
.mutex
);
785 sock
= mdev
->tconn
->data
.socket
;
787 if (likely(sock
!= NULL
)) {
788 enum drbd_packet cmd
=
789 apv
>= 89 ? P_SYNC_PARAM89
: P_SYNC_PARAM
;
791 p
= &mdev
->tconn
->data
.sbuf
.rs_param_95
;
793 /* initialize verify_alg and csums_alg */
794 memset(p
->verify_alg
, 0, 2 * SHARED_SECRET_MAX
);
796 if (get_ldev(mdev
)) {
797 p
->rate
= cpu_to_be32(mdev
->ldev
->dc
.resync_rate
);
798 p
->c_plan_ahead
= cpu_to_be32(mdev
->ldev
->dc
.c_plan_ahead
);
799 p
->c_delay_target
= cpu_to_be32(mdev
->ldev
->dc
.c_delay_target
);
800 p
->c_fill_target
= cpu_to_be32(mdev
->ldev
->dc
.c_fill_target
);
801 p
->c_max_rate
= cpu_to_be32(mdev
->ldev
->dc
.c_max_rate
);
804 p
->rate
= cpu_to_be32(DRBD_RATE_DEF
);
805 p
->c_plan_ahead
= cpu_to_be32(DRBD_C_PLAN_AHEAD_DEF
);
806 p
->c_delay_target
= cpu_to_be32(DRBD_C_DELAY_TARGET_DEF
);
807 p
->c_fill_target
= cpu_to_be32(DRBD_C_FILL_TARGET_DEF
);
808 p
->c_max_rate
= cpu_to_be32(DRBD_C_MAX_RATE_DEF
);
812 strcpy(p
->verify_alg
, mdev
->tconn
->net_conf
->verify_alg
);
814 strcpy(p
->csums_alg
, mdev
->tconn
->net_conf
->csums_alg
);
816 err
= _drbd_send_cmd(mdev
, sock
, cmd
, &p
->head
, size
, 0);
820 mutex_unlock(&mdev
->tconn
->data
.mutex
);
825 int drbd_send_protocol(struct drbd_tconn
*tconn
)
827 struct p_protocol
*p
;
830 size
= sizeof(struct p_protocol
);
832 if (tconn
->agreed_pro_version
>= 87)
833 size
+= strlen(tconn
->net_conf
->integrity_alg
) + 1;
835 /* we must not recurse into our own queue,
836 * as that is blocked during handshake */
837 p
= kmalloc(size
, GFP_NOIO
);
841 p
->protocol
= cpu_to_be32(tconn
->net_conf
->wire_protocol
);
842 p
->after_sb_0p
= cpu_to_be32(tconn
->net_conf
->after_sb_0p
);
843 p
->after_sb_1p
= cpu_to_be32(tconn
->net_conf
->after_sb_1p
);
844 p
->after_sb_2p
= cpu_to_be32(tconn
->net_conf
->after_sb_2p
);
845 p
->two_primaries
= cpu_to_be32(tconn
->net_conf
->two_primaries
);
848 if (tconn
->net_conf
->want_lose
)
850 if (tconn
->net_conf
->dry_run
) {
851 if (tconn
->agreed_pro_version
>= 92)
854 conn_err(tconn
, "--dry-run is not supported by peer");
859 p
->conn_flags
= cpu_to_be32(cf
);
861 if (tconn
->agreed_pro_version
>= 87)
862 strcpy(p
->integrity_alg
, tconn
->net_conf
->integrity_alg
);
864 rv
= !conn_send_cmd2(tconn
, P_PROTOCOL
, p
->head
.payload
, size
- sizeof(struct p_header
));
869 int _drbd_send_uuids(struct drbd_conf
*mdev
, u64 uuid_flags
)
874 if (!get_ldev_if_state(mdev
, D_NEGOTIATING
))
877 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
878 p
.uuid
[i
] = mdev
->ldev
? cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]) : 0;
880 mdev
->comm_bm_set
= drbd_bm_total_weight(mdev
);
881 p
.uuid
[UI_SIZE
] = cpu_to_be64(mdev
->comm_bm_set
);
882 uuid_flags
|= mdev
->tconn
->net_conf
->want_lose
? 1 : 0;
883 uuid_flags
|= test_bit(CRASHED_PRIMARY
, &mdev
->flags
) ? 2 : 0;
884 uuid_flags
|= mdev
->new_state_tmp
.disk
== D_INCONSISTENT
? 4 : 0;
885 p
.uuid
[UI_FLAGS
] = cpu_to_be64(uuid_flags
);
889 return !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, P_UUIDS
, &p
.head
, sizeof(p
));
892 int drbd_send_uuids(struct drbd_conf
*mdev
)
894 return _drbd_send_uuids(mdev
, 0);
897 int drbd_send_uuids_skip_initial_sync(struct drbd_conf
*mdev
)
899 return _drbd_send_uuids(mdev
, 8);
902 void drbd_print_uuids(struct drbd_conf
*mdev
, const char *text
)
904 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
905 u64
*uuid
= mdev
->ldev
->md
.uuid
;
906 dev_info(DEV
, "%s %016llX:%016llX:%016llX:%016llX\n",
908 (unsigned long long)uuid
[UI_CURRENT
],
909 (unsigned long long)uuid
[UI_BITMAP
],
910 (unsigned long long)uuid
[UI_HISTORY_START
],
911 (unsigned long long)uuid
[UI_HISTORY_END
]);
914 dev_info(DEV
, "%s effective data uuid: %016llX\n",
916 (unsigned long long)mdev
->ed_uuid
);
920 int drbd_gen_and_send_sync_uuid(struct drbd_conf
*mdev
)
925 D_ASSERT(mdev
->state
.disk
== D_UP_TO_DATE
);
927 uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
] + UUID_NEW_BM_OFFSET
;
928 drbd_uuid_set(mdev
, UI_BITMAP
, uuid
);
929 drbd_print_uuids(mdev
, "updated sync UUID");
931 p
.uuid
= cpu_to_be64(uuid
);
933 return !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, P_SYNC_UUID
, &p
.head
, sizeof(p
));
936 int drbd_send_sizes(struct drbd_conf
*mdev
, int trigger_reply
, enum dds_flags flags
)
939 sector_t d_size
, u_size
;
940 int q_order_type
, max_bio_size
;
943 if (get_ldev_if_state(mdev
, D_NEGOTIATING
)) {
944 D_ASSERT(mdev
->ldev
->backing_bdev
);
945 d_size
= drbd_get_max_capacity(mdev
->ldev
);
946 u_size
= mdev
->ldev
->dc
.disk_size
;
947 q_order_type
= drbd_queue_order_type(mdev
);
948 max_bio_size
= queue_max_hw_sectors(mdev
->ldev
->backing_bdev
->bd_disk
->queue
) << 9;
949 max_bio_size
= min_t(int, max_bio_size
, DRBD_MAX_BIO_SIZE
);
954 q_order_type
= QUEUE_ORDERED_NONE
;
955 max_bio_size
= DRBD_MAX_BIO_SIZE
; /* ... multiple BIOs per peer_request */
958 p
.d_size
= cpu_to_be64(d_size
);
959 p
.u_size
= cpu_to_be64(u_size
);
960 p
.c_size
= cpu_to_be64(trigger_reply
? 0 : drbd_get_capacity(mdev
->this_bdev
));
961 p
.max_bio_size
= cpu_to_be32(max_bio_size
);
962 p
.queue_order_type
= cpu_to_be16(q_order_type
);
963 p
.dds_flags
= cpu_to_be16(flags
);
965 ok
= !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, P_SIZES
, &p
.head
, sizeof(p
));
970 * drbd_send_state() - Sends the drbd state to the peer
971 * @mdev: DRBD device.
973 int drbd_send_state(struct drbd_conf
*mdev
)
979 mutex_lock(&mdev
->tconn
->data
.mutex
);
981 p
.state
= cpu_to_be32(mdev
->state
.i
); /* Within the send mutex */
982 sock
= mdev
->tconn
->data
.socket
;
984 if (likely(sock
!= NULL
))
985 err
= _drbd_send_cmd(mdev
, sock
, P_STATE
, &p
.head
, sizeof(p
), 0);
987 mutex_unlock(&mdev
->tconn
->data
.mutex
);
992 int _conn_send_state_req(struct drbd_tconn
*tconn
, int vnr
, enum drbd_packet cmd
,
993 union drbd_state mask
, union drbd_state val
)
995 struct p_req_state p
;
997 p
.mask
= cpu_to_be32(mask
.i
);
998 p
.val
= cpu_to_be32(val
.i
);
1000 return !conn_send_cmd(tconn
, vnr
, &tconn
->data
, cmd
, &p
.head
, sizeof(p
));
1003 int drbd_send_sr_reply(struct drbd_conf
*mdev
, enum drbd_state_rv retcode
)
1005 struct p_req_state_reply p
;
1007 p
.retcode
= cpu_to_be32(retcode
);
1009 return !drbd_send_cmd(mdev
, &mdev
->tconn
->meta
, P_STATE_CHG_REPLY
, &p
.head
, sizeof(p
));
1012 int conn_send_sr_reply(struct drbd_tconn
*tconn
, enum drbd_state_rv retcode
)
1014 struct p_req_state_reply p
;
1015 enum drbd_packet cmd
= tconn
->agreed_pro_version
< 100 ? P_STATE_CHG_REPLY
: P_CONN_ST_CHG_REPLY
;
1017 p
.retcode
= cpu_to_be32(retcode
);
1019 return !conn_send_cmd(tconn
, 0, &tconn
->meta
, cmd
, &p
.head
, sizeof(p
));
1022 int fill_bitmap_rle_bits(struct drbd_conf
*mdev
,
1023 struct p_compressed_bm
*p
,
1024 struct bm_xfer_ctx
*c
)
1026 struct bitstream bs
;
1027 unsigned long plain_bits
;
1034 /* may we use this feature? */
1035 if ((mdev
->tconn
->net_conf
->use_rle
== 0) ||
1036 (mdev
->tconn
->agreed_pro_version
< 90))
1039 if (c
->bit_offset
>= c
->bm_bits
)
1040 return 0; /* nothing to do. */
1042 /* use at most thus many bytes */
1043 bitstream_init(&bs
, p
->code
, BM_PACKET_VLI_BYTES_MAX
, 0);
1044 memset(p
->code
, 0, BM_PACKET_VLI_BYTES_MAX
);
1045 /* plain bits covered in this code string */
1048 /* p->encoding & 0x80 stores whether the first run length is set.
1049 * bit offset is implicit.
1050 * start with toggle == 2 to be able to tell the first iteration */
1053 /* see how much plain bits we can stuff into one packet
1054 * using RLE and VLI. */
1056 tmp
= (toggle
== 0) ? _drbd_bm_find_next_zero(mdev
, c
->bit_offset
)
1057 : _drbd_bm_find_next(mdev
, c
->bit_offset
);
1060 rl
= tmp
- c
->bit_offset
;
1062 if (toggle
== 2) { /* first iteration */
1064 /* the first checked bit was set,
1065 * store start value, */
1066 DCBP_set_start(p
, 1);
1067 /* but skip encoding of zero run length */
1071 DCBP_set_start(p
, 0);
1074 /* paranoia: catch zero runlength.
1075 * can only happen if bitmap is modified while we scan it. */
1077 dev_err(DEV
, "unexpected zero runlength while encoding bitmap "
1078 "t:%u bo:%lu\n", toggle
, c
->bit_offset
);
1082 bits
= vli_encode_bits(&bs
, rl
);
1083 if (bits
== -ENOBUFS
) /* buffer full */
1086 dev_err(DEV
, "error while encoding bitmap: %d\n", bits
);
1092 c
->bit_offset
= tmp
;
1093 } while (c
->bit_offset
< c
->bm_bits
);
1095 len
= bs
.cur
.b
- p
->code
+ !!bs
.cur
.bit
;
1097 if (plain_bits
< (len
<< 3)) {
1098 /* incompressible with this method.
1099 * we need to rewind both word and bit position. */
1100 c
->bit_offset
-= plain_bits
;
1101 bm_xfer_ctx_bit_to_word_offset(c
);
1102 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1106 /* RLE + VLI was able to compress it just fine.
1107 * update c->word_offset. */
1108 bm_xfer_ctx_bit_to_word_offset(c
);
1110 /* store pad_bits */
1111 DCBP_set_pad_bits(p
, (8 - bs
.cur
.bit
) & 0x7);
1117 * send_bitmap_rle_or_plain
1119 * Return 0 when done, 1 when another iteration is needed, and a negative error
1120 * code upon failure.
1123 send_bitmap_rle_or_plain(struct drbd_conf
*mdev
,
1124 struct p_header
*h
, struct bm_xfer_ctx
*c
)
1126 struct p_compressed_bm
*p
= (void*)h
;
1127 unsigned long num_words
;
1131 len
= fill_bitmap_rle_bits(mdev
, p
, c
);
1137 DCBP_set_code(p
, RLE_VLI_Bits
);
1138 ok
= !_drbd_send_cmd(mdev
, mdev
->tconn
->data
.socket
, P_COMPRESSED_BITMAP
, h
,
1139 sizeof(*p
) + len
, 0);
1142 c
->bytes
[0] += sizeof(*p
) + len
;
1144 if (c
->bit_offset
>= c
->bm_bits
)
1147 /* was not compressible.
1148 * send a buffer full of plain text bits instead. */
1149 num_words
= min_t(size_t, BM_PACKET_WORDS
, c
->bm_words
- c
->word_offset
);
1150 len
= num_words
* sizeof(long);
1152 drbd_bm_get_lel(mdev
, c
->word_offset
, num_words
, (unsigned long*)h
->payload
);
1153 ok
= !_drbd_send_cmd(mdev
, mdev
->tconn
->data
.socket
, P_BITMAP
,
1154 h
, sizeof(struct p_header80
) + len
, 0);
1155 c
->word_offset
+= num_words
;
1156 c
->bit_offset
= c
->word_offset
* BITS_PER_LONG
;
1159 c
->bytes
[1] += sizeof(struct p_header80
) + len
;
1161 if (c
->bit_offset
> c
->bm_bits
)
1162 c
->bit_offset
= c
->bm_bits
;
1166 INFO_bm_xfer_stats(mdev
, "send", c
);
1174 /* See the comment at receive_bitmap() */
1175 int _drbd_send_bitmap(struct drbd_conf
*mdev
)
1177 struct bm_xfer_ctx c
;
1181 if (!expect(mdev
->bitmap
))
1184 /* maybe we should use some per thread scratch page,
1185 * and allocate that during initial device creation? */
1186 p
= (struct p_header
*) __get_free_page(GFP_NOIO
);
1188 dev_err(DEV
, "failed to allocate one page buffer in %s\n", __func__
);
1192 if (get_ldev(mdev
)) {
1193 if (drbd_md_test_flag(mdev
->ldev
, MDF_FULL_SYNC
)) {
1194 dev_info(DEV
, "Writing the whole bitmap, MDF_FullSync was set.\n");
1195 drbd_bm_set_all(mdev
);
1196 if (drbd_bm_write(mdev
)) {
1197 /* write_bm did fail! Leave full sync flag set in Meta P_DATA
1198 * but otherwise process as per normal - need to tell other
1199 * side that a full resync is required! */
1200 dev_err(DEV
, "Failed to write bitmap to disk!\n");
1202 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
1209 c
= (struct bm_xfer_ctx
) {
1210 .bm_bits
= drbd_bm_bits(mdev
),
1211 .bm_words
= drbd_bm_words(mdev
),
1215 err
= send_bitmap_rle_or_plain(mdev
, p
, &c
);
1218 free_page((unsigned long) p
);
1222 int drbd_send_bitmap(struct drbd_conf
*mdev
)
1226 if (drbd_get_data_sock(mdev
->tconn
))
1228 err
= !_drbd_send_bitmap(mdev
);
1229 drbd_put_data_sock(mdev
->tconn
);
1233 int drbd_send_b_ack(struct drbd_conf
*mdev
, u32 barrier_nr
, u32 set_size
)
1236 struct p_barrier_ack p
;
1238 p
.barrier
= barrier_nr
;
1239 p
.set_size
= cpu_to_be32(set_size
);
1241 if (mdev
->state
.conn
< C_CONNECTED
)
1243 ok
= !drbd_send_cmd(mdev
, &mdev
->tconn
->meta
, P_BARRIER_ACK
, &p
.head
, sizeof(p
));
1248 * _drbd_send_ack() - Sends an ack packet
1249 * @mdev: DRBD device.
1250 * @cmd: Packet command code.
1251 * @sector: sector, needs to be in big endian byte order
1252 * @blksize: size in byte, needs to be in big endian byte order
1253 * @block_id: Id, big endian byte order
1255 static int _drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1256 u64 sector
, u32 blksize
, u64 block_id
)
1259 struct p_block_ack p
;
1262 p
.block_id
= block_id
;
1263 p
.blksize
= blksize
;
1264 p
.seq_num
= cpu_to_be32(atomic_inc_return(&mdev
->packet_seq
));
1266 if (!mdev
->tconn
->meta
.socket
|| mdev
->state
.conn
< C_CONNECTED
)
1268 ok
= !drbd_send_cmd(mdev
, &mdev
->tconn
->meta
, cmd
, &p
.head
, sizeof(p
));
1272 /* dp->sector and dp->block_id already/still in network byte order,
1273 * data_size is payload size according to dp->head,
1274 * and may need to be corrected for digest size. */
1275 int drbd_send_ack_dp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1276 struct p_data
*dp
, int data_size
)
1278 data_size
-= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_r_tfm
) ?
1279 crypto_hash_digestsize(mdev
->tconn
->integrity_r_tfm
) : 0;
1280 return _drbd_send_ack(mdev
, cmd
, dp
->sector
, cpu_to_be32(data_size
),
1284 int drbd_send_ack_rp(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1285 struct p_block_req
*rp
)
1287 return _drbd_send_ack(mdev
, cmd
, rp
->sector
, rp
->blksize
, rp
->block_id
);
1291 * drbd_send_ack() - Sends an ack packet
1292 * @mdev: DRBD device
1293 * @cmd: packet command code
1294 * @peer_req: peer request
1296 int drbd_send_ack(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1297 struct drbd_peer_request
*peer_req
)
1299 return _drbd_send_ack(mdev
, cmd
,
1300 cpu_to_be64(peer_req
->i
.sector
),
1301 cpu_to_be32(peer_req
->i
.size
),
1302 peer_req
->block_id
);
1305 /* This function misuses the block_id field to signal if the blocks
1306 * are is sync or not. */
1307 int drbd_send_ack_ex(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1308 sector_t sector
, int blksize
, u64 block_id
)
1310 return _drbd_send_ack(mdev
, cmd
,
1311 cpu_to_be64(sector
),
1312 cpu_to_be32(blksize
),
1313 cpu_to_be64(block_id
));
1316 int drbd_send_drequest(struct drbd_conf
*mdev
, int cmd
,
1317 sector_t sector
, int size
, u64 block_id
)
1320 struct p_block_req p
;
1322 p
.sector
= cpu_to_be64(sector
);
1323 p
.block_id
= block_id
;
1324 p
.blksize
= cpu_to_be32(size
);
1326 ok
= !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, cmd
, &p
.head
, sizeof(p
));
1330 int drbd_send_drequest_csum(struct drbd_conf
*mdev
, sector_t sector
, int size
,
1331 void *digest
, int digest_size
, enum drbd_packet cmd
)
1334 struct p_block_req p
;
1336 prepare_header(mdev
, &p
.head
, cmd
, sizeof(p
) - sizeof(struct p_header
) + digest_size
);
1337 p
.sector
= cpu_to_be64(sector
);
1338 p
.block_id
= ID_SYNCER
/* unused */;
1339 p
.blksize
= cpu_to_be32(size
);
1341 mutex_lock(&mdev
->tconn
->data
.mutex
);
1343 ok
= (sizeof(p
) == drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, &p
, sizeof(p
), 0));
1344 ok
= ok
&& (digest_size
== drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, digest
, digest_size
, 0));
1346 mutex_unlock(&mdev
->tconn
->data
.mutex
);
1351 int drbd_send_ov_request(struct drbd_conf
*mdev
, sector_t sector
, int size
)
1354 struct p_block_req p
;
1356 p
.sector
= cpu_to_be64(sector
);
1357 p
.block_id
= ID_SYNCER
/* unused */;
1358 p
.blksize
= cpu_to_be32(size
);
1360 ok
= !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, P_OV_REQUEST
, &p
.head
, sizeof(p
));
1364 /* called on sndtimeo
1365 * returns false if we should retry,
1366 * true if we think connection is dead
1368 static int we_should_drop_the_connection(struct drbd_tconn
*tconn
, struct socket
*sock
)
1371 /* long elapsed = (long)(jiffies - mdev->last_received); */
1373 drop_it
= tconn
->meta
.socket
== sock
1374 || !tconn
->asender
.task
1375 || get_t_state(&tconn
->asender
) != RUNNING
1376 || tconn
->cstate
< C_WF_REPORT_PARAMS
;
1381 drop_it
= !--tconn
->ko_count
;
1383 conn_err(tconn
, "[%s/%d] sock_sendmsg time expired, ko = %u\n",
1384 current
->comm
, current
->pid
, tconn
->ko_count
);
1385 request_ping(tconn
);
1388 return drop_it
; /* && (mdev->state == R_PRIMARY) */;
1391 static void drbd_update_congested(struct drbd_tconn
*tconn
)
1393 struct sock
*sk
= tconn
->data
.socket
->sk
;
1394 if (sk
->sk_wmem_queued
> sk
->sk_sndbuf
* 4 / 5)
1395 set_bit(NET_CONGESTED
, &tconn
->flags
);
1398 /* The idea of sendpage seems to be to put some kind of reference
1399 * to the page into the skb, and to hand it over to the NIC. In
1400 * this process get_page() gets called.
1402 * As soon as the page was really sent over the network put_page()
1403 * gets called by some part of the network layer. [ NIC driver? ]
1405 * [ get_page() / put_page() increment/decrement the count. If count
1406 * reaches 0 the page will be freed. ]
1408 * This works nicely with pages from FSs.
1409 * But this means that in protocol A we might signal IO completion too early!
1411 * In order not to corrupt data during a resync we must make sure
1412 * that we do not reuse our own buffer pages (EEs) to early, therefore
1413 * we have the net_ee list.
1415 * XFS seems to have problems, still, it submits pages with page_count == 0!
1416 * As a workaround, we disable sendpage on pages
1417 * with page_count == 0 or PageSlab.
1419 static int _drbd_no_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1420 int offset
, size_t size
, unsigned msg_flags
)
1422 int sent
= drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, kmap(page
) + offset
, size
, msg_flags
);
1425 mdev
->send_cnt
+= size
>>9;
1426 return sent
== size
;
1429 static int _drbd_send_page(struct drbd_conf
*mdev
, struct page
*page
,
1430 int offset
, size_t size
, unsigned msg_flags
)
1432 mm_segment_t oldfs
= get_fs();
1436 /* e.g. XFS meta- & log-data is in slab pages, which have a
1437 * page_count of 0 and/or have PageSlab() set.
1438 * we cannot use send_page for those, as that does get_page();
1439 * put_page(); and would cause either a VM_BUG directly, or
1440 * __page_cache_release a page that would actually still be referenced
1441 * by someone, leading to some obscure delayed Oops somewhere else. */
1442 if (disable_sendpage
|| (page_count(page
) < 1) || PageSlab(page
))
1443 return _drbd_no_send_page(mdev
, page
, offset
, size
, msg_flags
);
1445 msg_flags
|= MSG_NOSIGNAL
;
1446 drbd_update_congested(mdev
->tconn
);
1449 sent
= mdev
->tconn
->data
.socket
->ops
->sendpage(mdev
->tconn
->data
.socket
, page
,
1452 if (sent
== -EAGAIN
) {
1453 if (we_should_drop_the_connection(mdev
->tconn
,
1454 mdev
->tconn
->data
.socket
))
1460 dev_warn(DEV
, "%s: size=%d len=%d sent=%d\n",
1461 __func__
, (int)size
, len
, sent
);
1466 } while (len
> 0 /* THINK && mdev->cstate >= C_CONNECTED*/);
1468 clear_bit(NET_CONGESTED
, &mdev
->tconn
->flags
);
1472 mdev
->send_cnt
+= size
>>9;
1476 static int _drbd_send_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1478 struct bio_vec
*bvec
;
1480 /* hint all but last page with MSG_MORE */
1481 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1482 if (!_drbd_no_send_page(mdev
, bvec
->bv_page
,
1483 bvec
->bv_offset
, bvec
->bv_len
,
1484 i
== bio
->bi_vcnt
-1 ? 0 : MSG_MORE
))
1490 static int _drbd_send_zc_bio(struct drbd_conf
*mdev
, struct bio
*bio
)
1492 struct bio_vec
*bvec
;
1494 /* hint all but last page with MSG_MORE */
1495 __bio_for_each_segment(bvec
, bio
, i
, 0) {
1496 if (!_drbd_send_page(mdev
, bvec
->bv_page
,
1497 bvec
->bv_offset
, bvec
->bv_len
,
1498 i
== bio
->bi_vcnt
-1 ? 0 : MSG_MORE
))
1504 static int _drbd_send_zc_ee(struct drbd_conf
*mdev
,
1505 struct drbd_peer_request
*peer_req
)
1507 struct page
*page
= peer_req
->pages
;
1508 unsigned len
= peer_req
->i
.size
;
1510 /* hint all but last page with MSG_MORE */
1511 page_chain_for_each(page
) {
1512 unsigned l
= min_t(unsigned, len
, PAGE_SIZE
);
1513 if (!_drbd_send_page(mdev
, page
, 0, l
,
1514 page_chain_next(page
) ? MSG_MORE
: 0))
1521 static u32
bio_flags_to_wire(struct drbd_conf
*mdev
, unsigned long bi_rw
)
1523 if (mdev
->tconn
->agreed_pro_version
>= 95)
1524 return (bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0) |
1525 (bi_rw
& REQ_FUA
? DP_FUA
: 0) |
1526 (bi_rw
& REQ_FLUSH
? DP_FLUSH
: 0) |
1527 (bi_rw
& REQ_DISCARD
? DP_DISCARD
: 0);
1529 return bi_rw
& REQ_SYNC
? DP_RW_SYNC
: 0;
1532 /* Used to send write requests
1533 * R_PRIMARY -> Peer (P_DATA)
1535 int drbd_send_dblock(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1539 unsigned int dp_flags
= 0;
1543 if (drbd_get_data_sock(mdev
->tconn
))
1546 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1547 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1549 prepare_header(mdev
, &p
.head
, P_DATA
, sizeof(p
) - sizeof(struct p_header
) + dgs
+ req
->i
.size
);
1550 p
.sector
= cpu_to_be64(req
->i
.sector
);
1551 p
.block_id
= (unsigned long)req
;
1552 p
.seq_num
= cpu_to_be32(req
->seq_num
= atomic_inc_return(&mdev
->packet_seq
));
1554 dp_flags
= bio_flags_to_wire(mdev
, req
->master_bio
->bi_rw
);
1556 if (mdev
->state
.conn
>= C_SYNC_SOURCE
&&
1557 mdev
->state
.conn
<= C_PAUSED_SYNC_T
)
1558 dp_flags
|= DP_MAY_SET_IN_SYNC
;
1560 p
.dp_flags
= cpu_to_be32(dp_flags
);
1561 set_bit(UNPLUG_REMOTE
, &mdev
->flags
);
1563 drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, &p
, sizeof(p
), dgs
? MSG_MORE
: 0));
1565 dgb
= mdev
->tconn
->int_dig_out
;
1566 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, dgb
);
1567 ok
= dgs
== drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, dgb
, dgs
, 0);
1570 /* For protocol A, we have to memcpy the payload into
1571 * socket buffers, as we may complete right away
1572 * as soon as we handed it over to tcp, at which point the data
1573 * pages may become invalid.
1575 * For data-integrity enabled, we copy it as well, so we can be
1576 * sure that even if the bio pages may still be modified, it
1577 * won't change the data on the wire, thus if the digest checks
1578 * out ok after sending on this side, but does not fit on the
1579 * receiving side, we sure have detected corruption elsewhere.
1581 if (mdev
->tconn
->net_conf
->wire_protocol
== DRBD_PROT_A
|| dgs
)
1582 ok
= _drbd_send_bio(mdev
, req
->master_bio
);
1584 ok
= _drbd_send_zc_bio(mdev
, req
->master_bio
);
1586 /* double check digest, sometimes buffers have been modified in flight. */
1587 if (dgs
> 0 && dgs
<= 64) {
1588 /* 64 byte, 512 bit, is the largest digest size
1589 * currently supported in kernel crypto. */
1590 unsigned char digest
[64];
1591 drbd_csum_bio(mdev
, mdev
->tconn
->integrity_w_tfm
, req
->master_bio
, digest
);
1592 if (memcmp(mdev
->tconn
->int_dig_out
, digest
, dgs
)) {
1594 "Digest mismatch, buffer modified by upper layers during write: %llus +%u\n",
1595 (unsigned long long)req
->i
.sector
, req
->i
.size
);
1597 } /* else if (dgs > 64) {
1598 ... Be noisy about digest too large ...
1602 drbd_put_data_sock(mdev
->tconn
);
1607 /* answer packet, used to send data back for read requests:
1608 * Peer -> (diskless) R_PRIMARY (P_DATA_REPLY)
1609 * C_SYNC_SOURCE -> C_SYNC_TARGET (P_RS_DATA_REPLY)
1611 int drbd_send_block(struct drbd_conf
*mdev
, enum drbd_packet cmd
,
1612 struct drbd_peer_request
*peer_req
)
1619 dgs
= (mdev
->tconn
->agreed_pro_version
>= 87 && mdev
->tconn
->integrity_w_tfm
) ?
1620 crypto_hash_digestsize(mdev
->tconn
->integrity_w_tfm
) : 0;
1622 prepare_header(mdev
, &p
.head
, cmd
, sizeof(p
) -
1623 sizeof(struct p_header80
) +
1624 dgs
+ peer_req
->i
.size
);
1625 p
.sector
= cpu_to_be64(peer_req
->i
.sector
);
1626 p
.block_id
= peer_req
->block_id
;
1627 p
.seq_num
= 0; /* unused */
1629 /* Only called by our kernel thread.
1630 * This one may be interrupted by DRBD_SIG and/or DRBD_SIGKILL
1631 * in response to admin command or module unload.
1633 if (drbd_get_data_sock(mdev
->tconn
))
1636 ok
= sizeof(p
) == drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, &p
, sizeof(p
), dgs
? MSG_MORE
: 0);
1638 dgb
= mdev
->tconn
->int_dig_out
;
1639 drbd_csum_ee(mdev
, mdev
->tconn
->integrity_w_tfm
, peer_req
, dgb
);
1640 ok
= dgs
== drbd_send(mdev
->tconn
, mdev
->tconn
->data
.socket
, dgb
, dgs
, 0);
1643 ok
= _drbd_send_zc_ee(mdev
, peer_req
);
1645 drbd_put_data_sock(mdev
->tconn
);
1650 int drbd_send_oos(struct drbd_conf
*mdev
, struct drbd_request
*req
)
1652 struct p_block_desc p
;
1654 p
.sector
= cpu_to_be64(req
->i
.sector
);
1655 p
.blksize
= cpu_to_be32(req
->i
.size
);
1657 return !drbd_send_cmd(mdev
, &mdev
->tconn
->data
, P_OUT_OF_SYNC
, &p
.head
, sizeof(p
));
1661 drbd_send distinguishes two cases:
1663 Packets sent via the data socket "sock"
1664 and packets sent via the meta data socket "msock"
1667 -----------------+-------------------------+------------------------------
1668 timeout conf.timeout / 2 conf.timeout / 2
1669 timeout action send a ping via msock Abort communication
1670 and close all sockets
1674 * you must have down()ed the appropriate [m]sock_mutex elsewhere!
1676 int drbd_send(struct drbd_tconn
*tconn
, struct socket
*sock
,
1677 void *buf
, size_t size
, unsigned msg_flags
)
1686 /* THINK if (signal_pending) return ... ? */
1691 msg
.msg_name
= NULL
;
1692 msg
.msg_namelen
= 0;
1693 msg
.msg_control
= NULL
;
1694 msg
.msg_controllen
= 0;
1695 msg
.msg_flags
= msg_flags
| MSG_NOSIGNAL
;
1697 if (sock
== tconn
->data
.socket
) {
1698 tconn
->ko_count
= tconn
->net_conf
->ko_count
;
1699 drbd_update_congested(tconn
);
1703 * tcp_sendmsg does _not_ use its size parameter at all ?
1705 * -EAGAIN on timeout, -EINTR on signal.
1708 * do we need to block DRBD_SIG if sock == &meta.socket ??
1709 * otherwise wake_asender() might interrupt some send_*Ack !
1711 rv
= kernel_sendmsg(sock
, &msg
, &iov
, 1, size
);
1712 if (rv
== -EAGAIN
) {
1713 if (we_should_drop_the_connection(tconn
, sock
))
1719 flush_signals(current
);
1727 } while (sent
< size
);
1729 if (sock
== tconn
->data
.socket
)
1730 clear_bit(NET_CONGESTED
, &tconn
->flags
);
1733 if (rv
!= -EAGAIN
) {
1734 conn_err(tconn
, "%s_sendmsg returned %d\n",
1735 sock
== tconn
->meta
.socket
? "msock" : "sock",
1737 conn_request_state(tconn
, NS(conn
, C_BROKEN_PIPE
), CS_HARD
);
1739 conn_request_state(tconn
, NS(conn
, C_TIMEOUT
), CS_HARD
);
1746 * drbd_send_all - Send an entire buffer
1748 * Returns 0 upon success and a negative error value otherwise.
1750 int drbd_send_all(struct drbd_tconn
*tconn
, struct socket
*sock
, void *buffer
,
1751 size_t size
, unsigned msg_flags
)
1755 err
= drbd_send(tconn
, sock
, buffer
, size
, msg_flags
);
1763 static int drbd_open(struct block_device
*bdev
, fmode_t mode
)
1765 struct drbd_conf
*mdev
= bdev
->bd_disk
->private_data
;
1766 unsigned long flags
;
1769 mutex_lock(&drbd_main_mutex
);
1770 spin_lock_irqsave(&mdev
->tconn
->req_lock
, flags
);
1771 /* to have a stable mdev->state.role
1772 * and no race with updating open_cnt */
1774 if (mdev
->state
.role
!= R_PRIMARY
) {
1775 if (mode
& FMODE_WRITE
)
1777 else if (!allow_oos
)
1783 spin_unlock_irqrestore(&mdev
->tconn
->req_lock
, flags
);
1784 mutex_unlock(&drbd_main_mutex
);
1789 static int drbd_release(struct gendisk
*gd
, fmode_t mode
)
1791 struct drbd_conf
*mdev
= gd
->private_data
;
1792 mutex_lock(&drbd_main_mutex
);
1794 mutex_unlock(&drbd_main_mutex
);
1798 static void drbd_set_defaults(struct drbd_conf
*mdev
)
1800 /* Beware! The actual layout differs
1801 * between big endian and little endian */
1802 mdev
->state
= (union drbd_state
) {
1803 { .role
= R_SECONDARY
,
1805 .conn
= C_STANDALONE
,
1814 void drbd_init_set_defaults(struct drbd_conf
*mdev
)
1816 /* the memset(,0,) did most of this.
1817 * note: only assignments, no allocation in here */
1819 drbd_set_defaults(mdev
);
1821 atomic_set(&mdev
->ap_bio_cnt
, 0);
1822 atomic_set(&mdev
->ap_pending_cnt
, 0);
1823 atomic_set(&mdev
->rs_pending_cnt
, 0);
1824 atomic_set(&mdev
->unacked_cnt
, 0);
1825 atomic_set(&mdev
->local_cnt
, 0);
1826 atomic_set(&mdev
->pp_in_use
, 0);
1827 atomic_set(&mdev
->pp_in_use_by_net
, 0);
1828 atomic_set(&mdev
->rs_sect_in
, 0);
1829 atomic_set(&mdev
->rs_sect_ev
, 0);
1830 atomic_set(&mdev
->ap_in_flight
, 0);
1832 mutex_init(&mdev
->md_io_mutex
);
1833 mutex_init(&mdev
->own_state_mutex
);
1834 mdev
->state_mutex
= &mdev
->own_state_mutex
;
1836 spin_lock_init(&mdev
->al_lock
);
1837 spin_lock_init(&mdev
->peer_seq_lock
);
1838 spin_lock_init(&mdev
->epoch_lock
);
1840 INIT_LIST_HEAD(&mdev
->active_ee
);
1841 INIT_LIST_HEAD(&mdev
->sync_ee
);
1842 INIT_LIST_HEAD(&mdev
->done_ee
);
1843 INIT_LIST_HEAD(&mdev
->read_ee
);
1844 INIT_LIST_HEAD(&mdev
->net_ee
);
1845 INIT_LIST_HEAD(&mdev
->resync_reads
);
1846 INIT_LIST_HEAD(&mdev
->resync_work
.list
);
1847 INIT_LIST_HEAD(&mdev
->unplug_work
.list
);
1848 INIT_LIST_HEAD(&mdev
->go_diskless
.list
);
1849 INIT_LIST_HEAD(&mdev
->md_sync_work
.list
);
1850 INIT_LIST_HEAD(&mdev
->start_resync_work
.list
);
1851 INIT_LIST_HEAD(&mdev
->bm_io_work
.w
.list
);
1853 mdev
->resync_work
.cb
= w_resync_timer
;
1854 mdev
->unplug_work
.cb
= w_send_write_hint
;
1855 mdev
->go_diskless
.cb
= w_go_diskless
;
1856 mdev
->md_sync_work
.cb
= w_md_sync
;
1857 mdev
->bm_io_work
.w
.cb
= w_bitmap_io
;
1858 mdev
->start_resync_work
.cb
= w_start_resync
;
1860 mdev
->resync_work
.mdev
= mdev
;
1861 mdev
->unplug_work
.mdev
= mdev
;
1862 mdev
->go_diskless
.mdev
= mdev
;
1863 mdev
->md_sync_work
.mdev
= mdev
;
1864 mdev
->bm_io_work
.w
.mdev
= mdev
;
1865 mdev
->start_resync_work
.mdev
= mdev
;
1867 init_timer(&mdev
->resync_timer
);
1868 init_timer(&mdev
->md_sync_timer
);
1869 init_timer(&mdev
->start_resync_timer
);
1870 init_timer(&mdev
->request_timer
);
1871 mdev
->resync_timer
.function
= resync_timer_fn
;
1872 mdev
->resync_timer
.data
= (unsigned long) mdev
;
1873 mdev
->md_sync_timer
.function
= md_sync_timer_fn
;
1874 mdev
->md_sync_timer
.data
= (unsigned long) mdev
;
1875 mdev
->start_resync_timer
.function
= start_resync_timer_fn
;
1876 mdev
->start_resync_timer
.data
= (unsigned long) mdev
;
1877 mdev
->request_timer
.function
= request_timer_fn
;
1878 mdev
->request_timer
.data
= (unsigned long) mdev
;
1880 init_waitqueue_head(&mdev
->misc_wait
);
1881 init_waitqueue_head(&mdev
->state_wait
);
1882 init_waitqueue_head(&mdev
->ee_wait
);
1883 init_waitqueue_head(&mdev
->al_wait
);
1884 init_waitqueue_head(&mdev
->seq_wait
);
1886 /* mdev->tconn->agreed_pro_version gets initialized in drbd_connect() */
1887 mdev
->write_ordering
= WO_bdev_flush
;
1888 mdev
->resync_wenr
= LC_FREE
;
1889 mdev
->peer_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1890 mdev
->local_max_bio_size
= DRBD_MAX_BIO_SIZE_SAFE
;
1893 void drbd_mdev_cleanup(struct drbd_conf
*mdev
)
1896 if (mdev
->tconn
->receiver
.t_state
!= NONE
)
1897 dev_err(DEV
, "ASSERT FAILED: receiver t_state == %d expected 0.\n",
1898 mdev
->tconn
->receiver
.t_state
);
1900 /* no need to lock it, I'm the only thread alive */
1901 if (atomic_read(&mdev
->current_epoch
->epoch_size
) != 0)
1902 dev_err(DEV
, "epoch_size:%d\n", atomic_read(&mdev
->current_epoch
->epoch_size
));
1912 mdev
->rs_failed
= 0;
1913 mdev
->rs_last_events
= 0;
1914 mdev
->rs_last_sect_ev
= 0;
1915 for (i
= 0; i
< DRBD_SYNC_MARKS
; i
++) {
1916 mdev
->rs_mark_left
[i
] = 0;
1917 mdev
->rs_mark_time
[i
] = 0;
1919 D_ASSERT(mdev
->tconn
->net_conf
== NULL
);
1921 drbd_set_my_capacity(mdev
, 0);
1923 /* maybe never allocated. */
1924 drbd_bm_resize(mdev
, 0, 1);
1925 drbd_bm_cleanup(mdev
);
1928 drbd_free_resources(mdev
);
1929 clear_bit(AL_SUSPENDED
, &mdev
->flags
);
1932 * currently we drbd_init_ee only on module load, so
1933 * we may do drbd_release_ee only on module unload!
1935 D_ASSERT(list_empty(&mdev
->active_ee
));
1936 D_ASSERT(list_empty(&mdev
->sync_ee
));
1937 D_ASSERT(list_empty(&mdev
->done_ee
));
1938 D_ASSERT(list_empty(&mdev
->read_ee
));
1939 D_ASSERT(list_empty(&mdev
->net_ee
));
1940 D_ASSERT(list_empty(&mdev
->resync_reads
));
1941 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
1942 D_ASSERT(list_empty(&mdev
->tconn
->meta
.work
.q
));
1943 D_ASSERT(list_empty(&mdev
->resync_work
.list
));
1944 D_ASSERT(list_empty(&mdev
->unplug_work
.list
));
1945 D_ASSERT(list_empty(&mdev
->go_diskless
.list
));
1947 drbd_set_defaults(mdev
);
1951 static void drbd_destroy_mempools(void)
1955 while (drbd_pp_pool
) {
1956 page
= drbd_pp_pool
;
1957 drbd_pp_pool
= (struct page
*)page_private(page
);
1962 /* D_ASSERT(atomic_read(&drbd_pp_vacant)==0); */
1964 if (drbd_md_io_bio_set
)
1965 bioset_free(drbd_md_io_bio_set
);
1966 if (drbd_md_io_page_pool
)
1967 mempool_destroy(drbd_md_io_page_pool
);
1968 if (drbd_ee_mempool
)
1969 mempool_destroy(drbd_ee_mempool
);
1970 if (drbd_request_mempool
)
1971 mempool_destroy(drbd_request_mempool
);
1973 kmem_cache_destroy(drbd_ee_cache
);
1974 if (drbd_request_cache
)
1975 kmem_cache_destroy(drbd_request_cache
);
1976 if (drbd_bm_ext_cache
)
1977 kmem_cache_destroy(drbd_bm_ext_cache
);
1978 if (drbd_al_ext_cache
)
1979 kmem_cache_destroy(drbd_al_ext_cache
);
1981 drbd_md_io_bio_set
= NULL
;
1982 drbd_md_io_page_pool
= NULL
;
1983 drbd_ee_mempool
= NULL
;
1984 drbd_request_mempool
= NULL
;
1985 drbd_ee_cache
= NULL
;
1986 drbd_request_cache
= NULL
;
1987 drbd_bm_ext_cache
= NULL
;
1988 drbd_al_ext_cache
= NULL
;
1993 static int drbd_create_mempools(void)
1996 const int number
= (DRBD_MAX_BIO_SIZE
/PAGE_SIZE
) * minor_count
;
1999 /* prepare our caches and mempools */
2000 drbd_request_mempool
= NULL
;
2001 drbd_ee_cache
= NULL
;
2002 drbd_request_cache
= NULL
;
2003 drbd_bm_ext_cache
= NULL
;
2004 drbd_al_ext_cache
= NULL
;
2005 drbd_pp_pool
= NULL
;
2006 drbd_md_io_page_pool
= NULL
;
2007 drbd_md_io_bio_set
= NULL
;
2010 drbd_request_cache
= kmem_cache_create(
2011 "drbd_req", sizeof(struct drbd_request
), 0, 0, NULL
);
2012 if (drbd_request_cache
== NULL
)
2015 drbd_ee_cache
= kmem_cache_create(
2016 "drbd_ee", sizeof(struct drbd_peer_request
), 0, 0, NULL
);
2017 if (drbd_ee_cache
== NULL
)
2020 drbd_bm_ext_cache
= kmem_cache_create(
2021 "drbd_bm", sizeof(struct bm_extent
), 0, 0, NULL
);
2022 if (drbd_bm_ext_cache
== NULL
)
2025 drbd_al_ext_cache
= kmem_cache_create(
2026 "drbd_al", sizeof(struct lc_element
), 0, 0, NULL
);
2027 if (drbd_al_ext_cache
== NULL
)
2031 drbd_md_io_bio_set
= bioset_create(DRBD_MIN_POOL_PAGES
, 0);
2032 if (drbd_md_io_bio_set
== NULL
)
2035 drbd_md_io_page_pool
= mempool_create_page_pool(DRBD_MIN_POOL_PAGES
, 0);
2036 if (drbd_md_io_page_pool
== NULL
)
2039 drbd_request_mempool
= mempool_create(number
,
2040 mempool_alloc_slab
, mempool_free_slab
, drbd_request_cache
);
2041 if (drbd_request_mempool
== NULL
)
2044 drbd_ee_mempool
= mempool_create(number
,
2045 mempool_alloc_slab
, mempool_free_slab
, drbd_ee_cache
);
2046 if (drbd_ee_mempool
== NULL
)
2049 /* drbd's page pool */
2050 spin_lock_init(&drbd_pp_lock
);
2052 for (i
= 0; i
< number
; i
++) {
2053 page
= alloc_page(GFP_HIGHUSER
);
2056 set_page_private(page
, (unsigned long)drbd_pp_pool
);
2057 drbd_pp_pool
= page
;
2059 drbd_pp_vacant
= number
;
2064 drbd_destroy_mempools(); /* in case we allocated some */
2068 static int drbd_notify_sys(struct notifier_block
*this, unsigned long code
,
2071 /* just so we have it. you never know what interesting things we
2072 * might want to do here some day...
2078 static struct notifier_block drbd_notifier
= {
2079 .notifier_call
= drbd_notify_sys
,
2082 static void drbd_release_ee_lists(struct drbd_conf
*mdev
)
2086 rr
= drbd_release_ee(mdev
, &mdev
->active_ee
);
2088 dev_err(DEV
, "%d EEs in active list found!\n", rr
);
2090 rr
= drbd_release_ee(mdev
, &mdev
->sync_ee
);
2092 dev_err(DEV
, "%d EEs in sync list found!\n", rr
);
2094 rr
= drbd_release_ee(mdev
, &mdev
->read_ee
);
2096 dev_err(DEV
, "%d EEs in read list found!\n", rr
);
2098 rr
= drbd_release_ee(mdev
, &mdev
->done_ee
);
2100 dev_err(DEV
, "%d EEs in done list found!\n", rr
);
2102 rr
= drbd_release_ee(mdev
, &mdev
->net_ee
);
2104 dev_err(DEV
, "%d EEs in net list found!\n", rr
);
2107 /* caution. no locking. */
2108 void drbd_delete_device(unsigned int minor
)
2110 struct drbd_conf
*mdev
= minor_to_mdev(minor
);
2115 idr_remove(&mdev
->tconn
->volumes
, mdev
->vnr
);
2116 idr_remove(&minors
, minor
);
2119 /* paranoia asserts */
2120 D_ASSERT(mdev
->open_cnt
== 0);
2121 D_ASSERT(list_empty(&mdev
->tconn
->data
.work
.q
));
2122 /* end paranoia asserts */
2124 del_gendisk(mdev
->vdisk
);
2126 /* cleanup stuff that may have been allocated during
2127 * device (re-)configuration or state changes */
2129 if (mdev
->this_bdev
)
2130 bdput(mdev
->this_bdev
);
2132 drbd_free_resources(mdev
);
2134 drbd_release_ee_lists(mdev
);
2136 lc_destroy(mdev
->act_log
);
2137 lc_destroy(mdev
->resync
);
2139 kfree(mdev
->p_uuid
);
2140 /* mdev->p_uuid = NULL; */
2142 /* cleanup the rest that has been
2143 * allocated from drbd_new_device
2144 * and actually free the mdev itself */
2145 drbd_free_mdev(mdev
);
2148 static void drbd_cleanup(void)
2151 struct drbd_conf
*mdev
;
2153 unregister_reboot_notifier(&drbd_notifier
);
2155 /* first remove proc,
2156 * drbdsetup uses it's presence to detect
2157 * whether DRBD is loaded.
2158 * If we would get stuck in proc removal,
2159 * but have netlink already deregistered,
2160 * some drbdsetup commands may wait forever
2164 remove_proc_entry("drbd", NULL
);
2166 drbd_genl_unregister();
2168 idr_for_each_entry(&minors
, mdev
, i
)
2169 drbd_delete_device(i
);
2170 drbd_destroy_mempools();
2171 unregister_blkdev(DRBD_MAJOR
, "drbd");
2173 idr_destroy(&minors
);
2175 printk(KERN_INFO
"drbd: module cleanup done.\n");
2179 * drbd_congested() - Callback for pdflush
2180 * @congested_data: User data
2181 * @bdi_bits: Bits pdflush is currently interested in
2183 * Returns 1<<BDI_async_congested and/or 1<<BDI_sync_congested if we are congested.
2185 static int drbd_congested(void *congested_data
, int bdi_bits
)
2187 struct drbd_conf
*mdev
= congested_data
;
2188 struct request_queue
*q
;
2192 if (!may_inc_ap_bio(mdev
)) {
2193 /* DRBD has frozen IO */
2199 if (get_ldev(mdev
)) {
2200 q
= bdev_get_queue(mdev
->ldev
->backing_bdev
);
2201 r
= bdi_congested(&q
->backing_dev_info
, bdi_bits
);
2207 if (bdi_bits
& (1 << BDI_async_congested
) && test_bit(NET_CONGESTED
, &mdev
->tconn
->flags
)) {
2208 r
|= (1 << BDI_async_congested
);
2209 reason
= reason
== 'b' ? 'a' : 'n';
2213 mdev
->congestion_reason
= reason
;
2217 static void drbd_init_workqueue(struct drbd_work_queue
* wq
)
2219 sema_init(&wq
->s
, 0);
2220 spin_lock_init(&wq
->q_lock
);
2221 INIT_LIST_HEAD(&wq
->q
);
2224 struct drbd_tconn
*conn_by_name(const char *name
)
2226 struct drbd_tconn
*tconn
;
2228 if (!name
|| !name
[0])
2231 mutex_lock(&drbd_cfg_mutex
);
2232 list_for_each_entry(tconn
, &drbd_tconns
, all_tconn
) {
2233 if (!strcmp(tconn
->name
, name
))
2238 mutex_unlock(&drbd_cfg_mutex
);
2242 struct drbd_tconn
*drbd_new_tconn(const char *name
)
2244 struct drbd_tconn
*tconn
;
2246 tconn
= kzalloc(sizeof(struct drbd_tconn
), GFP_KERNEL
);
2250 tconn
->name
= kstrdup(name
, GFP_KERNEL
);
2254 if (!zalloc_cpumask_var(&tconn
->cpu_mask
, GFP_KERNEL
))
2257 if (!tl_init(tconn
))
2260 tconn
->cstate
= C_STANDALONE
;
2261 mutex_init(&tconn
->cstate_mutex
);
2262 spin_lock_init(&tconn
->req_lock
);
2263 atomic_set(&tconn
->net_cnt
, 0);
2264 init_waitqueue_head(&tconn
->net_cnt_wait
);
2265 init_waitqueue_head(&tconn
->ping_wait
);
2266 idr_init(&tconn
->volumes
);
2268 drbd_init_workqueue(&tconn
->data
.work
);
2269 mutex_init(&tconn
->data
.mutex
);
2271 drbd_init_workqueue(&tconn
->meta
.work
);
2272 mutex_init(&tconn
->meta
.mutex
);
2274 drbd_thread_init(tconn
, &tconn
->receiver
, drbdd_init
, "receiver");
2275 drbd_thread_init(tconn
, &tconn
->worker
, drbd_worker
, "worker");
2276 drbd_thread_init(tconn
, &tconn
->asender
, drbd_asender
, "asender");
2278 tconn
->res_opts
= (struct res_opts
) {
2279 {}, 0, /* cpu_mask */
2280 DRBD_ON_NO_DATA_DEF
, /* on_no_data */
2283 mutex_lock(&drbd_cfg_mutex
);
2284 list_add_tail(&tconn
->all_tconn
, &drbd_tconns
);
2285 mutex_unlock(&drbd_cfg_mutex
);
2291 free_cpumask_var(tconn
->cpu_mask
);
2298 void drbd_free_tconn(struct drbd_tconn
*tconn
)
2300 list_del(&tconn
->all_tconn
);
2301 idr_destroy(&tconn
->volumes
);
2303 free_cpumask_var(tconn
->cpu_mask
);
2305 kfree(tconn
->int_dig_out
);
2306 kfree(tconn
->int_dig_in
);
2307 kfree(tconn
->int_dig_vv
);
2311 enum drbd_ret_code
conn_new_minor(struct drbd_tconn
*tconn
, unsigned int minor
, int vnr
)
2313 struct drbd_conf
*mdev
;
2314 struct gendisk
*disk
;
2315 struct request_queue
*q
;
2317 int minor_got
= minor
;
2318 enum drbd_ret_code err
= ERR_NOMEM
;
2320 mdev
= minor_to_mdev(minor
);
2322 return ERR_MINOR_EXISTS
;
2324 /* GFP_KERNEL, we are outside of all write-out paths */
2325 mdev
= kzalloc(sizeof(struct drbd_conf
), GFP_KERNEL
);
2329 mdev
->tconn
= tconn
;
2330 mdev
->minor
= minor
;
2333 drbd_init_set_defaults(mdev
);
2335 q
= blk_alloc_queue(GFP_KERNEL
);
2339 q
->queuedata
= mdev
;
2341 disk
= alloc_disk(1);
2346 set_disk_ro(disk
, true);
2349 disk
->major
= DRBD_MAJOR
;
2350 disk
->first_minor
= minor
;
2351 disk
->fops
= &drbd_ops
;
2352 sprintf(disk
->disk_name
, "drbd%d", minor
);
2353 disk
->private_data
= mdev
;
2355 mdev
->this_bdev
= bdget(MKDEV(DRBD_MAJOR
, minor
));
2356 /* we have no partitions. we contain only ourselves. */
2357 mdev
->this_bdev
->bd_contains
= mdev
->this_bdev
;
2359 q
->backing_dev_info
.congested_fn
= drbd_congested
;
2360 q
->backing_dev_info
.congested_data
= mdev
;
2362 blk_queue_make_request(q
, drbd_make_request
);
2363 /* Setting the max_hw_sectors to an odd value of 8kibyte here
2364 This triggers a max_bio_size message upon first attach or connect */
2365 blk_queue_max_hw_sectors(q
, DRBD_MAX_BIO_SIZE_SAFE
>> 8);
2366 blk_queue_bounce_limit(q
, BLK_BOUNCE_ANY
);
2367 blk_queue_merge_bvec(q
, drbd_merge_bvec
);
2368 q
->queue_lock
= &mdev
->tconn
->req_lock
; /* needed since we use */
2370 mdev
->md_io_page
= alloc_page(GFP_KERNEL
);
2371 if (!mdev
->md_io_page
)
2372 goto out_no_io_page
;
2374 if (drbd_bm_init(mdev
))
2376 mdev
->read_requests
= RB_ROOT
;
2377 mdev
->write_requests
= RB_ROOT
;
2379 mdev
->current_epoch
= kzalloc(sizeof(struct drbd_epoch
), GFP_KERNEL
);
2380 if (!mdev
->current_epoch
)
2383 INIT_LIST_HEAD(&mdev
->current_epoch
->list
);
2386 if (!idr_pre_get(&minors
, GFP_KERNEL
))
2387 goto out_no_minor_idr
;
2388 if (idr_get_new_above(&minors
, mdev
, minor
, &minor_got
))
2389 goto out_no_minor_idr
;
2390 if (minor_got
!= minor
) {
2391 err
= ERR_MINOR_EXISTS
;
2392 drbd_msg_put_info("requested minor exists already");
2393 goto out_idr_remove_minor
;
2396 if (!idr_pre_get(&tconn
->volumes
, GFP_KERNEL
))
2397 goto out_idr_remove_minor
;
2398 if (idr_get_new_above(&tconn
->volumes
, mdev
, vnr
, &vnr_got
))
2399 goto out_idr_remove_minor
;
2400 if (vnr_got
!= vnr
) {
2401 err
= ERR_INVALID_REQUEST
;
2402 drbd_msg_put_info("requested volume exists already");
2403 goto out_idr_remove_vol
;
2407 /* inherit the connection state */
2408 mdev
->state
.conn
= tconn
->cstate
;
2409 if (mdev
->state
.conn
== C_WF_REPORT_PARAMS
)
2410 drbd_connected(vnr
, mdev
, tconn
);
2415 idr_remove(&tconn
->volumes
, vnr_got
);
2416 out_idr_remove_minor
:
2417 idr_remove(&minors
, minor_got
);
2420 kfree(mdev
->current_epoch
);
2422 drbd_bm_cleanup(mdev
);
2424 __free_page(mdev
->md_io_page
);
2428 blk_cleanup_queue(q
);
2434 /* counterpart of drbd_new_device.
2435 * last part of drbd_delete_device. */
2436 void drbd_free_mdev(struct drbd_conf
*mdev
)
2438 kfree(mdev
->current_epoch
);
2439 if (mdev
->bitmap
) /* should no longer be there. */
2440 drbd_bm_cleanup(mdev
);
2441 __free_page(mdev
->md_io_page
);
2442 put_disk(mdev
->vdisk
);
2443 blk_cleanup_queue(mdev
->rq_queue
);
2448 int __init
drbd_init(void)
2452 BUILD_BUG_ON(sizeof(struct p_header80
) != sizeof(struct p_header95
));
2453 BUILD_BUG_ON(sizeof(struct p_handshake
) != 80);
2455 if (minor_count
< DRBD_MINOR_COUNT_MIN
|| minor_count
> DRBD_MINOR_COUNT_MAX
) {
2457 "drbd: invalid minor_count (%d)\n", minor_count
);
2465 err
= register_blkdev(DRBD_MAJOR
, "drbd");
2468 "drbd: unable to register block device major %d\n",
2473 err
= drbd_genl_register();
2475 printk(KERN_ERR
"drbd: unable to register generic netlink family\n");
2480 register_reboot_notifier(&drbd_notifier
);
2483 * allocate all necessary structs
2487 init_waitqueue_head(&drbd_pp_wait
);
2489 drbd_proc
= NULL
; /* play safe for drbd_cleanup */
2492 err
= drbd_create_mempools();
2496 drbd_proc
= proc_create_data("drbd", S_IFREG
| S_IRUGO
, NULL
, &drbd_proc_fops
, NULL
);
2498 printk(KERN_ERR
"drbd: unable to register proc file\n");
2502 rwlock_init(&global_state_lock
);
2503 INIT_LIST_HEAD(&drbd_tconns
);
2505 printk(KERN_INFO
"drbd: initialized. "
2506 "Version: " REL_VERSION
" (api:%d/proto:%d-%d)\n",
2507 API_VERSION
, PRO_VERSION_MIN
, PRO_VERSION_MAX
);
2508 printk(KERN_INFO
"drbd: %s\n", drbd_buildtag());
2509 printk(KERN_INFO
"drbd: registered as block device major %d\n",
2512 return 0; /* Success! */
2517 /* currently always the case */
2518 printk(KERN_ERR
"drbd: ran out of memory\n");
2520 printk(KERN_ERR
"drbd: initialization failure\n");
2524 void drbd_free_bc(struct drbd_backing_dev
*ldev
)
2529 blkdev_put(ldev
->backing_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2530 blkdev_put(ldev
->md_bdev
, FMODE_READ
| FMODE_WRITE
| FMODE_EXCL
);
2535 void drbd_free_sock(struct drbd_tconn
*tconn
)
2537 if (tconn
->data
.socket
) {
2538 mutex_lock(&tconn
->data
.mutex
);
2539 kernel_sock_shutdown(tconn
->data
.socket
, SHUT_RDWR
);
2540 sock_release(tconn
->data
.socket
);
2541 tconn
->data
.socket
= NULL
;
2542 mutex_unlock(&tconn
->data
.mutex
);
2544 if (tconn
->meta
.socket
) {
2545 mutex_lock(&tconn
->meta
.mutex
);
2546 kernel_sock_shutdown(tconn
->meta
.socket
, SHUT_RDWR
);
2547 sock_release(tconn
->meta
.socket
);
2548 tconn
->meta
.socket
= NULL
;
2549 mutex_unlock(&tconn
->meta
.mutex
);
2554 void drbd_free_resources(struct drbd_conf
*mdev
)
2556 crypto_free_hash(mdev
->tconn
->csums_tfm
);
2557 mdev
->tconn
->csums_tfm
= NULL
;
2558 crypto_free_hash(mdev
->tconn
->verify_tfm
);
2559 mdev
->tconn
->verify_tfm
= NULL
;
2560 crypto_free_hash(mdev
->tconn
->cram_hmac_tfm
);
2561 mdev
->tconn
->cram_hmac_tfm
= NULL
;
2562 crypto_free_hash(mdev
->tconn
->integrity_w_tfm
);
2563 mdev
->tconn
->integrity_w_tfm
= NULL
;
2564 crypto_free_hash(mdev
->tconn
->integrity_r_tfm
);
2565 mdev
->tconn
->integrity_r_tfm
= NULL
;
2567 drbd_free_sock(mdev
->tconn
);
2570 drbd_free_bc(mdev
->ldev
);
2571 mdev
->ldev
= NULL
;);
2574 /* meta data management */
2576 struct meta_data_on_disk
{
2577 u64 la_size
; /* last agreed size. */
2578 u64 uuid
[UI_SIZE
]; /* UUIDs. */
2581 u32 flags
; /* MDF */
2584 u32 al_offset
; /* offset to this block */
2585 u32 al_nr_extents
; /* important for restoring the AL */
2586 /* `-- act_log->nr_elements <-- ldev->dc.al_extents */
2587 u32 bm_offset
; /* offset to the bitmap, from here */
2588 u32 bm_bytes_per_bit
; /* BM_BLOCK_SIZE */
2589 u32 la_peer_max_bio_size
; /* last peer max_bio_size */
2590 u32 reserved_u32
[3];
2595 * drbd_md_sync() - Writes the meta data super block if the MD_DIRTY flag bit is set
2596 * @mdev: DRBD device.
2598 void drbd_md_sync(struct drbd_conf
*mdev
)
2600 struct meta_data_on_disk
*buffer
;
2604 del_timer(&mdev
->md_sync_timer
);
2605 /* timer may be rearmed by drbd_md_mark_dirty() now. */
2606 if (!test_and_clear_bit(MD_DIRTY
, &mdev
->flags
))
2609 /* We use here D_FAILED and not D_ATTACHING because we try to write
2610 * metadata even if we detach due to a disk failure! */
2611 if (!get_ldev_if_state(mdev
, D_FAILED
))
2614 mutex_lock(&mdev
->md_io_mutex
);
2615 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2616 memset(buffer
, 0, 512);
2618 buffer
->la_size
= cpu_to_be64(drbd_get_capacity(mdev
->this_bdev
));
2619 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2620 buffer
->uuid
[i
] = cpu_to_be64(mdev
->ldev
->md
.uuid
[i
]);
2621 buffer
->flags
= cpu_to_be32(mdev
->ldev
->md
.flags
);
2622 buffer
->magic
= cpu_to_be32(DRBD_MD_MAGIC
);
2624 buffer
->md_size_sect
= cpu_to_be32(mdev
->ldev
->md
.md_size_sect
);
2625 buffer
->al_offset
= cpu_to_be32(mdev
->ldev
->md
.al_offset
);
2626 buffer
->al_nr_extents
= cpu_to_be32(mdev
->act_log
->nr_elements
);
2627 buffer
->bm_bytes_per_bit
= cpu_to_be32(BM_BLOCK_SIZE
);
2628 buffer
->device_uuid
= cpu_to_be64(mdev
->ldev
->md
.device_uuid
);
2630 buffer
->bm_offset
= cpu_to_be32(mdev
->ldev
->md
.bm_offset
);
2631 buffer
->la_peer_max_bio_size
= cpu_to_be32(mdev
->peer_max_bio_size
);
2633 D_ASSERT(drbd_md_ss__(mdev
, mdev
->ldev
) == mdev
->ldev
->md
.md_offset
);
2634 sector
= mdev
->ldev
->md
.md_offset
;
2636 if (!drbd_md_sync_page_io(mdev
, mdev
->ldev
, sector
, WRITE
)) {
2637 /* this was a try anyways ... */
2638 dev_err(DEV
, "meta data update failed!\n");
2639 drbd_chk_io_error(mdev
, 1, true);
2642 /* Update mdev->ldev->md.la_size_sect,
2643 * since we updated it on metadata. */
2644 mdev
->ldev
->md
.la_size_sect
= drbd_get_capacity(mdev
->this_bdev
);
2646 mutex_unlock(&mdev
->md_io_mutex
);
2651 * drbd_md_read() - Reads in the meta data super block
2652 * @mdev: DRBD device.
2653 * @bdev: Device from which the meta data should be read in.
2655 * Return 0 (NO_ERROR) on success, and an enum drbd_ret_code in case
2656 * something goes wrong. Currently only: ERR_IO_MD_DISK, ERR_MD_INVALID.
2658 int drbd_md_read(struct drbd_conf
*mdev
, struct drbd_backing_dev
*bdev
)
2660 struct meta_data_on_disk
*buffer
;
2661 int i
, rv
= NO_ERROR
;
2663 if (!get_ldev_if_state(mdev
, D_ATTACHING
))
2664 return ERR_IO_MD_DISK
;
2666 mutex_lock(&mdev
->md_io_mutex
);
2667 buffer
= (struct meta_data_on_disk
*)page_address(mdev
->md_io_page
);
2669 if (!drbd_md_sync_page_io(mdev
, bdev
, bdev
->md
.md_offset
, READ
)) {
2670 /* NOTE: can't do normal error processing here as this is
2671 called BEFORE disk is attached */
2672 dev_err(DEV
, "Error while reading metadata.\n");
2673 rv
= ERR_IO_MD_DISK
;
2677 if (buffer
->magic
!= cpu_to_be32(DRBD_MD_MAGIC
)) {
2678 dev_err(DEV
, "Error while reading metadata, magic not found.\n");
2679 rv
= ERR_MD_INVALID
;
2682 if (be32_to_cpu(buffer
->al_offset
) != bdev
->md
.al_offset
) {
2683 dev_err(DEV
, "unexpected al_offset: %d (expected %d)\n",
2684 be32_to_cpu(buffer
->al_offset
), bdev
->md
.al_offset
);
2685 rv
= ERR_MD_INVALID
;
2688 if (be32_to_cpu(buffer
->bm_offset
) != bdev
->md
.bm_offset
) {
2689 dev_err(DEV
, "unexpected bm_offset: %d (expected %d)\n",
2690 be32_to_cpu(buffer
->bm_offset
), bdev
->md
.bm_offset
);
2691 rv
= ERR_MD_INVALID
;
2694 if (be32_to_cpu(buffer
->md_size_sect
) != bdev
->md
.md_size_sect
) {
2695 dev_err(DEV
, "unexpected md_size: %u (expected %u)\n",
2696 be32_to_cpu(buffer
->md_size_sect
), bdev
->md
.md_size_sect
);
2697 rv
= ERR_MD_INVALID
;
2701 if (be32_to_cpu(buffer
->bm_bytes_per_bit
) != BM_BLOCK_SIZE
) {
2702 dev_err(DEV
, "unexpected bm_bytes_per_bit: %u (expected %u)\n",
2703 be32_to_cpu(buffer
->bm_bytes_per_bit
), BM_BLOCK_SIZE
);
2704 rv
= ERR_MD_INVALID
;
2708 bdev
->md
.la_size_sect
= be64_to_cpu(buffer
->la_size
);
2709 for (i
= UI_CURRENT
; i
< UI_SIZE
; i
++)
2710 bdev
->md
.uuid
[i
] = be64_to_cpu(buffer
->uuid
[i
]);
2711 bdev
->md
.flags
= be32_to_cpu(buffer
->flags
);
2712 bdev
->dc
.al_extents
= be32_to_cpu(buffer
->al_nr_extents
);
2713 bdev
->md
.device_uuid
= be64_to_cpu(buffer
->device_uuid
);
2715 spin_lock_irq(&mdev
->tconn
->req_lock
);
2716 if (mdev
->state
.conn
< C_CONNECTED
) {
2718 peer
= be32_to_cpu(buffer
->la_peer_max_bio_size
);
2719 peer
= max_t(int, peer
, DRBD_MAX_BIO_SIZE_SAFE
);
2720 mdev
->peer_max_bio_size
= peer
;
2722 spin_unlock_irq(&mdev
->tconn
->req_lock
);
2724 if (bdev
->dc
.al_extents
< 7)
2725 bdev
->dc
.al_extents
= 127;
2728 mutex_unlock(&mdev
->md_io_mutex
);
2735 * drbd_md_mark_dirty() - Mark meta data super block as dirty
2736 * @mdev: DRBD device.
2738 * Call this function if you change anything that should be written to
2739 * the meta-data super block. This function sets MD_DIRTY, and starts a
2740 * timer that ensures that within five seconds you have to call drbd_md_sync().
2743 void drbd_md_mark_dirty_(struct drbd_conf
*mdev
, unsigned int line
, const char *func
)
2745 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
)) {
2746 mod_timer(&mdev
->md_sync_timer
, jiffies
+ HZ
);
2747 mdev
->last_md_mark_dirty
.line
= line
;
2748 mdev
->last_md_mark_dirty
.func
= func
;
2752 void drbd_md_mark_dirty(struct drbd_conf
*mdev
)
2754 if (!test_and_set_bit(MD_DIRTY
, &mdev
->flags
))
2755 mod_timer(&mdev
->md_sync_timer
, jiffies
+ 5*HZ
);
2759 static void drbd_uuid_move_history(struct drbd_conf
*mdev
) __must_hold(local
)
2763 for (i
= UI_HISTORY_START
; i
< UI_HISTORY_END
; i
++)
2764 mdev
->ldev
->md
.uuid
[i
+1] = mdev
->ldev
->md
.uuid
[i
];
2767 void _drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2769 if (idx
== UI_CURRENT
) {
2770 if (mdev
->state
.role
== R_PRIMARY
)
2775 drbd_set_ed_uuid(mdev
, val
);
2778 mdev
->ldev
->md
.uuid
[idx
] = val
;
2779 drbd_md_mark_dirty(mdev
);
2783 void drbd_uuid_set(struct drbd_conf
*mdev
, int idx
, u64 val
) __must_hold(local
)
2785 if (mdev
->ldev
->md
.uuid
[idx
]) {
2786 drbd_uuid_move_history(mdev
);
2787 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[idx
];
2789 _drbd_uuid_set(mdev
, idx
, val
);
2793 * drbd_uuid_new_current() - Creates a new current UUID
2794 * @mdev: DRBD device.
2796 * Creates a new current UUID, and rotates the old current UUID into
2797 * the bitmap slot. Causes an incremental resync upon next connect.
2799 void drbd_uuid_new_current(struct drbd_conf
*mdev
) __must_hold(local
)
2802 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2805 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2807 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = mdev
->ldev
->md
.uuid
[UI_CURRENT
];
2809 get_random_bytes(&val
, sizeof(u64
));
2810 _drbd_uuid_set(mdev
, UI_CURRENT
, val
);
2811 drbd_print_uuids(mdev
, "new current UUID");
2812 /* get it to stable storage _now_ */
2816 void drbd_uuid_set_bm(struct drbd_conf
*mdev
, u64 val
) __must_hold(local
)
2818 if (mdev
->ldev
->md
.uuid
[UI_BITMAP
] == 0 && val
== 0)
2822 drbd_uuid_move_history(mdev
);
2823 mdev
->ldev
->md
.uuid
[UI_HISTORY_START
] = mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2824 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = 0;
2826 unsigned long long bm_uuid
= mdev
->ldev
->md
.uuid
[UI_BITMAP
];
2828 dev_warn(DEV
, "bm UUID was already set: %llX\n", bm_uuid
);
2830 mdev
->ldev
->md
.uuid
[UI_BITMAP
] = val
& ~((u64
)1);
2832 drbd_md_mark_dirty(mdev
);
2836 * drbd_bmio_set_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2837 * @mdev: DRBD device.
2839 * Sets all bits in the bitmap and writes the whole bitmap to stable storage.
2841 int drbd_bmio_set_n_write(struct drbd_conf
*mdev
)
2845 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2846 drbd_md_set_flag(mdev
, MDF_FULL_SYNC
);
2848 drbd_bm_set_all(mdev
);
2850 rv
= drbd_bm_write(mdev
);
2853 drbd_md_clear_flag(mdev
, MDF_FULL_SYNC
);
2864 * drbd_bmio_clear_n_write() - io_fn for drbd_queue_bitmap_io() or drbd_bitmap_io()
2865 * @mdev: DRBD device.
2867 * Clears all bits in the bitmap and writes the whole bitmap to stable storage.
2869 int drbd_bmio_clear_n_write(struct drbd_conf
*mdev
)
2873 drbd_resume_al(mdev
);
2874 if (get_ldev_if_state(mdev
, D_ATTACHING
)) {
2875 drbd_bm_clear_all(mdev
);
2876 rv
= drbd_bm_write(mdev
);
2883 static int w_bitmap_io(struct drbd_work
*w
, int unused
)
2885 struct bm_io_work
*work
= container_of(w
, struct bm_io_work
, w
);
2886 struct drbd_conf
*mdev
= w
->mdev
;
2889 D_ASSERT(atomic_read(&mdev
->ap_bio_cnt
) == 0);
2891 if (get_ldev(mdev
)) {
2892 drbd_bm_lock(mdev
, work
->why
, work
->flags
);
2893 rv
= work
->io_fn(mdev
);
2894 drbd_bm_unlock(mdev
);
2898 clear_bit_unlock(BITMAP_IO
, &mdev
->flags
);
2899 wake_up(&mdev
->misc_wait
);
2902 work
->done(mdev
, rv
);
2904 clear_bit(BITMAP_IO_QUEUED
, &mdev
->flags
);
2911 void drbd_ldev_destroy(struct drbd_conf
*mdev
)
2913 lc_destroy(mdev
->resync
);
2914 mdev
->resync
= NULL
;
2915 lc_destroy(mdev
->act_log
);
2916 mdev
->act_log
= NULL
;
2918 drbd_free_bc(mdev
->ldev
);
2919 mdev
->ldev
= NULL
;);
2921 clear_bit(GO_DISKLESS
, &mdev
->flags
);
2924 static int w_go_diskless(struct drbd_work
*w
, int unused
)
2926 struct drbd_conf
*mdev
= w
->mdev
;
2928 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
2929 /* we cannot assert local_cnt == 0 here, as get_ldev_if_state will
2930 * inc/dec it frequently. Once we are D_DISKLESS, no one will touch
2931 * the protected members anymore, though, so once put_ldev reaches zero
2932 * again, it will be safe to free them. */
2933 drbd_force_state(mdev
, NS(disk
, D_DISKLESS
));
2937 void drbd_go_diskless(struct drbd_conf
*mdev
)
2939 D_ASSERT(mdev
->state
.disk
== D_FAILED
);
2940 if (!test_and_set_bit(GO_DISKLESS
, &mdev
->flags
))
2941 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->go_diskless
);
2945 * drbd_queue_bitmap_io() - Queues an IO operation on the whole bitmap
2946 * @mdev: DRBD device.
2947 * @io_fn: IO callback to be called when bitmap IO is possible
2948 * @done: callback to be called after the bitmap IO was performed
2949 * @why: Descriptive text of the reason for doing the IO
2951 * While IO on the bitmap happens we freeze application IO thus we ensure
2952 * that drbd_set_out_of_sync() can not be called. This function MAY ONLY be
2953 * called from worker context. It MUST NOT be used while a previous such
2954 * work is still pending!
2956 void drbd_queue_bitmap_io(struct drbd_conf
*mdev
,
2957 int (*io_fn
)(struct drbd_conf
*),
2958 void (*done
)(struct drbd_conf
*, int),
2959 char *why
, enum bm_flag flags
)
2961 D_ASSERT(current
== mdev
->tconn
->worker
.task
);
2963 D_ASSERT(!test_bit(BITMAP_IO_QUEUED
, &mdev
->flags
));
2964 D_ASSERT(!test_bit(BITMAP_IO
, &mdev
->flags
));
2965 D_ASSERT(list_empty(&mdev
->bm_io_work
.w
.list
));
2966 if (mdev
->bm_io_work
.why
)
2967 dev_err(DEV
, "FIXME going to queue '%s' but '%s' still pending?\n",
2968 why
, mdev
->bm_io_work
.why
);
2970 mdev
->bm_io_work
.io_fn
= io_fn
;
2971 mdev
->bm_io_work
.done
= done
;
2972 mdev
->bm_io_work
.why
= why
;
2973 mdev
->bm_io_work
.flags
= flags
;
2975 spin_lock_irq(&mdev
->tconn
->req_lock
);
2976 set_bit(BITMAP_IO
, &mdev
->flags
);
2977 if (atomic_read(&mdev
->ap_bio_cnt
) == 0) {
2978 if (!test_and_set_bit(BITMAP_IO_QUEUED
, &mdev
->flags
))
2979 drbd_queue_work(&mdev
->tconn
->data
.work
, &mdev
->bm_io_work
.w
);
2981 spin_unlock_irq(&mdev
->tconn
->req_lock
);
2985 * drbd_bitmap_io() - Does an IO operation on the whole bitmap
2986 * @mdev: DRBD device.
2987 * @io_fn: IO callback to be called when bitmap IO is possible
2988 * @why: Descriptive text of the reason for doing the IO
2990 * freezes application IO while that the actual IO operations runs. This
2991 * functions MAY NOT be called from worker context.
2993 int drbd_bitmap_io(struct drbd_conf
*mdev
, int (*io_fn
)(struct drbd_conf
*),
2994 char *why
, enum bm_flag flags
)
2998 D_ASSERT(current
!= mdev
->tconn
->worker
.task
);
3000 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3001 drbd_suspend_io(mdev
);
3003 drbd_bm_lock(mdev
, why
, flags
);
3005 drbd_bm_unlock(mdev
);
3007 if ((flags
& BM_LOCKED_SET_ALLOWED
) == 0)
3008 drbd_resume_io(mdev
);
3013 void drbd_md_set_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3015 if ((mdev
->ldev
->md
.flags
& flag
) != flag
) {
3016 drbd_md_mark_dirty(mdev
);
3017 mdev
->ldev
->md
.flags
|= flag
;
3021 void drbd_md_clear_flag(struct drbd_conf
*mdev
, int flag
) __must_hold(local
)
3023 if ((mdev
->ldev
->md
.flags
& flag
) != 0) {
3024 drbd_md_mark_dirty(mdev
);
3025 mdev
->ldev
->md
.flags
&= ~flag
;
3028 int drbd_md_test_flag(struct drbd_backing_dev
*bdev
, int flag
)
3030 return (bdev
->md
.flags
& flag
) != 0;
3033 static void md_sync_timer_fn(unsigned long data
)
3035 struct drbd_conf
*mdev
= (struct drbd_conf
*) data
;
3037 drbd_queue_work_front(&mdev
->tconn
->data
.work
, &mdev
->md_sync_work
);
3040 static int w_md_sync(struct drbd_work
*w
, int unused
)
3042 struct drbd_conf
*mdev
= w
->mdev
;
3044 dev_warn(DEV
, "md_sync_timer expired! Worker calls drbd_md_sync().\n");
3046 dev_warn(DEV
, "last md_mark_dirty: %s:%u\n",
3047 mdev
->last_md_mark_dirty
.func
, mdev
->last_md_mark_dirty
.line
);
3053 const char *cmdname(enum drbd_packet cmd
)
3055 /* THINK may need to become several global tables
3056 * when we want to support more than
3057 * one PRO_VERSION */
3058 static const char *cmdnames
[] = {
3060 [P_DATA_REPLY
] = "DataReply",
3061 [P_RS_DATA_REPLY
] = "RSDataReply",
3062 [P_BARRIER
] = "Barrier",
3063 [P_BITMAP
] = "ReportBitMap",
3064 [P_BECOME_SYNC_TARGET
] = "BecomeSyncTarget",
3065 [P_BECOME_SYNC_SOURCE
] = "BecomeSyncSource",
3066 [P_UNPLUG_REMOTE
] = "UnplugRemote",
3067 [P_DATA_REQUEST
] = "DataRequest",
3068 [P_RS_DATA_REQUEST
] = "RSDataRequest",
3069 [P_SYNC_PARAM
] = "SyncParam",
3070 [P_SYNC_PARAM89
] = "SyncParam89",
3071 [P_PROTOCOL
] = "ReportProtocol",
3072 [P_UUIDS
] = "ReportUUIDs",
3073 [P_SIZES
] = "ReportSizes",
3074 [P_STATE
] = "ReportState",
3075 [P_SYNC_UUID
] = "ReportSyncUUID",
3076 [P_AUTH_CHALLENGE
] = "AuthChallenge",
3077 [P_AUTH_RESPONSE
] = "AuthResponse",
3079 [P_PING_ACK
] = "PingAck",
3080 [P_RECV_ACK
] = "RecvAck",
3081 [P_WRITE_ACK
] = "WriteAck",
3082 [P_RS_WRITE_ACK
] = "RSWriteAck",
3083 [P_DISCARD_WRITE
] = "DiscardWrite",
3084 [P_NEG_ACK
] = "NegAck",
3085 [P_NEG_DREPLY
] = "NegDReply",
3086 [P_NEG_RS_DREPLY
] = "NegRSDReply",
3087 [P_BARRIER_ACK
] = "BarrierAck",
3088 [P_STATE_CHG_REQ
] = "StateChgRequest",
3089 [P_STATE_CHG_REPLY
] = "StateChgReply",
3090 [P_OV_REQUEST
] = "OVRequest",
3091 [P_OV_REPLY
] = "OVReply",
3092 [P_OV_RESULT
] = "OVResult",
3093 [P_CSUM_RS_REQUEST
] = "CsumRSRequest",
3094 [P_RS_IS_IN_SYNC
] = "CsumRSIsInSync",
3095 [P_COMPRESSED_BITMAP
] = "CBitmap",
3096 [P_DELAY_PROBE
] = "DelayProbe",
3097 [P_OUT_OF_SYNC
] = "OutOfSync",
3098 [P_RETRY_WRITE
] = "RetryWrite",
3101 if (cmd
== P_HAND_SHAKE_M
)
3102 return "HandShakeM";
3103 if (cmd
== P_HAND_SHAKE_S
)
3104 return "HandShakeS";
3105 if (cmd
== P_HAND_SHAKE
)
3107 if (cmd
>= ARRAY_SIZE(cmdnames
))
3109 return cmdnames
[cmd
];
3113 * drbd_wait_misc - wait for a request to make progress
3114 * @mdev: device associated with the request
3115 * @i: the struct drbd_interval embedded in struct drbd_request or
3116 * struct drbd_peer_request
3118 int drbd_wait_misc(struct drbd_conf
*mdev
, struct drbd_interval
*i
)
3120 struct net_conf
*net_conf
= mdev
->tconn
->net_conf
;
3126 timeout
= MAX_SCHEDULE_TIMEOUT
;
3127 if (net_conf
->ko_count
)
3128 timeout
= net_conf
->timeout
* HZ
/ 10 * net_conf
->ko_count
;
3130 /* Indicate to wake up mdev->misc_wait on progress. */
3132 prepare_to_wait(&mdev
->misc_wait
, &wait
, TASK_INTERRUPTIBLE
);
3133 spin_unlock_irq(&mdev
->tconn
->req_lock
);
3134 timeout
= schedule_timeout(timeout
);
3135 finish_wait(&mdev
->misc_wait
, &wait
);
3136 spin_lock_irq(&mdev
->tconn
->req_lock
);
3137 if (!timeout
|| mdev
->state
.conn
< C_CONNECTED
)
3139 if (signal_pending(current
))
3140 return -ERESTARTSYS
;
3144 #ifdef CONFIG_DRBD_FAULT_INJECTION
3145 /* Fault insertion support including random number generator shamelessly
3146 * stolen from kernel/rcutorture.c */
3147 struct fault_random_state
{
3148 unsigned long state
;
3149 unsigned long count
;
3152 #define FAULT_RANDOM_MULT 39916801 /* prime */
3153 #define FAULT_RANDOM_ADD 479001701 /* prime */
3154 #define FAULT_RANDOM_REFRESH 10000
3157 * Crude but fast random-number generator. Uses a linear congruential
3158 * generator, with occasional help from get_random_bytes().
3160 static unsigned long
3161 _drbd_fault_random(struct fault_random_state
*rsp
)
3165 if (!rsp
->count
--) {
3166 get_random_bytes(&refresh
, sizeof(refresh
));
3167 rsp
->state
+= refresh
;
3168 rsp
->count
= FAULT_RANDOM_REFRESH
;
3170 rsp
->state
= rsp
->state
* FAULT_RANDOM_MULT
+ FAULT_RANDOM_ADD
;
3171 return swahw32(rsp
->state
);
3175 _drbd_fault_str(unsigned int type
) {
3176 static char *_faults
[] = {
3177 [DRBD_FAULT_MD_WR
] = "Meta-data write",
3178 [DRBD_FAULT_MD_RD
] = "Meta-data read",
3179 [DRBD_FAULT_RS_WR
] = "Resync write",
3180 [DRBD_FAULT_RS_RD
] = "Resync read",
3181 [DRBD_FAULT_DT_WR
] = "Data write",
3182 [DRBD_FAULT_DT_RD
] = "Data read",
3183 [DRBD_FAULT_DT_RA
] = "Data read ahead",
3184 [DRBD_FAULT_BM_ALLOC
] = "BM allocation",
3185 [DRBD_FAULT_AL_EE
] = "EE allocation",
3186 [DRBD_FAULT_RECEIVE
] = "receive data corruption",
3189 return (type
< DRBD_FAULT_MAX
) ? _faults
[type
] : "**Unknown**";
3193 _drbd_insert_fault(struct drbd_conf
*mdev
, unsigned int type
)
3195 static struct fault_random_state rrs
= {0, 0};
3197 unsigned int ret
= (
3199 ((1 << mdev_to_minor(mdev
)) & fault_devs
) != 0) &&
3200 (((_drbd_fault_random(&rrs
) % 100) + 1) <= fault_rate
));
3205 if (__ratelimit(&drbd_ratelimit_state
))
3206 dev_warn(DEV
, "***Simulating %s failure\n",
3207 _drbd_fault_str(type
));
3214 const char *drbd_buildtag(void)
3216 /* DRBD built from external sources has here a reference to the
3217 git hash of the source code. */
3219 static char buildtag
[38] = "\0uilt-in";
3221 if (buildtag
[0] == 0) {
3222 #ifdef CONFIG_MODULES
3223 if (THIS_MODULE
!= NULL
)
3224 sprintf(buildtag
, "srcversion: %-24s", THIS_MODULE
->srcversion
);
3233 module_init(drbd_init
)
3234 module_exit(drbd_cleanup
)
3236 EXPORT_SYMBOL(drbd_conn_str
);
3237 EXPORT_SYMBOL(drbd_role_str
);
3238 EXPORT_SYMBOL(drbd_disk_str
);
3239 EXPORT_SYMBOL(drbd_set_st_err_str
);