]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - fs/block_dev.c
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net
[mirror_ubuntu-zesty-kernel.git] / fs / block_dev.c
CommitLineData
1da177e4
LT
1/*
2 * linux/fs/block_dev.c
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
6 */
7
1da177e4
LT
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/fcntl.h>
11#include <linux/slab.h>
12#include <linux/kmod.h>
13#include <linux/major.h>
7db9cfd3 14#include <linux/device_cgroup.h>
1da177e4
LT
15#include <linux/highmem.h>
16#include <linux/blkdev.h>
66114cad 17#include <linux/backing-dev.h>
1da177e4
LT
18#include <linux/module.h>
19#include <linux/blkpg.h>
b502bd11 20#include <linux/magic.h>
1da177e4 21#include <linux/buffer_head.h>
ff01bb48 22#include <linux/swap.h>
585d3bc0 23#include <linux/pagevec.h>
811d736f 24#include <linux/writeback.h>
1da177e4
LT
25#include <linux/mpage.h>
26#include <linux/mount.h>
27#include <linux/uio.h>
28#include <linux/namei.h>
1368c4f2 29#include <linux/log2.h>
ff01bb48 30#include <linux/cleancache.h>
c94c2acf 31#include <linux/dax.h>
1da177e4 32#include <asm/uaccess.h>
07f3f05c 33#include "internal.h"
1da177e4
LT
34
35struct bdev_inode {
36 struct block_device bdev;
37 struct inode vfs_inode;
38};
39
4c54ac62
AB
40static const struct address_space_operations def_blk_aops;
41
1da177e4
LT
42static inline struct bdev_inode *BDEV_I(struct inode *inode)
43{
44 return container_of(inode, struct bdev_inode, vfs_inode);
45}
46
ff5053f6 47struct block_device *I_BDEV(struct inode *inode)
1da177e4
LT
48{
49 return &BDEV_I(inode)->bdev;
50}
1da177e4
LT
51EXPORT_SYMBOL(I_BDEV);
52
dbd3ca50 53static void bdev_write_inode(struct block_device *bdev)
564f00f6 54{
dbd3ca50
VG
55 struct inode *inode = bdev->bd_inode;
56 int ret;
57
564f00f6
CH
58 spin_lock(&inode->i_lock);
59 while (inode->i_state & I_DIRTY) {
60 spin_unlock(&inode->i_lock);
dbd3ca50
VG
61 ret = write_inode_now(inode, true);
62 if (ret) {
63 char name[BDEVNAME_SIZE];
64 pr_warn_ratelimited("VFS: Dirty inode writeback failed "
65 "for block device %s (err=%d).\n",
66 bdevname(bdev, name), ret);
67 }
564f00f6
CH
68 spin_lock(&inode->i_lock);
69 }
70 spin_unlock(&inode->i_lock);
71}
72
f9a14399 73/* Kill _all_ buffers and pagecache , dirty or not.. */
ff01bb48 74void kill_bdev(struct block_device *bdev)
1da177e4 75{
ff01bb48
AV
76 struct address_space *mapping = bdev->bd_inode->i_mapping;
77
f9fe48be 78 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
f9a14399 79 return;
ff01bb48 80
f9a14399 81 invalidate_bh_lrus();
ff01bb48 82 truncate_inode_pages(mapping, 0);
1da177e4 83}
ff01bb48
AV
84EXPORT_SYMBOL(kill_bdev);
85
86/* Invalidate clean unused buffers and pagecache. */
87void invalidate_bdev(struct block_device *bdev)
88{
89 struct address_space *mapping = bdev->bd_inode->i_mapping;
90
91 if (mapping->nrpages == 0)
92 return;
93
94 invalidate_bh_lrus();
95 lru_add_drain_all(); /* make sure all lru add caches are flushed */
96 invalidate_mapping_pages(mapping, 0, -1);
97 /* 99% of the time, we don't need to flush the cleancache on the bdev.
98 * But, for the strange corners, lets be cautious
99 */
3167760f 100 cleancache_invalidate_inode(mapping);
ff01bb48
AV
101}
102EXPORT_SYMBOL(invalidate_bdev);
1da177e4
LT
103
104int set_blocksize(struct block_device *bdev, int size)
105{
106 /* Size must be a power of two, and between 512 and PAGE_SIZE */
1368c4f2 107 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
1da177e4
LT
108 return -EINVAL;
109
110 /* Size cannot be smaller than the size supported by the device */
e1defc4f 111 if (size < bdev_logical_block_size(bdev))
1da177e4
LT
112 return -EINVAL;
113
114 /* Don't change the size if it is same as current */
115 if (bdev->bd_block_size != size) {
116 sync_blockdev(bdev);
117 bdev->bd_block_size = size;
118 bdev->bd_inode->i_blkbits = blksize_bits(size);
119 kill_bdev(bdev);
120 }
121 return 0;
122}
123
124EXPORT_SYMBOL(set_blocksize);
125
126int sb_set_blocksize(struct super_block *sb, int size)
127{
1da177e4
LT
128 if (set_blocksize(sb->s_bdev, size))
129 return 0;
130 /* If we get here, we know size is power of two
131 * and it's value is between 512 and PAGE_SIZE */
132 sb->s_blocksize = size;
38885bd4 133 sb->s_blocksize_bits = blksize_bits(size);
1da177e4
LT
134 return sb->s_blocksize;
135}
136
137EXPORT_SYMBOL(sb_set_blocksize);
138
139int sb_min_blocksize(struct super_block *sb, int size)
140{
e1defc4f 141 int minsize = bdev_logical_block_size(sb->s_bdev);
1da177e4
LT
142 if (size < minsize)
143 size = minsize;
144 return sb_set_blocksize(sb, size);
145}
146
147EXPORT_SYMBOL(sb_min_blocksize);
148
149static int
150blkdev_get_block(struct inode *inode, sector_t iblock,
151 struct buffer_head *bh, int create)
152{
1da177e4
LT
153 bh->b_bdev = I_BDEV(inode);
154 bh->b_blocknr = iblock;
155 set_buffer_mapped(bh);
156 return 0;
157}
158
4ebb16ca
DW
159static struct inode *bdev_file_inode(struct file *file)
160{
161 return file->f_mapping->host;
162}
163
b2e895db 164static ssize_t
22c6186e 165blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
b2e895db
AM
166{
167 struct file *file = iocb->ki_filp;
4ebb16ca 168 struct inode *inode = bdev_file_inode(file);
b2e895db 169
bbab37dd
MW
170 if (IS_DAX(inode))
171 return dax_do_io(iocb, inode, iter, offset, blkdev_get_block,
172 NULL, DIO_SKIP_DIO_COUNT);
17f8c842 173 return __blockdev_direct_IO(iocb, inode, I_BDEV(inode), iter, offset,
fe0f07d0
JA
174 blkdev_get_block, NULL, NULL,
175 DIO_SKIP_DIO_COUNT);
b2e895db
AM
176}
177
5cee5815
JK
178int __sync_blockdev(struct block_device *bdev, int wait)
179{
180 if (!bdev)
181 return 0;
182 if (!wait)
183 return filemap_flush(bdev->bd_inode->i_mapping);
184 return filemap_write_and_wait(bdev->bd_inode->i_mapping);
185}
186
585d3bc0
NP
187/*
188 * Write out and wait upon all the dirty data associated with a block
189 * device via its mapping. Does not take the superblock lock.
190 */
191int sync_blockdev(struct block_device *bdev)
192{
5cee5815 193 return __sync_blockdev(bdev, 1);
585d3bc0
NP
194}
195EXPORT_SYMBOL(sync_blockdev);
196
197/*
198 * Write out and wait upon all dirty data associated with this
199 * device. Filesystem data as well as the underlying block
200 * device. Takes the superblock lock.
201 */
202int fsync_bdev(struct block_device *bdev)
203{
204 struct super_block *sb = get_super(bdev);
205 if (sb) {
60b0680f 206 int res = sync_filesystem(sb);
585d3bc0
NP
207 drop_super(sb);
208 return res;
209 }
210 return sync_blockdev(bdev);
211}
47e4491b 212EXPORT_SYMBOL(fsync_bdev);
585d3bc0
NP
213
214/**
215 * freeze_bdev -- lock a filesystem and force it into a consistent state
216 * @bdev: blockdevice to lock
217 *
585d3bc0
NP
218 * If a superblock is found on this device, we take the s_umount semaphore
219 * on it to make sure nobody unmounts until the snapshot creation is done.
220 * The reference counter (bd_fsfreeze_count) guarantees that only the last
221 * unfreeze process can unfreeze the frozen filesystem actually when multiple
222 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
223 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
224 * actually.
225 */
226struct super_block *freeze_bdev(struct block_device *bdev)
227{
228 struct super_block *sb;
229 int error = 0;
230
231 mutex_lock(&bdev->bd_fsfreeze_mutex);
4504230a
CH
232 if (++bdev->bd_fsfreeze_count > 1) {
233 /*
234 * We don't even need to grab a reference - the first call
235 * to freeze_bdev grab an active reference and only the last
236 * thaw_bdev drops it.
237 */
585d3bc0 238 sb = get_super(bdev);
4504230a
CH
239 drop_super(sb);
240 mutex_unlock(&bdev->bd_fsfreeze_mutex);
241 return sb;
242 }
243
244 sb = get_active_super(bdev);
245 if (!sb)
246 goto out;
48b6bca6
BM
247 if (sb->s_op->freeze_super)
248 error = sb->s_op->freeze_super(sb);
249 else
250 error = freeze_super(sb);
18e9e510
JB
251 if (error) {
252 deactivate_super(sb);
253 bdev->bd_fsfreeze_count--;
585d3bc0 254 mutex_unlock(&bdev->bd_fsfreeze_mutex);
18e9e510 255 return ERR_PTR(error);
585d3bc0 256 }
18e9e510 257 deactivate_super(sb);
4504230a 258 out:
585d3bc0
NP
259 sync_blockdev(bdev);
260 mutex_unlock(&bdev->bd_fsfreeze_mutex);
4fadd7bb 261 return sb; /* thaw_bdev releases s->s_umount */
585d3bc0
NP
262}
263EXPORT_SYMBOL(freeze_bdev);
264
265/**
266 * thaw_bdev -- unlock filesystem
267 * @bdev: blockdevice to unlock
268 * @sb: associated superblock
269 *
270 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
271 */
272int thaw_bdev(struct block_device *bdev, struct super_block *sb)
273{
4504230a 274 int error = -EINVAL;
585d3bc0
NP
275
276 mutex_lock(&bdev->bd_fsfreeze_mutex);
4504230a 277 if (!bdev->bd_fsfreeze_count)
18e9e510 278 goto out;
4504230a
CH
279
280 error = 0;
281 if (--bdev->bd_fsfreeze_count > 0)
18e9e510 282 goto out;
4504230a
CH
283
284 if (!sb)
18e9e510 285 goto out;
4504230a 286
48b6bca6
BM
287 if (sb->s_op->thaw_super)
288 error = sb->s_op->thaw_super(sb);
289 else
290 error = thaw_super(sb);
18e9e510
JB
291 if (error) {
292 bdev->bd_fsfreeze_count++;
293 mutex_unlock(&bdev->bd_fsfreeze_mutex);
294 return error;
295 }
296out:
585d3bc0
NP
297 mutex_unlock(&bdev->bd_fsfreeze_mutex);
298 return 0;
299}
300EXPORT_SYMBOL(thaw_bdev);
301
1da177e4
LT
302static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
303{
304 return block_write_full_page(page, blkdev_get_block, wbc);
305}
306
307static int blkdev_readpage(struct file * file, struct page * page)
308{
309 return block_read_full_page(page, blkdev_get_block);
310}
311
447f05bb
AM
312static int blkdev_readpages(struct file *file, struct address_space *mapping,
313 struct list_head *pages, unsigned nr_pages)
314{
315 return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block);
316}
317
6272b5a5
NP
318static int blkdev_write_begin(struct file *file, struct address_space *mapping,
319 loff_t pos, unsigned len, unsigned flags,
320 struct page **pagep, void **fsdata)
1da177e4 321{
155130a4
CH
322 return block_write_begin(mapping, pos, len, flags, pagep,
323 blkdev_get_block);
1da177e4
LT
324}
325
6272b5a5
NP
326static int blkdev_write_end(struct file *file, struct address_space *mapping,
327 loff_t pos, unsigned len, unsigned copied,
328 struct page *page, void *fsdata)
1da177e4 329{
6272b5a5
NP
330 int ret;
331 ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);
332
333 unlock_page(page);
334 page_cache_release(page);
335
336 return ret;
1da177e4
LT
337}
338
339/*
340 * private llseek:
496ad9aa 341 * for a block special file file_inode(file)->i_size is zero
1da177e4
LT
342 * so we compute the size by hand (just as in block_read/write above)
343 */
965c8e59 344static loff_t block_llseek(struct file *file, loff_t offset, int whence)
1da177e4 345{
4ebb16ca 346 struct inode *bd_inode = bdev_file_inode(file);
1da177e4
LT
347 loff_t retval;
348
5955102c 349 inode_lock(bd_inode);
5d48f3a2 350 retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
5955102c 351 inode_unlock(bd_inode);
1da177e4
LT
352 return retval;
353}
354
02c24a82 355int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
1da177e4 356{
4ebb16ca 357 struct inode *bd_inode = bdev_file_inode(filp);
b8af67e2 358 struct block_device *bdev = I_BDEV(bd_inode);
ab0a9735 359 int error;
da5aa861
RW
360
361 error = filemap_write_and_wait_range(filp->f_mapping, start, end);
362 if (error)
363 return error;
ab0a9735 364
b8af67e2
AB
365 /*
366 * There is no need to serialise calls to blkdev_issue_flush with
367 * i_mutex and doing so causes performance issues with concurrent
368 * O_SYNC writers to a block device.
369 */
dd3932ed 370 error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL);
ab0a9735
CH
371 if (error == -EOPNOTSUPP)
372 error = 0;
b8af67e2 373
ab0a9735 374 return error;
1da177e4 375}
b1dd3b28 376EXPORT_SYMBOL(blkdev_fsync);
1da177e4 377
47a191fd
MW
378/**
379 * bdev_read_page() - Start reading a page from a block device
380 * @bdev: The device to read the page from
381 * @sector: The offset on the device to read the page to (need not be aligned)
382 * @page: The page to read
383 *
384 * On entry, the page should be locked. It will be unlocked when the page
385 * has been read. If the block driver implements rw_page synchronously,
386 * that will be true on exit from this function, but it need not be.
387 *
388 * Errors returned by this function are usually "soft", eg out of memory, or
389 * queue full; callers should try a different route to read this page rather
390 * than propagate an error back up the stack.
391 *
392 * Return: negative errno if an error occurs, 0 if submission was successful.
393 */
394int bdev_read_page(struct block_device *bdev, sector_t sector,
395 struct page *page)
396{
397 const struct block_device_operations *ops = bdev->bd_disk->fops;
2e6edc95
DW
398 int result = -EOPNOTSUPP;
399
f68eb1e7 400 if (!ops->rw_page || bdev_get_integrity(bdev))
2e6edc95
DW
401 return result;
402
6f3b0e8b 403 result = blk_queue_enter(bdev->bd_queue, false);
2e6edc95
DW
404 if (result)
405 return result;
406 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, READ);
407 blk_queue_exit(bdev->bd_queue);
408 return result;
47a191fd
MW
409}
410EXPORT_SYMBOL_GPL(bdev_read_page);
411
412/**
413 * bdev_write_page() - Start writing a page to a block device
414 * @bdev: The device to write the page to
415 * @sector: The offset on the device to write the page to (need not be aligned)
416 * @page: The page to write
417 * @wbc: The writeback_control for the write
418 *
419 * On entry, the page should be locked and not currently under writeback.
420 * On exit, if the write started successfully, the page will be unlocked and
421 * under writeback. If the write failed already (eg the driver failed to
422 * queue the page to the device), the page will still be locked. If the
423 * caller is a ->writepage implementation, it will need to unlock the page.
424 *
425 * Errors returned by this function are usually "soft", eg out of memory, or
426 * queue full; callers should try a different route to write this page rather
427 * than propagate an error back up the stack.
428 *
429 * Return: negative errno if an error occurs, 0 if submission was successful.
430 */
431int bdev_write_page(struct block_device *bdev, sector_t sector,
432 struct page *page, struct writeback_control *wbc)
433{
434 int result;
435 int rw = (wbc->sync_mode == WB_SYNC_ALL) ? WRITE_SYNC : WRITE;
436 const struct block_device_operations *ops = bdev->bd_disk->fops;
2e6edc95 437
f68eb1e7 438 if (!ops->rw_page || bdev_get_integrity(bdev))
47a191fd 439 return -EOPNOTSUPP;
6f3b0e8b 440 result = blk_queue_enter(bdev->bd_queue, false);
2e6edc95
DW
441 if (result)
442 return result;
443
47a191fd
MW
444 set_page_writeback(page);
445 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, rw);
446 if (result)
447 end_page_writeback(page);
448 else
449 unlock_page(page);
2e6edc95 450 blk_queue_exit(bdev->bd_queue);
47a191fd
MW
451 return result;
452}
453EXPORT_SYMBOL_GPL(bdev_write_page);
454
dd22f551
MW
455/**
456 * bdev_direct_access() - Get the address for directly-accessibly memory
457 * @bdev: The device containing the memory
b2e0d162 458 * @dax: control and output parameters for ->direct_access
dd22f551
MW
459 *
460 * If a block device is made up of directly addressable memory, this function
461 * will tell the caller the PFN and the address of the memory. The address
462 * may be directly dereferenced within the kernel without the need to call
463 * ioremap(), kmap() or similar. The PFN is suitable for inserting into
464 * page tables.
465 *
466 * Return: negative errno if an error occurs, otherwise the number of bytes
467 * accessible at this address.
468 */
b2e0d162 469long bdev_direct_access(struct block_device *bdev, struct blk_dax_ctl *dax)
dd22f551 470{
b2e0d162
DW
471 sector_t sector = dax->sector;
472 long avail, size = dax->size;
dd22f551
MW
473 const struct block_device_operations *ops = bdev->bd_disk->fops;
474
43c3dd08
MW
475 /*
476 * The device driver is allowed to sleep, in order to make the
477 * memory directly accessible.
478 */
479 might_sleep();
480
dd22f551
MW
481 if (size < 0)
482 return size;
483 if (!ops->direct_access)
484 return -EOPNOTSUPP;
485 if ((sector + DIV_ROUND_UP(size, 512)) >
486 part_nr_sects_read(bdev->bd_part))
487 return -ERANGE;
488 sector += get_start_sect(bdev);
489 if (sector % (PAGE_SIZE / 512))
490 return -EINVAL;
b2e0d162 491 avail = ops->direct_access(bdev, sector, &dax->addr, &dax->pfn);
dd22f551
MW
492 if (!avail)
493 return -ERANGE;
fe683ada
DW
494 if (avail > 0 && avail & ~PAGE_MASK)
495 return -ENXIO;
dd22f551
MW
496 return min(avail, size);
497}
498EXPORT_SYMBOL_GPL(bdev_direct_access);
499
1da177e4
LT
500/*
501 * pseudo-fs
502 */
503
504static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
e18b890b 505static struct kmem_cache * bdev_cachep __read_mostly;
1da177e4
LT
506
507static struct inode *bdev_alloc_inode(struct super_block *sb)
508{
e94b1766 509 struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
1da177e4
LT
510 if (!ei)
511 return NULL;
512 return &ei->vfs_inode;
513}
514
fa0d7e3d 515static void bdev_i_callback(struct rcu_head *head)
1da177e4 516{
fa0d7e3d 517 struct inode *inode = container_of(head, struct inode, i_rcu);
1da177e4
LT
518 struct bdev_inode *bdi = BDEV_I(inode);
519
1da177e4
LT
520 kmem_cache_free(bdev_cachep, bdi);
521}
522
fa0d7e3d
NP
523static void bdev_destroy_inode(struct inode *inode)
524{
525 call_rcu(&inode->i_rcu, bdev_i_callback);
526}
527
51cc5068 528static void init_once(void *foo)
1da177e4
LT
529{
530 struct bdev_inode *ei = (struct bdev_inode *) foo;
531 struct block_device *bdev = &ei->bdev;
532
a35afb83
CL
533 memset(bdev, 0, sizeof(*bdev));
534 mutex_init(&bdev->bd_mutex);
a35afb83
CL
535 INIT_LIST_HEAD(&bdev->bd_inodes);
536 INIT_LIST_HEAD(&bdev->bd_list);
49731baa
TH
537#ifdef CONFIG_SYSFS
538 INIT_LIST_HEAD(&bdev->bd_holder_disks);
539#endif
a35afb83 540 inode_init_once(&ei->vfs_inode);
fcccf502
TS
541 /* Initialize mutex for freeze. */
542 mutex_init(&bdev->bd_fsfreeze_mutex);
1da177e4
LT
543}
544
545static inline void __bd_forget(struct inode *inode)
546{
547 list_del_init(&inode->i_devices);
548 inode->i_bdev = NULL;
549 inode->i_mapping = &inode->i_data;
550}
551
b57922d9 552static void bdev_evict_inode(struct inode *inode)
1da177e4
LT
553{
554 struct block_device *bdev = &BDEV_I(inode)->bdev;
555 struct list_head *p;
91b0abe3 556 truncate_inode_pages_final(&inode->i_data);
b57922d9 557 invalidate_inode_buffers(inode); /* is it needed here? */
dbd5768f 558 clear_inode(inode);
1da177e4
LT
559 spin_lock(&bdev_lock);
560 while ( (p = bdev->bd_inodes.next) != &bdev->bd_inodes ) {
561 __bd_forget(list_entry(p, struct inode, i_devices));
562 }
563 list_del_init(&bdev->bd_list);
564 spin_unlock(&bdev_lock);
565}
566
ee9b6d61 567static const struct super_operations bdev_sops = {
1da177e4
LT
568 .statfs = simple_statfs,
569 .alloc_inode = bdev_alloc_inode,
570 .destroy_inode = bdev_destroy_inode,
571 .drop_inode = generic_delete_inode,
b57922d9 572 .evict_inode = bdev_evict_inode,
1da177e4
LT
573};
574
51139ada
AV
575static struct dentry *bd_mount(struct file_system_type *fs_type,
576 int flags, const char *dev_name, void *data)
1da177e4 577{
b502bd11 578 return mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC);
1da177e4
LT
579}
580
581static struct file_system_type bd_type = {
582 .name = "bdev",
51139ada 583 .mount = bd_mount,
1da177e4
LT
584 .kill_sb = kill_anon_super,
585};
586
a212b105
TH
587struct super_block *blockdev_superblock __read_mostly;
588EXPORT_SYMBOL_GPL(blockdev_superblock);
1da177e4
LT
589
590void __init bdev_cache_init(void)
591{
592 int err;
ace8577a 593 static struct vfsmount *bd_mnt;
c2acf7b9 594
1da177e4 595 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
fffb60f9 596 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
5d097056 597 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
20c2df83 598 init_once);
1da177e4
LT
599 err = register_filesystem(&bd_type);
600 if (err)
601 panic("Cannot register bdev pseudo-fs");
602 bd_mnt = kern_mount(&bd_type);
1da177e4
LT
603 if (IS_ERR(bd_mnt))
604 panic("Cannot create bdev pseudo-fs");
ace8577a 605 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */
1da177e4
LT
606}
607
608/*
609 * Most likely _very_ bad one - but then it's hardly critical for small
610 * /dev and can be fixed when somebody will need really large one.
611 * Keep in mind that it will be fed through icache hash function too.
612 */
613static inline unsigned long hash(dev_t dev)
614{
615 return MAJOR(dev)+MINOR(dev);
616}
617
618static int bdev_test(struct inode *inode, void *data)
619{
620 return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
621}
622
623static int bdev_set(struct inode *inode, void *data)
624{
625 BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
626 return 0;
627}
628
629static LIST_HEAD(all_bdevs);
630
631struct block_device *bdget(dev_t dev)
632{
633 struct block_device *bdev;
634 struct inode *inode;
635
c2acf7b9 636 inode = iget5_locked(blockdev_superblock, hash(dev),
1da177e4
LT
637 bdev_test, bdev_set, &dev);
638
639 if (!inode)
640 return NULL;
641
642 bdev = &BDEV_I(inode)->bdev;
643
644 if (inode->i_state & I_NEW) {
645 bdev->bd_contains = NULL;
782b94cd 646 bdev->bd_super = NULL;
1da177e4
LT
647 bdev->bd_inode = inode;
648 bdev->bd_block_size = (1 << inode->i_blkbits);
649 bdev->bd_part_count = 0;
650 bdev->bd_invalidated = 0;
651 inode->i_mode = S_IFBLK;
652 inode->i_rdev = dev;
653 inode->i_bdev = bdev;
654 inode->i_data.a_ops = &def_blk_aops;
655 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
1da177e4
LT
656 spin_lock(&bdev_lock);
657 list_add(&bdev->bd_list, &all_bdevs);
658 spin_unlock(&bdev_lock);
659 unlock_new_inode(inode);
660 }
661 return bdev;
662}
663
664EXPORT_SYMBOL(bdget);
665
dddac6a7
AJ
666/**
667 * bdgrab -- Grab a reference to an already referenced block device
668 * @bdev: Block device to grab a reference to.
669 */
670struct block_device *bdgrab(struct block_device *bdev)
671{
7de9c6ee 672 ihold(bdev->bd_inode);
dddac6a7
AJ
673 return bdev;
674}
c1681bf8 675EXPORT_SYMBOL(bdgrab);
dddac6a7 676
1da177e4
LT
677long nr_blockdev_pages(void)
678{
203a2935 679 struct block_device *bdev;
1da177e4
LT
680 long ret = 0;
681 spin_lock(&bdev_lock);
203a2935 682 list_for_each_entry(bdev, &all_bdevs, bd_list) {
1da177e4
LT
683 ret += bdev->bd_inode->i_mapping->nrpages;
684 }
685 spin_unlock(&bdev_lock);
686 return ret;
687}
688
689void bdput(struct block_device *bdev)
690{
691 iput(bdev->bd_inode);
692}
693
694EXPORT_SYMBOL(bdput);
695
696static struct block_device *bd_acquire(struct inode *inode)
697{
698 struct block_device *bdev;
09d967c6 699
1da177e4
LT
700 spin_lock(&bdev_lock);
701 bdev = inode->i_bdev;
09d967c6 702 if (bdev) {
ed8a9d2c 703 bdgrab(bdev);
1da177e4
LT
704 spin_unlock(&bdev_lock);
705 return bdev;
706 }
707 spin_unlock(&bdev_lock);
09d967c6 708
1da177e4
LT
709 bdev = bdget(inode->i_rdev);
710 if (bdev) {
711 spin_lock(&bdev_lock);
09d967c6
OH
712 if (!inode->i_bdev) {
713 /*
7de9c6ee 714 * We take an additional reference to bd_inode,
09d967c6
OH
715 * and it's released in clear_inode() of inode.
716 * So, we can access it via ->i_mapping always
717 * without igrab().
718 */
ed8a9d2c 719 bdgrab(bdev);
09d967c6
OH
720 inode->i_bdev = bdev;
721 inode->i_mapping = bdev->bd_inode->i_mapping;
722 list_add(&inode->i_devices, &bdev->bd_inodes);
723 }
1da177e4
LT
724 spin_unlock(&bdev_lock);
725 }
726 return bdev;
727}
728
729/* Call when you free inode */
730
731void bd_forget(struct inode *inode)
732{
09d967c6
OH
733 struct block_device *bdev = NULL;
734
1da177e4 735 spin_lock(&bdev_lock);
b4ea2eaa
YH
736 if (!sb_is_blkdev_sb(inode->i_sb))
737 bdev = inode->i_bdev;
738 __bd_forget(inode);
1da177e4 739 spin_unlock(&bdev_lock);
09d967c6
OH
740
741 if (bdev)
ed8a9d2c 742 bdput(bdev);
1da177e4
LT
743}
744
1a3cbbc5
TH
745/**
746 * bd_may_claim - test whether a block device can be claimed
747 * @bdev: block device of interest
748 * @whole: whole block device containing @bdev, may equal @bdev
749 * @holder: holder trying to claim @bdev
750 *
25985edc 751 * Test whether @bdev can be claimed by @holder.
1a3cbbc5
TH
752 *
753 * CONTEXT:
754 * spin_lock(&bdev_lock).
755 *
756 * RETURNS:
757 * %true if @bdev can be claimed, %false otherwise.
758 */
759static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
760 void *holder)
1da177e4 761{
1da177e4 762 if (bdev->bd_holder == holder)
1a3cbbc5 763 return true; /* already a holder */
1da177e4 764 else if (bdev->bd_holder != NULL)
1a3cbbc5 765 return false; /* held by someone else */
1da177e4 766 else if (bdev->bd_contains == bdev)
1a3cbbc5 767 return true; /* is a whole device which isn't held */
1da177e4 768
e525fd89 769 else if (whole->bd_holder == bd_may_claim)
1a3cbbc5
TH
770 return true; /* is a partition of a device that is being partitioned */
771 else if (whole->bd_holder != NULL)
772 return false; /* is a partition of a held device */
1da177e4 773 else
1a3cbbc5
TH
774 return true; /* is a partition of an un-held device */
775}
776
6b4517a7
TH
777/**
778 * bd_prepare_to_claim - prepare to claim a block device
779 * @bdev: block device of interest
780 * @whole: the whole device containing @bdev, may equal @bdev
781 * @holder: holder trying to claim @bdev
782 *
783 * Prepare to claim @bdev. This function fails if @bdev is already
784 * claimed by another holder and waits if another claiming is in
785 * progress. This function doesn't actually claim. On successful
786 * return, the caller has ownership of bd_claiming and bd_holder[s].
787 *
788 * CONTEXT:
789 * spin_lock(&bdev_lock). Might release bdev_lock, sleep and regrab
790 * it multiple times.
791 *
792 * RETURNS:
793 * 0 if @bdev can be claimed, -EBUSY otherwise.
794 */
795static int bd_prepare_to_claim(struct block_device *bdev,
796 struct block_device *whole, void *holder)
797{
798retry:
799 /* if someone else claimed, fail */
800 if (!bd_may_claim(bdev, whole, holder))
801 return -EBUSY;
802
e75aa858
TH
803 /* if claiming is already in progress, wait for it to finish */
804 if (whole->bd_claiming) {
6b4517a7
TH
805 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
806 DEFINE_WAIT(wait);
807
808 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
809 spin_unlock(&bdev_lock);
810 schedule();
811 finish_wait(wq, &wait);
812 spin_lock(&bdev_lock);
813 goto retry;
814 }
815
816 /* yay, all mine */
817 return 0;
818}
819
820/**
821 * bd_start_claiming - start claiming a block device
822 * @bdev: block device of interest
823 * @holder: holder trying to claim @bdev
824 *
825 * @bdev is about to be opened exclusively. Check @bdev can be opened
826 * exclusively and mark that an exclusive open is in progress. Each
827 * successful call to this function must be matched with a call to
b0018361
NP
828 * either bd_finish_claiming() or bd_abort_claiming() (which do not
829 * fail).
830 *
831 * This function is used to gain exclusive access to the block device
832 * without actually causing other exclusive open attempts to fail. It
833 * should be used when the open sequence itself requires exclusive
834 * access but may subsequently fail.
6b4517a7
TH
835 *
836 * CONTEXT:
837 * Might sleep.
838 *
839 * RETURNS:
840 * Pointer to the block device containing @bdev on success, ERR_PTR()
841 * value on failure.
842 */
843static struct block_device *bd_start_claiming(struct block_device *bdev,
844 void *holder)
845{
846 struct gendisk *disk;
847 struct block_device *whole;
848 int partno, err;
849
850 might_sleep();
851
852 /*
853 * @bdev might not have been initialized properly yet, look up
854 * and grab the outer block device the hard way.
855 */
856 disk = get_gendisk(bdev->bd_dev, &partno);
857 if (!disk)
858 return ERR_PTR(-ENXIO);
859
d4c208b8
TH
860 /*
861 * Normally, @bdev should equal what's returned from bdget_disk()
862 * if partno is 0; however, some drivers (floppy) use multiple
863 * bdev's for the same physical device and @bdev may be one of the
864 * aliases. Keep @bdev if partno is 0. This means claimer
865 * tracking is broken for those devices but it has always been that
866 * way.
867 */
868 if (partno)
869 whole = bdget_disk(disk, 0);
870 else
871 whole = bdgrab(bdev);
872
cf342570 873 module_put(disk->fops->owner);
6b4517a7
TH
874 put_disk(disk);
875 if (!whole)
876 return ERR_PTR(-ENOMEM);
877
878 /* prepare to claim, if successful, mark claiming in progress */
879 spin_lock(&bdev_lock);
880
881 err = bd_prepare_to_claim(bdev, whole, holder);
882 if (err == 0) {
883 whole->bd_claiming = holder;
884 spin_unlock(&bdev_lock);
885 return whole;
886 } else {
887 spin_unlock(&bdev_lock);
888 bdput(whole);
889 return ERR_PTR(err);
890 }
891}
892
641dc636 893#ifdef CONFIG_SYSFS
49731baa
TH
894struct bd_holder_disk {
895 struct list_head list;
896 struct gendisk *disk;
897 int refcnt;
898};
899
900static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
901 struct gendisk *disk)
902{
903 struct bd_holder_disk *holder;
904
905 list_for_each_entry(holder, &bdev->bd_holder_disks, list)
906 if (holder->disk == disk)
907 return holder;
908 return NULL;
909}
910
4d7dd8fd 911static int add_symlink(struct kobject *from, struct kobject *to)
641dc636 912{
4d7dd8fd 913 return sysfs_create_link(from, to, kobject_name(to));
641dc636
JN
914}
915
916static void del_symlink(struct kobject *from, struct kobject *to)
917{
641dc636
JN
918 sysfs_remove_link(from, kobject_name(to));
919}
920
df6c0cd9 921/**
e09b457b
TH
922 * bd_link_disk_holder - create symlinks between holding disk and slave bdev
923 * @bdev: the claimed slave bdev
924 * @disk: the holding disk
df6c0cd9 925 *
49731baa
TH
926 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
927 *
e09b457b 928 * This functions creates the following sysfs symlinks.
641dc636 929 *
e09b457b
TH
930 * - from "slaves" directory of the holder @disk to the claimed @bdev
931 * - from "holders" directory of the @bdev to the holder @disk
641dc636 932 *
e09b457b
TH
933 * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
934 * passed to bd_link_disk_holder(), then:
641dc636 935 *
e09b457b
TH
936 * /sys/block/dm-0/slaves/sda --> /sys/block/sda
937 * /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
641dc636 938 *
e09b457b
TH
939 * The caller must have claimed @bdev before calling this function and
940 * ensure that both @bdev and @disk are valid during the creation and
941 * lifetime of these symlinks.
641dc636 942 *
e09b457b
TH
943 * CONTEXT:
944 * Might sleep.
641dc636 945 *
e09b457b
TH
946 * RETURNS:
947 * 0 on success, -errno on failure.
641dc636 948 */
e09b457b 949int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
641dc636 950{
49731baa 951 struct bd_holder_disk *holder;
e09b457b 952 int ret = 0;
641dc636 953
2e7b651d 954 mutex_lock(&bdev->bd_mutex);
df6c0cd9 955
49731baa 956 WARN_ON_ONCE(!bdev->bd_holder);
4e91672c 957
e09b457b
TH
958 /* FIXME: remove the following once add_disk() handles errors */
959 if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
960 goto out_unlock;
4e91672c 961
49731baa
TH
962 holder = bd_find_holder_disk(bdev, disk);
963 if (holder) {
964 holder->refcnt++;
e09b457b 965 goto out_unlock;
49731baa 966 }
641dc636 967
49731baa
TH
968 holder = kzalloc(sizeof(*holder), GFP_KERNEL);
969 if (!holder) {
970 ret = -ENOMEM;
e09b457b
TH
971 goto out_unlock;
972 }
641dc636 973
49731baa
TH
974 INIT_LIST_HEAD(&holder->list);
975 holder->disk = disk;
976 holder->refcnt = 1;
977
978 ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
979 if (ret)
980 goto out_free;
981
982 ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
983 if (ret)
984 goto out_del;
e7407d16
TH
985 /*
986 * bdev could be deleted beneath us which would implicitly destroy
987 * the holder directory. Hold on to it.
988 */
989 kobject_get(bdev->bd_part->holder_dir);
49731baa
TH
990
991 list_add(&holder->list, &bdev->bd_holder_disks);
992 goto out_unlock;
993
994out_del:
995 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
996out_free:
997 kfree(holder);
e09b457b 998out_unlock:
b4cf1b72 999 mutex_unlock(&bdev->bd_mutex);
e09b457b 1000 return ret;
641dc636 1001}
e09b457b 1002EXPORT_SYMBOL_GPL(bd_link_disk_holder);
641dc636 1003
49731baa
TH
1004/**
1005 * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
1006 * @bdev: the calimed slave bdev
1007 * @disk: the holding disk
1008 *
1009 * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
1010 *
1011 * CONTEXT:
1012 * Might sleep.
1013 */
1014void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
641dc636 1015{
49731baa 1016 struct bd_holder_disk *holder;
641dc636 1017
49731baa 1018 mutex_lock(&bdev->bd_mutex);
641dc636 1019
49731baa
TH
1020 holder = bd_find_holder_disk(bdev, disk);
1021
1022 if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
1023 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1024 del_symlink(bdev->bd_part->holder_dir,
1025 &disk_to_dev(disk)->kobj);
e7407d16 1026 kobject_put(bdev->bd_part->holder_dir);
49731baa
TH
1027 list_del_init(&holder->list);
1028 kfree(holder);
1029 }
1030
1031 mutex_unlock(&bdev->bd_mutex);
1da177e4 1032}
49731baa 1033EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
641dc636 1034#endif
1da177e4 1035
56ade44b
AP
1036/**
1037 * flush_disk - invalidates all buffer-cache entries on a disk
1038 *
1039 * @bdev: struct block device to be flushed
e6eb5ce1 1040 * @kill_dirty: flag to guide handling of dirty inodes
56ade44b
AP
1041 *
1042 * Invalidates all buffer-cache entries on a disk. It should be called
1043 * when a disk has been changed -- either by a media change or online
1044 * resize.
1045 */
93b270f7 1046static void flush_disk(struct block_device *bdev, bool kill_dirty)
56ade44b 1047{
93b270f7 1048 if (__invalidate_device(bdev, kill_dirty)) {
56ade44b 1049 printk(KERN_WARNING "VFS: busy inodes on changed media or "
424081f3
DM
1050 "resized disk %s\n",
1051 bdev->bd_disk ? bdev->bd_disk->disk_name : "");
56ade44b
AP
1052 }
1053
1054 if (!bdev->bd_disk)
1055 return;
d27769ec 1056 if (disk_part_scan_enabled(bdev->bd_disk))
56ade44b
AP
1057 bdev->bd_invalidated = 1;
1058}
1059
c3279d14 1060/**
57d1b536 1061 * check_disk_size_change - checks for disk size change and adjusts bdev size.
c3279d14
AP
1062 * @disk: struct gendisk to check
1063 * @bdev: struct bdev to adjust.
1064 *
1065 * This routine checks to see if the bdev size does not match the disk size
1066 * and adjusts it if it differs.
1067 */
1068void check_disk_size_change(struct gendisk *disk, struct block_device *bdev)
1069{
1070 loff_t disk_size, bdev_size;
1071
1072 disk_size = (loff_t)get_capacity(disk) << 9;
1073 bdev_size = i_size_read(bdev->bd_inode);
1074 if (disk_size != bdev_size) {
c3279d14
AP
1075 printk(KERN_INFO
1076 "%s: detected capacity change from %lld to %lld\n",
424081f3 1077 disk->disk_name, bdev_size, disk_size);
c3279d14 1078 i_size_write(bdev->bd_inode, disk_size);
93b270f7 1079 flush_disk(bdev, false);
c3279d14
AP
1080 }
1081}
1082EXPORT_SYMBOL(check_disk_size_change);
1083
0c002c2f 1084/**
57d1b536 1085 * revalidate_disk - wrapper for lower-level driver's revalidate_disk call-back
0c002c2f
AP
1086 * @disk: struct gendisk to be revalidated
1087 *
1088 * This routine is a wrapper for lower-level driver's revalidate_disk
1089 * call-backs. It is used to do common pre and post operations needed
1090 * for all revalidate_disk operations.
1091 */
1092int revalidate_disk(struct gendisk *disk)
1093{
c3279d14 1094 struct block_device *bdev;
0c002c2f
AP
1095 int ret = 0;
1096
1097 if (disk->fops->revalidate_disk)
1098 ret = disk->fops->revalidate_disk(disk);
25520d55 1099 blk_integrity_revalidate(disk);
c3279d14
AP
1100 bdev = bdget_disk(disk, 0);
1101 if (!bdev)
1102 return ret;
1103
1104 mutex_lock(&bdev->bd_mutex);
1105 check_disk_size_change(disk, bdev);
7630b661 1106 bdev->bd_invalidated = 0;
c3279d14
AP
1107 mutex_unlock(&bdev->bd_mutex);
1108 bdput(bdev);
0c002c2f
AP
1109 return ret;
1110}
1111EXPORT_SYMBOL(revalidate_disk);
1112
1da177e4
LT
1113/*
1114 * This routine checks whether a removable media has been changed,
1115 * and invalidates all buffer-cache-entries in that case. This
1116 * is a relatively slow routine, so we have to try to minimize using
1117 * it. Thus it is called only upon a 'mount' or 'open'. This
1118 * is the best way of combining speed and utility, I think.
1119 * People changing diskettes in the middle of an operation deserve
1120 * to lose :-)
1121 */
1122int check_disk_change(struct block_device *bdev)
1123{
1124 struct gendisk *disk = bdev->bd_disk;
83d5cde4 1125 const struct block_device_operations *bdops = disk->fops;
77ea887e 1126 unsigned int events;
1da177e4 1127
77ea887e
TH
1128 events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
1129 DISK_EVENT_EJECT_REQUEST);
1130 if (!(events & DISK_EVENT_MEDIA_CHANGE))
1da177e4
LT
1131 return 0;
1132
93b270f7 1133 flush_disk(bdev, true);
1da177e4
LT
1134 if (bdops->revalidate_disk)
1135 bdops->revalidate_disk(bdev->bd_disk);
1da177e4
LT
1136 return 1;
1137}
1138
1139EXPORT_SYMBOL(check_disk_change);
1140
1141void bd_set_size(struct block_device *bdev, loff_t size)
1142{
e1defc4f 1143 unsigned bsize = bdev_logical_block_size(bdev);
1da177e4 1144
5955102c 1145 inode_lock(bdev->bd_inode);
d646a02a 1146 i_size_write(bdev->bd_inode, size);
5955102c 1147 inode_unlock(bdev->bd_inode);
1da177e4
LT
1148 while (bsize < PAGE_CACHE_SIZE) {
1149 if (size & bsize)
1150 break;
1151 bsize <<= 1;
1152 }
1153 bdev->bd_block_size = bsize;
1154 bdev->bd_inode->i_blkbits = blksize_bits(bsize);
1155}
1156EXPORT_SYMBOL(bd_set_size);
1157
4385bab1 1158static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
37be4124 1159
6d740cd5
PZ
1160/*
1161 * bd_mutex locking:
1162 *
1163 * mutex_lock(part->bd_mutex)
1164 * mutex_lock_nested(whole->bd_mutex, 1)
1165 */
1166
572c4892 1167static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)
1da177e4 1168{
1da177e4 1169 struct gendisk *disk;
523e1d39 1170 struct module *owner;
7db9cfd3 1171 int ret;
cf771cb5 1172 int partno;
fe6e9c1f
AV
1173 int perm = 0;
1174
572c4892 1175 if (mode & FMODE_READ)
fe6e9c1f 1176 perm |= MAY_READ;
572c4892 1177 if (mode & FMODE_WRITE)
fe6e9c1f
AV
1178 perm |= MAY_WRITE;
1179 /*
1180 * hooks: /n/, see "layering violations".
1181 */
b7300b78
CW
1182 if (!for_part) {
1183 ret = devcgroup_inode_permission(bdev->bd_inode, perm);
1184 if (ret != 0) {
1185 bdput(bdev);
1186 return ret;
1187 }
82666020 1188 }
7db9cfd3 1189
d3374825 1190 restart:
0762b8bd 1191
89f97496 1192 ret = -ENXIO;
cf771cb5 1193 disk = get_gendisk(bdev->bd_dev, &partno);
0762b8bd 1194 if (!disk)
6e9624b8 1195 goto out;
523e1d39 1196 owner = disk->fops->owner;
1da177e4 1197
69e02c59 1198 disk_block_events(disk);
6796bf54 1199 mutex_lock_nested(&bdev->bd_mutex, for_part);
1da177e4
LT
1200 if (!bdev->bd_openers) {
1201 bdev->bd_disk = disk;
87192a2a 1202 bdev->bd_queue = disk->queue;
1da177e4 1203 bdev->bd_contains = bdev;
03cdadb0
DW
1204 if (IS_ENABLED(CONFIG_BLK_DEV_DAX) && disk->fops->direct_access)
1205 bdev->bd_inode->i_flags = S_DAX;
1206 else
1207 bdev->bd_inode->i_flags = 0;
1208
cf771cb5 1209 if (!partno) {
89f97496
TH
1210 ret = -ENXIO;
1211 bdev->bd_part = disk_get_part(disk, partno);
1212 if (!bdev->bd_part)
1213 goto out_clear;
1214
1196f8b8 1215 ret = 0;
1da177e4 1216 if (disk->fops->open) {
572c4892 1217 ret = disk->fops->open(bdev, mode);
d3374825
N
1218 if (ret == -ERESTARTSYS) {
1219 /* Lost a race with 'disk' being
1220 * deleted, try again.
1221 * See md.c
1222 */
1223 disk_put_part(bdev->bd_part);
1224 bdev->bd_part = NULL;
d3374825 1225 bdev->bd_disk = NULL;
87192a2a 1226 bdev->bd_queue = NULL;
d3374825 1227 mutex_unlock(&bdev->bd_mutex);
69e02c59 1228 disk_unblock_events(disk);
69e02c59 1229 put_disk(disk);
523e1d39 1230 module_put(owner);
d3374825
N
1231 goto restart;
1232 }
1da177e4 1233 }
7e69723f 1234
5a023cdb 1235 if (!ret) {
7e69723f 1236 bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
5a023cdb
DW
1237 if (!blkdev_dax_capable(bdev))
1238 bdev->bd_inode->i_flags &= ~S_DAX;
1239 }
7e69723f 1240
1196f8b8
TH
1241 /*
1242 * If the device is invalidated, rescan partition
1243 * if open succeeded or failed with -ENOMEDIUM.
1244 * The latter is necessary to prevent ghost
1245 * partitions on a removed medium.
1246 */
fe316bf2
JN
1247 if (bdev->bd_invalidated) {
1248 if (!ret)
1249 rescan_partitions(disk, bdev);
1250 else if (ret == -ENOMEDIUM)
1251 invalidate_partitions(disk, bdev);
1252 }
5a023cdb 1253
1196f8b8
TH
1254 if (ret)
1255 goto out_clear;
1da177e4 1256 } else {
1da177e4
LT
1257 struct block_device *whole;
1258 whole = bdget_disk(disk, 0);
1259 ret = -ENOMEM;
1260 if (!whole)
0762b8bd 1261 goto out_clear;
37be4124 1262 BUG_ON(for_part);
572c4892 1263 ret = __blkdev_get(whole, mode, 1);
1da177e4 1264 if (ret)
0762b8bd 1265 goto out_clear;
1da177e4 1266 bdev->bd_contains = whole;
89f97496 1267 bdev->bd_part = disk_get_part(disk, partno);
e71bf0d0 1268 if (!(disk->flags & GENHD_FL_UP) ||
89f97496 1269 !bdev->bd_part || !bdev->bd_part->nr_sects) {
1da177e4 1270 ret = -ENXIO;
0762b8bd 1271 goto out_clear;
1da177e4 1272 }
89f97496 1273 bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
5a023cdb 1274 if (!blkdev_dax_capable(bdev))
f0b2e563 1275 bdev->bd_inode->i_flags &= ~S_DAX;
1da177e4
LT
1276 }
1277 } else {
1da177e4 1278 if (bdev->bd_contains == bdev) {
1196f8b8
TH
1279 ret = 0;
1280 if (bdev->bd_disk->fops->open)
572c4892 1281 ret = bdev->bd_disk->fops->open(bdev, mode);
1196f8b8 1282 /* the same as first opener case, read comment there */
fe316bf2
JN
1283 if (bdev->bd_invalidated) {
1284 if (!ret)
1285 rescan_partitions(bdev->bd_disk, bdev);
1286 else if (ret == -ENOMEDIUM)
1287 invalidate_partitions(bdev->bd_disk, bdev);
1288 }
1196f8b8
TH
1289 if (ret)
1290 goto out_unlock_bdev;
1da177e4 1291 }
69e02c59 1292 /* only one opener holds refs to the module and disk */
69e02c59 1293 put_disk(disk);
523e1d39 1294 module_put(owner);
1da177e4
LT
1295 }
1296 bdev->bd_openers++;
37be4124
N
1297 if (for_part)
1298 bdev->bd_part_count++;
c039e313 1299 mutex_unlock(&bdev->bd_mutex);
69e02c59 1300 disk_unblock_events(disk);
1da177e4
LT
1301 return 0;
1302
0762b8bd 1303 out_clear:
89f97496 1304 disk_put_part(bdev->bd_part);
1da177e4 1305 bdev->bd_disk = NULL;
0762b8bd 1306 bdev->bd_part = NULL;
87192a2a 1307 bdev->bd_queue = NULL;
1da177e4 1308 if (bdev != bdev->bd_contains)
572c4892 1309 __blkdev_put(bdev->bd_contains, mode, 1);
1da177e4 1310 bdev->bd_contains = NULL;
0762b8bd 1311 out_unlock_bdev:
c039e313 1312 mutex_unlock(&bdev->bd_mutex);
69e02c59 1313 disk_unblock_events(disk);
0762b8bd 1314 put_disk(disk);
523e1d39 1315 module_put(owner);
4345caba 1316 out:
0762b8bd
TH
1317 bdput(bdev);
1318
1da177e4
LT
1319 return ret;
1320}
1321
d4d77629
TH
1322/**
1323 * blkdev_get - open a block device
1324 * @bdev: block_device to open
1325 * @mode: FMODE_* mask
1326 * @holder: exclusive holder identifier
1327 *
1328 * Open @bdev with @mode. If @mode includes %FMODE_EXCL, @bdev is
1329 * open with exclusive access. Specifying %FMODE_EXCL with %NULL
1330 * @holder is invalid. Exclusive opens may nest for the same @holder.
1331 *
1332 * On success, the reference count of @bdev is unchanged. On failure,
1333 * @bdev is put.
1334 *
1335 * CONTEXT:
1336 * Might sleep.
1337 *
1338 * RETURNS:
1339 * 0 on success, -errno on failure.
1340 */
e525fd89 1341int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
1da177e4 1342{
e525fd89
TH
1343 struct block_device *whole = NULL;
1344 int res;
1345
1346 WARN_ON_ONCE((mode & FMODE_EXCL) && !holder);
1347
1348 if ((mode & FMODE_EXCL) && holder) {
1349 whole = bd_start_claiming(bdev, holder);
1350 if (IS_ERR(whole)) {
1351 bdput(bdev);
1352 return PTR_ERR(whole);
1353 }
1354 }
1355
1356 res = __blkdev_get(bdev, mode, 0);
1357
1358 if (whole) {
d4dc210f
TH
1359 struct gendisk *disk = whole->bd_disk;
1360
6a027eff 1361 /* finish claiming */
77ea887e 1362 mutex_lock(&bdev->bd_mutex);
6a027eff
TH
1363 spin_lock(&bdev_lock);
1364
77ea887e 1365 if (!res) {
6a027eff
TH
1366 BUG_ON(!bd_may_claim(bdev, whole, holder));
1367 /*
1368 * Note that for a whole device bd_holders
1369 * will be incremented twice, and bd_holder
1370 * will be set to bd_may_claim before being
1371 * set to holder
1372 */
1373 whole->bd_holders++;
1374 whole->bd_holder = bd_may_claim;
1375 bdev->bd_holders++;
1376 bdev->bd_holder = holder;
1377 }
1378
1379 /* tell others that we're done */
1380 BUG_ON(whole->bd_claiming != holder);
1381 whole->bd_claiming = NULL;
1382 wake_up_bit(&whole->bd_claiming, 0);
1383
1384 spin_unlock(&bdev_lock);
77ea887e
TH
1385
1386 /*
d4dc210f
TH
1387 * Block event polling for write claims if requested. Any
1388 * write holder makes the write_holder state stick until
1389 * all are released. This is good enough and tracking
1390 * individual writeable reference is too fragile given the
1391 * way @mode is used in blkdev_get/put().
77ea887e 1392 */
4c49ff3f
TH
1393 if (!res && (mode & FMODE_WRITE) && !bdev->bd_write_holder &&
1394 (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) {
77ea887e 1395 bdev->bd_write_holder = true;
d4dc210f 1396 disk_block_events(disk);
77ea887e
TH
1397 }
1398
1399 mutex_unlock(&bdev->bd_mutex);
6a027eff 1400 bdput(whole);
e525fd89
TH
1401 }
1402
1403 return res;
37be4124 1404}
1da177e4
LT
1405EXPORT_SYMBOL(blkdev_get);
1406
d4d77629
TH
1407/**
1408 * blkdev_get_by_path - open a block device by name
1409 * @path: path to the block device to open
1410 * @mode: FMODE_* mask
1411 * @holder: exclusive holder identifier
1412 *
1413 * Open the blockdevice described by the device file at @path. @mode
1414 * and @holder are identical to blkdev_get().
1415 *
1416 * On success, the returned block_device has reference count of one.
1417 *
1418 * CONTEXT:
1419 * Might sleep.
1420 *
1421 * RETURNS:
1422 * Pointer to block_device on success, ERR_PTR(-errno) on failure.
1423 */
1424struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
1425 void *holder)
1426{
1427 struct block_device *bdev;
1428 int err;
1429
1430 bdev = lookup_bdev(path);
1431 if (IS_ERR(bdev))
1432 return bdev;
1433
1434 err = blkdev_get(bdev, mode, holder);
1435 if (err)
1436 return ERR_PTR(err);
1437
e51900f7
CE
1438 if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) {
1439 blkdev_put(bdev, mode);
1440 return ERR_PTR(-EACCES);
1441 }
1442
d4d77629
TH
1443 return bdev;
1444}
1445EXPORT_SYMBOL(blkdev_get_by_path);
1446
1447/**
1448 * blkdev_get_by_dev - open a block device by device number
1449 * @dev: device number of block device to open
1450 * @mode: FMODE_* mask
1451 * @holder: exclusive holder identifier
1452 *
1453 * Open the blockdevice described by device number @dev. @mode and
1454 * @holder are identical to blkdev_get().
1455 *
1456 * Use it ONLY if you really do not have anything better - i.e. when
1457 * you are behind a truly sucky interface and all you are given is a
1458 * device number. _Never_ to be used for internal purposes. If you
1459 * ever need it - reconsider your API.
1460 *
1461 * On success, the returned block_device has reference count of one.
1462 *
1463 * CONTEXT:
1464 * Might sleep.
1465 *
1466 * RETURNS:
1467 * Pointer to block_device on success, ERR_PTR(-errno) on failure.
1468 */
1469struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
1470{
1471 struct block_device *bdev;
1472 int err;
1473
1474 bdev = bdget(dev);
1475 if (!bdev)
1476 return ERR_PTR(-ENOMEM);
1477
1478 err = blkdev_get(bdev, mode, holder);
1479 if (err)
1480 return ERR_PTR(err);
1481
1482 return bdev;
1483}
1484EXPORT_SYMBOL(blkdev_get_by_dev);
1485
1da177e4
LT
1486static int blkdev_open(struct inode * inode, struct file * filp)
1487{
1488 struct block_device *bdev;
1da177e4
LT
1489
1490 /*
1491 * Preserve backwards compatibility and allow large file access
1492 * even if userspace doesn't ask for it explicitly. Some mkfs
1493 * binary needs it. We might want to drop this workaround
1494 * during an unstable branch.
1495 */
1496 filp->f_flags |= O_LARGEFILE;
1497
572c4892
AV
1498 if (filp->f_flags & O_NDELAY)
1499 filp->f_mode |= FMODE_NDELAY;
1500 if (filp->f_flags & O_EXCL)
1501 filp->f_mode |= FMODE_EXCL;
1502 if ((filp->f_flags & O_ACCMODE) == 3)
1503 filp->f_mode |= FMODE_WRITE_IOCTL;
1504
1da177e4 1505 bdev = bd_acquire(inode);
6a2aae06
PE
1506 if (bdev == NULL)
1507 return -ENOMEM;
1da177e4 1508
572c4892
AV
1509 filp->f_mapping = bdev->bd_inode->i_mapping;
1510
e525fd89 1511 return blkdev_get(bdev, filp->f_mode, filp);
1da177e4
LT
1512}
1513
4385bab1 1514static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part)
2e7b651d 1515{
2e7b651d 1516 struct gendisk *disk = bdev->bd_disk;
37be4124 1517 struct block_device *victim = NULL;
2e7b651d 1518
6796bf54 1519 mutex_lock_nested(&bdev->bd_mutex, for_part);
37be4124
N
1520 if (for_part)
1521 bdev->bd_part_count--;
1522
2e7b651d 1523 if (!--bdev->bd_openers) {
6a027eff 1524 WARN_ON_ONCE(bdev->bd_holders);
2e7b651d
PZ
1525 sync_blockdev(bdev);
1526 kill_bdev(bdev);
43d1c0eb
ID
1527
1528 bdev_write_inode(bdev);
564f00f6 1529 /*
43d1c0eb
ID
1530 * Detaching bdev inode from its wb in __destroy_inode()
1531 * is too late: the queue which embeds its bdi (along with
1532 * root wb) can be gone as soon as we put_disk() below.
94007751 1533 */
43d1c0eb 1534 inode_detach_wb(bdev->bd_inode);
2e7b651d
PZ
1535 }
1536 if (bdev->bd_contains == bdev) {
1537 if (disk->fops->release)
db2a144b 1538 disk->fops->release(disk, mode);
2e7b651d
PZ
1539 }
1540 if (!bdev->bd_openers) {
1541 struct module *owner = disk->fops->owner;
1542
0762b8bd
TH
1543 disk_put_part(bdev->bd_part);
1544 bdev->bd_part = NULL;
2e7b651d 1545 bdev->bd_disk = NULL;
37be4124
N
1546 if (bdev != bdev->bd_contains)
1547 victim = bdev->bd_contains;
2e7b651d 1548 bdev->bd_contains = NULL;
523e1d39
TH
1549
1550 put_disk(disk);
1551 module_put(owner);
2e7b651d 1552 }
2e7b651d
PZ
1553 mutex_unlock(&bdev->bd_mutex);
1554 bdput(bdev);
37be4124 1555 if (victim)
9a1c3542 1556 __blkdev_put(victim, mode, 1);
2e7b651d
PZ
1557}
1558
4385bab1 1559void blkdev_put(struct block_device *bdev, fmode_t mode)
37be4124 1560{
85ef06d1
TH
1561 mutex_lock(&bdev->bd_mutex);
1562
e525fd89 1563 if (mode & FMODE_EXCL) {
6a027eff
TH
1564 bool bdev_free;
1565
1566 /*
1567 * Release a claim on the device. The holder fields
1568 * are protected with bdev_lock. bd_mutex is to
1569 * synchronize disk_holder unlinking.
1570 */
6a027eff
TH
1571 spin_lock(&bdev_lock);
1572
1573 WARN_ON_ONCE(--bdev->bd_holders < 0);
1574 WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0);
1575
1576 /* bd_contains might point to self, check in a separate step */
1577 if ((bdev_free = !bdev->bd_holders))
1578 bdev->bd_holder = NULL;
1579 if (!bdev->bd_contains->bd_holders)
1580 bdev->bd_contains->bd_holder = NULL;
1581
1582 spin_unlock(&bdev_lock);
1583
77ea887e
TH
1584 /*
1585 * If this was the last claim, remove holder link and
1586 * unblock evpoll if it was a write holder.
1587 */
85ef06d1
TH
1588 if (bdev_free && bdev->bd_write_holder) {
1589 disk_unblock_events(bdev->bd_disk);
1590 bdev->bd_write_holder = false;
77ea887e 1591 }
6936217c 1592 }
77ea887e 1593
85ef06d1
TH
1594 /*
1595 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
1596 * event. This is to ensure detection of media removal commanded
1597 * from userland - e.g. eject(1).
1598 */
1599 disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE);
1600
1601 mutex_unlock(&bdev->bd_mutex);
1602
4385bab1 1603 __blkdev_put(bdev, mode, 0);
37be4124 1604}
2e7b651d
PZ
1605EXPORT_SYMBOL(blkdev_put);
1606
1da177e4
LT
1607static int blkdev_close(struct inode * inode, struct file * filp)
1608{
4ebb16ca 1609 struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
4385bab1
AV
1610 blkdev_put(bdev, filp->f_mode);
1611 return 0;
1da177e4
LT
1612}
1613
bb93e3a5 1614static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
1da177e4 1615{
4ebb16ca 1616 struct block_device *bdev = I_BDEV(bdev_file_inode(file));
56b26add 1617 fmode_t mode = file->f_mode;
fd4ce1ac
CH
1618
1619 /*
1620 * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have
1621 * to updated it before every ioctl.
1622 */
56b26add 1623 if (file->f_flags & O_NDELAY)
fd4ce1ac
CH
1624 mode |= FMODE_NDELAY;
1625 else
1626 mode &= ~FMODE_NDELAY;
1627
56b26add 1628 return blkdev_ioctl(bdev, mode, cmd, arg);
1da177e4
LT
1629}
1630
eef99380
CH
1631/*
1632 * Write data to the block device. Only intended for the block device itself
1633 * and the raw driver which basically is a fake block device.
1634 *
1635 * Does not take i_mutex for the write and thus is not for general purpose
1636 * use.
1637 */
1456c0a8 1638ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
eef99380
CH
1639{
1640 struct file *file = iocb->ki_filp;
4ebb16ca 1641 struct inode *bd_inode = bdev_file_inode(file);
7ec7b94a 1642 loff_t size = i_size_read(bd_inode);
53362a05 1643 struct blk_plug plug;
eef99380 1644 ssize_t ret;
5f380c7f 1645
7ec7b94a
AV
1646 if (bdev_read_only(I_BDEV(bd_inode)))
1647 return -EPERM;
5f380c7f 1648
7ec7b94a 1649 if (!iov_iter_count(from))
5f380c7f
AV
1650 return 0;
1651
7ec7b94a
AV
1652 if (iocb->ki_pos >= size)
1653 return -ENOSPC;
1654
1655 iov_iter_truncate(from, size - iocb->ki_pos);
eef99380 1656
53362a05 1657 blk_start_plug(&plug);
1456c0a8 1658 ret = __generic_file_write_iter(iocb, from);
02afc27f 1659 if (ret > 0) {
eef99380 1660 ssize_t err;
1456c0a8 1661 err = generic_write_sync(file, iocb->ki_pos - ret, ret);
45d4f855 1662 if (err < 0)
eef99380
CH
1663 ret = err;
1664 }
53362a05 1665 blk_finish_plug(&plug);
eef99380
CH
1666 return ret;
1667}
1456c0a8 1668EXPORT_SYMBOL_GPL(blkdev_write_iter);
eef99380 1669
b2de525f 1670ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
684c9aae
LT
1671{
1672 struct file *file = iocb->ki_filp;
4ebb16ca 1673 struct inode *bd_inode = bdev_file_inode(file);
684c9aae 1674 loff_t size = i_size_read(bd_inode);
a886038b 1675 loff_t pos = iocb->ki_pos;
684c9aae
LT
1676
1677 if (pos >= size)
1678 return 0;
1679
1680 size -= pos;
a886038b
AV
1681 iov_iter_truncate(to, size);
1682 return generic_file_read_iter(iocb, to);
684c9aae 1683}
b2de525f 1684EXPORT_SYMBOL_GPL(blkdev_read_iter);
684c9aae 1685
87d8fe1e
TT
1686/*
1687 * Try to release a page associated with block device when the system
1688 * is under memory pressure.
1689 */
1690static int blkdev_releasepage(struct page *page, gfp_t wait)
1691{
1692 struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super;
1693
1694 if (super && super->s_op->bdev_try_to_free_page)
1695 return super->s_op->bdev_try_to_free_page(super, page, wait);
1696
1697 return try_to_free_buffers(page);
1698}
1699
7f6d5b52
RZ
1700static int blkdev_writepages(struct address_space *mapping,
1701 struct writeback_control *wbc)
1702{
1703 if (dax_mapping(mapping)) {
1704 struct block_device *bdev = I_BDEV(mapping->host);
1705
1706 return dax_writeback_mapping_range(mapping, bdev, wbc);
1707 }
1708 return generic_writepages(mapping, wbc);
1709}
1710
4c54ac62 1711static const struct address_space_operations def_blk_aops = {
1da177e4 1712 .readpage = blkdev_readpage,
447f05bb 1713 .readpages = blkdev_readpages,
1da177e4 1714 .writepage = blkdev_writepage,
6272b5a5
NP
1715 .write_begin = blkdev_write_begin,
1716 .write_end = blkdev_write_end,
7f6d5b52 1717 .writepages = blkdev_writepages,
87d8fe1e 1718 .releasepage = blkdev_releasepage,
1da177e4 1719 .direct_IO = blkdev_direct_IO,
b4597226 1720 .is_dirty_writeback = buffer_check_dirty_writeback,
1da177e4
LT
1721};
1722
5a023cdb
DW
1723#ifdef CONFIG_FS_DAX
1724/*
1725 * In the raw block case we do not need to contend with truncation nor
1726 * unwritten file extents. Without those concerns there is no need for
1727 * additional locking beyond the mmap_sem context that these routines
1728 * are already executing under.
1729 *
1730 * Note, there is no protection if the block device is dynamically
1731 * resized (partition grow/shrink) during a fault. A stable block device
1732 * size is already not enforced in the blkdev_direct_IO path.
1733 *
1734 * For DAX, it is the responsibility of the block device driver to
1735 * ensure the whole-disk device size is stable while requests are in
1736 * flight.
1737 *
1738 * Finally, unlike the filemap_page_mkwrite() case there is no
1739 * filesystem superblock to sync against freezing. We still include a
1740 * pfn_mkwrite callback for dax drivers to receive write fault
1741 * notifications.
1742 */
1743static int blkdev_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1744{
1745 return __dax_fault(vma, vmf, blkdev_get_block, NULL);
1746}
1747
9c5a05bc
RZ
1748static int blkdev_dax_pfn_mkwrite(struct vm_area_struct *vma,
1749 struct vm_fault *vmf)
1750{
1751 return dax_pfn_mkwrite(vma, vmf);
1752}
1753
5a023cdb
DW
1754static int blkdev_dax_pmd_fault(struct vm_area_struct *vma, unsigned long addr,
1755 pmd_t *pmd, unsigned int flags)
1756{
1757 return __dax_pmd_fault(vma, addr, pmd, flags, blkdev_get_block, NULL);
1758}
1759
5a023cdb 1760static const struct vm_operations_struct blkdev_dax_vm_ops = {
5a023cdb
DW
1761 .fault = blkdev_dax_fault,
1762 .pmd_fault = blkdev_dax_pmd_fault,
9c5a05bc 1763 .pfn_mkwrite = blkdev_dax_pfn_mkwrite,
5a023cdb
DW
1764};
1765
1766static const struct vm_operations_struct blkdev_default_vm_ops = {
5a023cdb
DW
1767 .fault = filemap_fault,
1768 .map_pages = filemap_map_pages,
1769};
1770
1771static int blkdev_mmap(struct file *file, struct vm_area_struct *vma)
1772{
1773 struct inode *bd_inode = bdev_file_inode(file);
5a023cdb
DW
1774
1775 file_accessed(file);
5a023cdb
DW
1776 if (IS_DAX(bd_inode)) {
1777 vma->vm_ops = &blkdev_dax_vm_ops;
1778 vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
1779 } else {
1780 vma->vm_ops = &blkdev_default_vm_ops;
1781 }
5a023cdb
DW
1782
1783 return 0;
1784}
1785#else
1786#define blkdev_mmap generic_file_mmap
1787#endif
1788
4b6f5d20 1789const struct file_operations def_blk_fops = {
1da177e4
LT
1790 .open = blkdev_open,
1791 .release = blkdev_close,
1792 .llseek = block_llseek,
a886038b 1793 .read_iter = blkdev_read_iter,
1456c0a8 1794 .write_iter = blkdev_write_iter,
5a023cdb 1795 .mmap = blkdev_mmap,
b1dd3b28 1796 .fsync = blkdev_fsync,
bb93e3a5 1797 .unlocked_ioctl = block_ioctl,
1da177e4
LT
1798#ifdef CONFIG_COMPAT
1799 .compat_ioctl = compat_blkdev_ioctl,
1800#endif
1e8b3332 1801 .splice_read = generic_file_splice_read,
8d020765 1802 .splice_write = iter_file_splice_write,
1da177e4
LT
1803};
1804
1805int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg)
1806{
1807 int res;
1808 mm_segment_t old_fs = get_fs();
1809 set_fs(KERNEL_DS);
56b26add 1810 res = blkdev_ioctl(bdev, 0, cmd, arg);
1da177e4
LT
1811 set_fs(old_fs);
1812 return res;
1813}
1814
1815EXPORT_SYMBOL(ioctl_by_bdev);
1816
1817/**
1818 * lookup_bdev - lookup a struct block_device by name
94e2959e 1819 * @pathname: special file representing the block device
1da177e4 1820 *
57d1b536 1821 * Get a reference to the blockdevice at @pathname in the current
1da177e4
LT
1822 * namespace if possible and return it. Return ERR_PTR(error)
1823 * otherwise.
1824 */
421748ec 1825struct block_device *lookup_bdev(const char *pathname)
1da177e4
LT
1826{
1827 struct block_device *bdev;
1828 struct inode *inode;
421748ec 1829 struct path path;
1da177e4
LT
1830 int error;
1831
421748ec 1832 if (!pathname || !*pathname)
1da177e4
LT
1833 return ERR_PTR(-EINVAL);
1834
421748ec 1835 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
1da177e4
LT
1836 if (error)
1837 return ERR_PTR(error);
1838
bb668734 1839 inode = d_backing_inode(path.dentry);
1da177e4
LT
1840 error = -ENOTBLK;
1841 if (!S_ISBLK(inode->i_mode))
1842 goto fail;
1843 error = -EACCES;
421748ec 1844 if (path.mnt->mnt_flags & MNT_NODEV)
1da177e4
LT
1845 goto fail;
1846 error = -ENOMEM;
1847 bdev = bd_acquire(inode);
1848 if (!bdev)
1849 goto fail;
1850out:
421748ec 1851 path_put(&path);
1da177e4
LT
1852 return bdev;
1853fail:
1854 bdev = ERR_PTR(error);
1855 goto out;
1856}
d5686b44 1857EXPORT_SYMBOL(lookup_bdev);
1da177e4 1858
93b270f7 1859int __invalidate_device(struct block_device *bdev, bool kill_dirty)
b71e8a4c
DH
1860{
1861 struct super_block *sb = get_super(bdev);
1862 int res = 0;
1863
1864 if (sb) {
1865 /*
1866 * no need to lock the super, get_super holds the
1867 * read mutex so the filesystem cannot go away
1868 * under us (->put_super runs with the write lock
1869 * hold).
1870 */
1871 shrink_dcache_sb(sb);
93b270f7 1872 res = invalidate_inodes(sb, kill_dirty);
b71e8a4c
DH
1873 drop_super(sb);
1874 }
f98393a6 1875 invalidate_bdev(bdev);
b71e8a4c
DH
1876 return res;
1877}
1878EXPORT_SYMBOL(__invalidate_device);
5c0d6b60
JK
1879
1880void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
1881{
1882 struct inode *inode, *old_inode = NULL;
1883
74278da9 1884 spin_lock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60
JK
1885 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
1886 struct address_space *mapping = inode->i_mapping;
1887
1888 spin_lock(&inode->i_lock);
1889 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
1890 mapping->nrpages == 0) {
1891 spin_unlock(&inode->i_lock);
1892 continue;
1893 }
1894 __iget(inode);
1895 spin_unlock(&inode->i_lock);
74278da9 1896 spin_unlock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60
JK
1897 /*
1898 * We hold a reference to 'inode' so it couldn't have been
1899 * removed from s_inodes list while we dropped the
74278da9 1900 * s_inode_list_lock We cannot iput the inode now as we can
5c0d6b60 1901 * be holding the last reference and we cannot iput it under
74278da9 1902 * s_inode_list_lock. So we keep the reference and iput it
5c0d6b60
JK
1903 * later.
1904 */
1905 iput(old_inode);
1906 old_inode = inode;
1907
1908 func(I_BDEV(inode), arg);
1909
74278da9 1910 spin_lock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60 1911 }
74278da9 1912 spin_unlock(&blockdev_superblock->s_inode_list_lock);
5c0d6b60
JK
1913 iput(old_inode);
1914}