4 * Copyright (C) 1991, 1992, 2002 Linus Torvalds
8 * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
10 * Removed a lot of unnecessary code and simplified things now that
11 * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
13 * Speed up hash, lru, and free list operations. Use gfp() for allocating
14 * hash table, use SLAB cache for buffer heads. SMP threading. -DaveM
16 * Added 32k buffer block sizes - these are required older ARM systems. - RMK
18 * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
21 #include <linux/config.h>
22 #include <linux/kernel.h>
23 #include <linux/syscalls.h>
26 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/smp_lock.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/module.h>
34 #include <linux/writeback.h>
35 #include <linux/hash.h>
36 #include <linux/suspend.h>
37 #include <linux/buffer_head.h>
38 #include <linux/bio.h>
39 #include <linux/notifier.h>
40 #include <linux/cpu.h>
41 #include <linux/bitops.h>
42 #include <linux/mpage.h>
44 static int fsync_buffers_list(spinlock_t
*lock
, struct list_head
*list
);
45 static void invalidate_bh_lrus(void);
47 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
50 init_buffer(struct buffer_head
*bh
, bh_end_io_t
*handler
, void *private)
52 bh
->b_end_io
= handler
;
53 bh
->b_private
= private;
56 static int sync_buffer(void *word
)
58 struct block_device
*bd
;
59 struct buffer_head
*bh
60 = container_of(word
, struct buffer_head
, b_state
);
65 blk_run_address_space(bd
->bd_inode
->i_mapping
);
70 void fastcall
__lock_buffer(struct buffer_head
*bh
)
72 wait_on_bit_lock(&bh
->b_state
, BH_Lock
, sync_buffer
,
73 TASK_UNINTERRUPTIBLE
);
75 EXPORT_SYMBOL(__lock_buffer
);
77 void fastcall
unlock_buffer(struct buffer_head
*bh
)
79 clear_buffer_locked(bh
);
80 smp_mb__after_clear_bit();
81 wake_up_bit(&bh
->b_state
, BH_Lock
);
85 * Block until a buffer comes unlocked. This doesn't stop it
86 * from becoming locked again - you have to lock it yourself
87 * if you want to preserve its state.
89 void __wait_on_buffer(struct buffer_head
* bh
)
91 wait_on_bit(&bh
->b_state
, BH_Lock
, sync_buffer
, TASK_UNINTERRUPTIBLE
);
95 __clear_page_buffers(struct page
*page
)
97 ClearPagePrivate(page
);
99 page_cache_release(page
);
102 static void buffer_io_error(struct buffer_head
*bh
)
104 char b
[BDEVNAME_SIZE
];
106 printk(KERN_ERR
"Buffer I/O error on device %s, logical block %Lu\n",
107 bdevname(bh
->b_bdev
, b
),
108 (unsigned long long)bh
->b_blocknr
);
112 * Default synchronous end-of-IO handler.. Just mark it up-to-date and
113 * unlock the buffer. This is what ll_rw_block uses too.
115 void end_buffer_read_sync(struct buffer_head
*bh
, int uptodate
)
118 set_buffer_uptodate(bh
);
120 /* This happens, due to failed READA attempts. */
121 clear_buffer_uptodate(bh
);
127 void end_buffer_write_sync(struct buffer_head
*bh
, int uptodate
)
129 char b
[BDEVNAME_SIZE
];
132 set_buffer_uptodate(bh
);
134 if (!buffer_eopnotsupp(bh
) && printk_ratelimit()) {
136 printk(KERN_WARNING
"lost page write due to "
138 bdevname(bh
->b_bdev
, b
));
140 set_buffer_write_io_error(bh
);
141 clear_buffer_uptodate(bh
);
148 * Write out and wait upon all the dirty data associated with a block
149 * device via its mapping. Does not take the superblock lock.
151 int sync_blockdev(struct block_device
*bdev
)
158 ret
= filemap_fdatawrite(bdev
->bd_inode
->i_mapping
);
159 err
= filemap_fdatawait(bdev
->bd_inode
->i_mapping
);
165 EXPORT_SYMBOL(sync_blockdev
);
168 * Write out and wait upon all dirty data associated with this
169 * superblock. Filesystem data as well as the underlying block
170 * device. Takes the superblock lock.
172 int fsync_super(struct super_block
*sb
)
174 sync_inodes_sb(sb
, 0);
177 if (sb
->s_dirt
&& sb
->s_op
->write_super
)
178 sb
->s_op
->write_super(sb
);
180 if (sb
->s_op
->sync_fs
)
181 sb
->s_op
->sync_fs(sb
, 1);
182 sync_blockdev(sb
->s_bdev
);
183 sync_inodes_sb(sb
, 1);
185 return sync_blockdev(sb
->s_bdev
);
189 * Write out and wait upon all dirty data associated with this
190 * device. Filesystem data as well as the underlying block
191 * device. Takes the superblock lock.
193 int fsync_bdev(struct block_device
*bdev
)
195 struct super_block
*sb
= get_super(bdev
);
197 int res
= fsync_super(sb
);
201 return sync_blockdev(bdev
);
205 * freeze_bdev -- lock a filesystem and force it into a consistent state
206 * @bdev: blockdevice to lock
208 * This takes the block device bd_mount_sem to make sure no new mounts
209 * happen on bdev until thaw_bdev() is called.
210 * If a superblock is found on this device, we take the s_umount semaphore
211 * on it to make sure nobody unmounts until the snapshot creation is done.
213 struct super_block
*freeze_bdev(struct block_device
*bdev
)
215 struct super_block
*sb
;
217 down(&bdev
->bd_mount_sem
);
218 sb
= get_super(bdev
);
219 if (sb
&& !(sb
->s_flags
& MS_RDONLY
)) {
220 sb
->s_frozen
= SB_FREEZE_WRITE
;
223 sync_inodes_sb(sb
, 0);
227 if (sb
->s_dirt
&& sb
->s_op
->write_super
)
228 sb
->s_op
->write_super(sb
);
231 if (sb
->s_op
->sync_fs
)
232 sb
->s_op
->sync_fs(sb
, 1);
234 sync_blockdev(sb
->s_bdev
);
235 sync_inodes_sb(sb
, 1);
237 sb
->s_frozen
= SB_FREEZE_TRANS
;
240 sync_blockdev(sb
->s_bdev
);
242 if (sb
->s_op
->write_super_lockfs
)
243 sb
->s_op
->write_super_lockfs(sb
);
247 return sb
; /* thaw_bdev releases s->s_umount and bd_mount_sem */
249 EXPORT_SYMBOL(freeze_bdev
);
252 * thaw_bdev -- unlock filesystem
253 * @bdev: blockdevice to unlock
254 * @sb: associated superblock
256 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
258 void thaw_bdev(struct block_device
*bdev
, struct super_block
*sb
)
261 BUG_ON(sb
->s_bdev
!= bdev
);
263 if (sb
->s_op
->unlockfs
)
264 sb
->s_op
->unlockfs(sb
);
265 sb
->s_frozen
= SB_UNFROZEN
;
267 wake_up(&sb
->s_wait_unfrozen
);
271 up(&bdev
->bd_mount_sem
);
273 EXPORT_SYMBOL(thaw_bdev
);
276 * sync everything. Start out by waking pdflush, because that writes back
277 * all queues in parallel.
279 static void do_sync(unsigned long wait
)
282 sync_inodes(0); /* All mappings, inodes and their blockdevs */
284 sync_supers(); /* Write the superblocks */
285 sync_filesystems(0); /* Start syncing the filesystems */
286 sync_filesystems(wait
); /* Waitingly sync the filesystems */
287 sync_inodes(wait
); /* Mappings, inodes and blockdevs, again. */
289 printk("Emergency Sync complete\n");
290 if (unlikely(laptop_mode
))
291 laptop_sync_completion();
294 asmlinkage
long sys_sync(void)
300 void emergency_sync(void)
302 pdflush_operation(do_sync
, 0);
306 * Generic function to fsync a file.
308 * filp may be NULL if called via the msync of a vma.
311 int file_fsync(struct file
*filp
, struct dentry
*dentry
, int datasync
)
313 struct inode
* inode
= dentry
->d_inode
;
314 struct super_block
* sb
;
317 /* sync the inode to buffers */
318 ret
= write_inode_now(inode
, 0);
320 /* sync the superblock to buffers */
323 if (sb
->s_op
->write_super
)
324 sb
->s_op
->write_super(sb
);
327 /* .. finally sync the buffers to disk */
328 err
= sync_blockdev(sb
->s_bdev
);
334 asmlinkage
long sys_fsync(unsigned int fd
)
337 struct address_space
*mapping
;
345 mapping
= file
->f_mapping
;
348 if (!file
->f_op
|| !file
->f_op
->fsync
) {
349 /* Why? We can still call filemap_fdatawrite */
353 current
->flags
|= PF_SYNCWRITE
;
354 ret
= filemap_fdatawrite(mapping
);
357 * We need to protect against concurrent writers,
358 * which could cause livelocks in fsync_buffers_list
360 down(&mapping
->host
->i_sem
);
361 err
= file
->f_op
->fsync(file
, file
->f_dentry
, 0);
364 up(&mapping
->host
->i_sem
);
365 err
= filemap_fdatawait(mapping
);
368 current
->flags
&= ~PF_SYNCWRITE
;
376 asmlinkage
long sys_fdatasync(unsigned int fd
)
379 struct address_space
*mapping
;
388 if (!file
->f_op
|| !file
->f_op
->fsync
)
391 mapping
= file
->f_mapping
;
393 current
->flags
|= PF_SYNCWRITE
;
394 ret
= filemap_fdatawrite(mapping
);
395 down(&mapping
->host
->i_sem
);
396 err
= file
->f_op
->fsync(file
, file
->f_dentry
, 1);
399 up(&mapping
->host
->i_sem
);
400 err
= filemap_fdatawait(mapping
);
403 current
->flags
&= ~PF_SYNCWRITE
;
412 * Various filesystems appear to want __find_get_block to be non-blocking.
413 * But it's the page lock which protects the buffers. To get around this,
414 * we get exclusion from try_to_free_buffers with the blockdev mapping's
417 * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
418 * may be quite high. This code could TryLock the page, and if that
419 * succeeds, there is no need to take private_lock. (But if
420 * private_lock is contended then so is mapping->tree_lock).
422 static struct buffer_head
*
423 __find_get_block_slow(struct block_device
*bdev
, sector_t block
, int unused
)
425 struct inode
*bd_inode
= bdev
->bd_inode
;
426 struct address_space
*bd_mapping
= bd_inode
->i_mapping
;
427 struct buffer_head
*ret
= NULL
;
429 struct buffer_head
*bh
;
430 struct buffer_head
*head
;
434 index
= block
>> (PAGE_CACHE_SHIFT
- bd_inode
->i_blkbits
);
435 page
= find_get_page(bd_mapping
, index
);
439 spin_lock(&bd_mapping
->private_lock
);
440 if (!page_has_buffers(page
))
442 head
= page_buffers(page
);
445 if (bh
->b_blocknr
== block
) {
450 if (!buffer_mapped(bh
))
452 bh
= bh
->b_this_page
;
453 } while (bh
!= head
);
455 /* we might be here because some of the buffers on this page are
456 * not mapped. This is due to various races between
457 * file io on the block device and getblk. It gets dealt with
458 * elsewhere, don't buffer_error if we had some unmapped buffers
461 printk("__find_get_block_slow() failed. "
462 "block=%llu, b_blocknr=%llu\n",
463 (unsigned long long)block
, (unsigned long long)bh
->b_blocknr
);
464 printk("b_state=0x%08lx, b_size=%u\n", bh
->b_state
, bh
->b_size
);
465 printk("device blocksize: %d\n", 1 << bd_inode
->i_blkbits
);
468 spin_unlock(&bd_mapping
->private_lock
);
469 page_cache_release(page
);
474 /* If invalidate_buffers() will trash dirty buffers, it means some kind
475 of fs corruption is going on. Trashing dirty data always imply losing
476 information that was supposed to be just stored on the physical layer
479 Thus invalidate_buffers in general usage is not allwowed to trash
480 dirty buffers. For example ioctl(FLSBLKBUF) expects dirty data to
481 be preserved. These buffers are simply skipped.
483 We also skip buffers which are still in use. For example this can
484 happen if a userspace program is reading the block device.
486 NOTE: In the case where the user removed a removable-media-disk even if
487 there's still dirty data not synced on disk (due a bug in the device driver
488 or due an error of the user), by not destroying the dirty buffers we could
489 generate corruption also on the next media inserted, thus a parameter is
490 necessary to handle this case in the most safe way possible (trying
491 to not corrupt also the new disk inserted with the data belonging to
492 the old now corrupted disk). Also for the ramdisk the natural thing
493 to do in order to release the ramdisk memory is to destroy dirty buffers.
495 These are two special cases. Normal usage imply the device driver
496 to issue a sync on the device (without waiting I/O completion) and
497 then an invalidate_buffers call that doesn't trash dirty buffers.
499 For handling cache coherency with the blkdev pagecache the 'update' case
500 is been introduced. It is needed to re-read from disk any pinned
501 buffer. NOTE: re-reading from disk is destructive so we can do it only
502 when we assume nobody is changing the buffercache under our I/O and when
503 we think the disk contains more recent information than the buffercache.
504 The update == 1 pass marks the buffers we need to update, the update == 2
505 pass does the actual I/O. */
506 void invalidate_bdev(struct block_device
*bdev
, int destroy_dirty_buffers
)
508 invalidate_bh_lrus();
510 * FIXME: what about destroy_dirty_buffers?
511 * We really want to use invalidate_inode_pages2() for
512 * that, but not until that's cleaned up.
514 invalidate_inode_pages(bdev
->bd_inode
->i_mapping
);
518 * Kick pdflush then try to free up some ZONE_NORMAL memory.
520 static void free_more_memory(void)
525 wakeup_bdflush(1024);
528 for_each_pgdat(pgdat
) {
529 zones
= pgdat
->node_zonelists
[GFP_NOFS
&GFP_ZONEMASK
].zones
;
531 try_to_free_pages(zones
, GFP_NOFS
, 0);
536 * I/O completion handler for block_read_full_page() - pages
537 * which come unlocked at the end of I/O.
539 static void end_buffer_async_read(struct buffer_head
*bh
, int uptodate
)
541 static DEFINE_SPINLOCK(page_uptodate_lock
);
543 struct buffer_head
*tmp
;
545 int page_uptodate
= 1;
547 BUG_ON(!buffer_async_read(bh
));
551 set_buffer_uptodate(bh
);
553 clear_buffer_uptodate(bh
);
554 if (printk_ratelimit())
560 * Be _very_ careful from here on. Bad things can happen if
561 * two buffer heads end IO at almost the same time and both
562 * decide that the page is now completely done.
564 spin_lock_irqsave(&page_uptodate_lock
, flags
);
565 clear_buffer_async_read(bh
);
569 if (!buffer_uptodate(tmp
))
571 if (buffer_async_read(tmp
)) {
572 BUG_ON(!buffer_locked(tmp
));
575 tmp
= tmp
->b_this_page
;
577 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
580 * If none of the buffers had errors and they are all
581 * uptodate then we can set the page uptodate.
583 if (page_uptodate
&& !PageError(page
))
584 SetPageUptodate(page
);
589 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
594 * Completion handler for block_write_full_page() - pages which are unlocked
595 * during I/O, and which have PageWriteback cleared upon I/O completion.
597 void end_buffer_async_write(struct buffer_head
*bh
, int uptodate
)
599 char b
[BDEVNAME_SIZE
];
600 static DEFINE_SPINLOCK(page_uptodate_lock
);
602 struct buffer_head
*tmp
;
605 BUG_ON(!buffer_async_write(bh
));
609 set_buffer_uptodate(bh
);
611 if (printk_ratelimit()) {
613 printk(KERN_WARNING
"lost page write due to "
615 bdevname(bh
->b_bdev
, b
));
617 set_bit(AS_EIO
, &page
->mapping
->flags
);
618 clear_buffer_uptodate(bh
);
622 spin_lock_irqsave(&page_uptodate_lock
, flags
);
623 clear_buffer_async_write(bh
);
625 tmp
= bh
->b_this_page
;
627 if (buffer_async_write(tmp
)) {
628 BUG_ON(!buffer_locked(tmp
));
631 tmp
= tmp
->b_this_page
;
633 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
634 end_page_writeback(page
);
638 spin_unlock_irqrestore(&page_uptodate_lock
, flags
);
643 * If a page's buffers are under async readin (end_buffer_async_read
644 * completion) then there is a possibility that another thread of
645 * control could lock one of the buffers after it has completed
646 * but while some of the other buffers have not completed. This
647 * locked buffer would confuse end_buffer_async_read() into not unlocking
648 * the page. So the absence of BH_Async_Read tells end_buffer_async_read()
649 * that this buffer is not under async I/O.
651 * The page comes unlocked when it has no locked buffer_async buffers
654 * PageLocked prevents anyone starting new async I/O reads any of
657 * PageWriteback is used to prevent simultaneous writeout of the same
660 * PageLocked prevents anyone from starting writeback of a page which is
661 * under read I/O (PageWriteback is only ever set against a locked page).
663 static void mark_buffer_async_read(struct buffer_head
*bh
)
665 bh
->b_end_io
= end_buffer_async_read
;
666 set_buffer_async_read(bh
);
669 void mark_buffer_async_write(struct buffer_head
*bh
)
671 bh
->b_end_io
= end_buffer_async_write
;
672 set_buffer_async_write(bh
);
674 EXPORT_SYMBOL(mark_buffer_async_write
);
678 * fs/buffer.c contains helper functions for buffer-backed address space's
679 * fsync functions. A common requirement for buffer-based filesystems is
680 * that certain data from the backing blockdev needs to be written out for
681 * a successful fsync(). For example, ext2 indirect blocks need to be
682 * written back and waited upon before fsync() returns.
684 * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
685 * inode_has_buffers() and invalidate_inode_buffers() are provided for the
686 * management of a list of dependent buffers at ->i_mapping->private_list.
688 * Locking is a little subtle: try_to_free_buffers() will remove buffers
689 * from their controlling inode's queue when they are being freed. But
690 * try_to_free_buffers() will be operating against the *blockdev* mapping
691 * at the time, not against the S_ISREG file which depends on those buffers.
692 * So the locking for private_list is via the private_lock in the address_space
693 * which backs the buffers. Which is different from the address_space
694 * against which the buffers are listed. So for a particular address_space,
695 * mapping->private_lock does *not* protect mapping->private_list! In fact,
696 * mapping->private_list will always be protected by the backing blockdev's
699 * Which introduces a requirement: all buffers on an address_space's
700 * ->private_list must be from the same address_space: the blockdev's.
702 * address_spaces which do not place buffers at ->private_list via these
703 * utility functions are free to use private_lock and private_list for
704 * whatever they want. The only requirement is that list_empty(private_list)
705 * be true at clear_inode() time.
707 * FIXME: clear_inode should not call invalidate_inode_buffers(). The
708 * filesystems should do that. invalidate_inode_buffers() should just go
709 * BUG_ON(!list_empty).
711 * FIXME: mark_buffer_dirty_inode() is a data-plane operation. It should
712 * take an address_space, not an inode. And it should be called
713 * mark_buffer_dirty_fsync() to clearly define why those buffers are being
716 * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
717 * list if it is already on a list. Because if the buffer is on a list,
718 * it *must* already be on the right one. If not, the filesystem is being
719 * silly. This will save a ton of locking. But first we have to ensure
720 * that buffers are taken *off* the old inode's list when they are freed
721 * (presumably in truncate). That requires careful auditing of all
722 * filesystems (do it inside bforget()). It could also be done by bringing
727 * The buffer's backing address_space's private_lock must be held
729 static inline void __remove_assoc_queue(struct buffer_head
*bh
)
731 list_del_init(&bh
->b_assoc_buffers
);
734 int inode_has_buffers(struct inode
*inode
)
736 return !list_empty(&inode
->i_data
.private_list
);
740 * osync is designed to support O_SYNC io. It waits synchronously for
741 * all already-submitted IO to complete, but does not queue any new
742 * writes to the disk.
744 * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
745 * you dirty the buffers, and then use osync_inode_buffers to wait for
746 * completion. Any other dirty buffers which are not yet queued for
747 * write will not be flushed to disk by the osync.
749 static int osync_buffers_list(spinlock_t
*lock
, struct list_head
*list
)
751 struct buffer_head
*bh
;
757 list_for_each_prev(p
, list
) {
759 if (buffer_locked(bh
)) {
763 if (!buffer_uptodate(bh
))
775 * sync_mapping_buffers - write out and wait upon a mapping's "associated"
777 * @mapping: the mapping which wants those buffers written
779 * Starts I/O against the buffers at mapping->private_list, and waits upon
782 * Basically, this is a convenience function for fsync().
783 * @mapping is a file or directory which needs those buffers to be written for
784 * a successful fsync().
786 int sync_mapping_buffers(struct address_space
*mapping
)
788 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
790 if (buffer_mapping
== NULL
|| list_empty(&mapping
->private_list
))
793 return fsync_buffers_list(&buffer_mapping
->private_lock
,
794 &mapping
->private_list
);
796 EXPORT_SYMBOL(sync_mapping_buffers
);
799 * Called when we've recently written block `bblock', and it is known that
800 * `bblock' was for a buffer_boundary() buffer. This means that the block at
801 * `bblock + 1' is probably a dirty indirect block. Hunt it down and, if it's
802 * dirty, schedule it for IO. So that indirects merge nicely with their data.
804 void write_boundary_block(struct block_device
*bdev
,
805 sector_t bblock
, unsigned blocksize
)
807 struct buffer_head
*bh
= __find_get_block(bdev
, bblock
+ 1, blocksize
);
809 if (buffer_dirty(bh
))
810 ll_rw_block(WRITE
, 1, &bh
);
815 void mark_buffer_dirty_inode(struct buffer_head
*bh
, struct inode
*inode
)
817 struct address_space
*mapping
= inode
->i_mapping
;
818 struct address_space
*buffer_mapping
= bh
->b_page
->mapping
;
820 mark_buffer_dirty(bh
);
821 if (!mapping
->assoc_mapping
) {
822 mapping
->assoc_mapping
= buffer_mapping
;
824 if (mapping
->assoc_mapping
!= buffer_mapping
)
827 if (list_empty(&bh
->b_assoc_buffers
)) {
828 spin_lock(&buffer_mapping
->private_lock
);
829 list_move_tail(&bh
->b_assoc_buffers
,
830 &mapping
->private_list
);
831 spin_unlock(&buffer_mapping
->private_lock
);
834 EXPORT_SYMBOL(mark_buffer_dirty_inode
);
837 * Add a page to the dirty page list.
839 * It is a sad fact of life that this function is called from several places
840 * deeply under spinlocking. It may not sleep.
842 * If the page has buffers, the uptodate buffers are set dirty, to preserve
843 * dirty-state coherency between the page and the buffers. It the page does
844 * not have buffers then when they are later attached they will all be set
847 * The buffers are dirtied before the page is dirtied. There's a small race
848 * window in which a writepage caller may see the page cleanness but not the
849 * buffer dirtiness. That's fine. If this code were to set the page dirty
850 * before the buffers, a concurrent writepage caller could clear the page dirty
851 * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
852 * page on the dirty page list.
854 * We use private_lock to lock against try_to_free_buffers while using the
855 * page's buffer list. Also use this to protect against clean buffers being
856 * added to the page after it was set dirty.
858 * FIXME: may need to call ->reservepage here as well. That's rather up to the
859 * address_space though.
861 int __set_page_dirty_buffers(struct page
*page
)
863 struct address_space
* const mapping
= page
->mapping
;
865 spin_lock(&mapping
->private_lock
);
866 if (page_has_buffers(page
)) {
867 struct buffer_head
*head
= page_buffers(page
);
868 struct buffer_head
*bh
= head
;
871 set_buffer_dirty(bh
);
872 bh
= bh
->b_this_page
;
873 } while (bh
!= head
);
875 spin_unlock(&mapping
->private_lock
);
877 if (!TestSetPageDirty(page
)) {
878 write_lock_irq(&mapping
->tree_lock
);
879 if (page
->mapping
) { /* Race with truncate? */
880 if (mapping_cap_account_dirty(mapping
))
881 inc_page_state(nr_dirty
);
882 radix_tree_tag_set(&mapping
->page_tree
,
884 PAGECACHE_TAG_DIRTY
);
886 write_unlock_irq(&mapping
->tree_lock
);
887 __mark_inode_dirty(mapping
->host
, I_DIRTY_PAGES
);
892 EXPORT_SYMBOL(__set_page_dirty_buffers
);
895 * Write out and wait upon a list of buffers.
897 * We have conflicting pressures: we want to make sure that all
898 * initially dirty buffers get waited on, but that any subsequently
899 * dirtied buffers don't. After all, we don't want fsync to last
900 * forever if somebody is actively writing to the file.
902 * Do this in two main stages: first we copy dirty buffers to a
903 * temporary inode list, queueing the writes as we go. Then we clean
904 * up, waiting for those writes to complete.
906 * During this second stage, any subsequent updates to the file may end
907 * up refiling the buffer on the original inode's dirty list again, so
908 * there is a chance we will end up with a buffer queued for write but
909 * not yet completed on that list. So, as a final cleanup we go through
910 * the osync code to catch these locked, dirty buffers without requeuing
911 * any newly dirty buffers for write.
913 static int fsync_buffers_list(spinlock_t
*lock
, struct list_head
*list
)
915 struct buffer_head
*bh
;
916 struct list_head tmp
;
919 INIT_LIST_HEAD(&tmp
);
922 while (!list_empty(list
)) {
923 bh
= BH_ENTRY(list
->next
);
924 list_del_init(&bh
->b_assoc_buffers
);
925 if (buffer_dirty(bh
) || buffer_locked(bh
)) {
926 list_add(&bh
->b_assoc_buffers
, &tmp
);
927 if (buffer_dirty(bh
)) {
931 * Ensure any pending I/O completes so that
932 * ll_rw_block() actually writes the current
933 * contents - it is a noop if I/O is still in
934 * flight on potentially older contents.
937 ll_rw_block(WRITE
, 1, &bh
);
944 while (!list_empty(&tmp
)) {
945 bh
= BH_ENTRY(tmp
.prev
);
946 __remove_assoc_queue(bh
);
950 if (!buffer_uptodate(bh
))
957 err2
= osync_buffers_list(lock
, list
);
965 * Invalidate any and all dirty buffers on a given inode. We are
966 * probably unmounting the fs, but that doesn't mean we have already
967 * done a sync(). Just drop the buffers from the inode list.
969 * NOTE: we take the inode's blockdev's mapping's private_lock. Which
970 * assumes that all the buffers are against the blockdev. Not true
973 void invalidate_inode_buffers(struct inode
*inode
)
975 if (inode_has_buffers(inode
)) {
976 struct address_space
*mapping
= &inode
->i_data
;
977 struct list_head
*list
= &mapping
->private_list
;
978 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
980 spin_lock(&buffer_mapping
->private_lock
);
981 while (!list_empty(list
))
982 __remove_assoc_queue(BH_ENTRY(list
->next
));
983 spin_unlock(&buffer_mapping
->private_lock
);
988 * Remove any clean buffers from the inode's buffer list. This is called
989 * when we're trying to free the inode itself. Those buffers can pin it.
991 * Returns true if all buffers were removed.
993 int remove_inode_buffers(struct inode
*inode
)
997 if (inode_has_buffers(inode
)) {
998 struct address_space
*mapping
= &inode
->i_data
;
999 struct list_head
*list
= &mapping
->private_list
;
1000 struct address_space
*buffer_mapping
= mapping
->assoc_mapping
;
1002 spin_lock(&buffer_mapping
->private_lock
);
1003 while (!list_empty(list
)) {
1004 struct buffer_head
*bh
= BH_ENTRY(list
->next
);
1005 if (buffer_dirty(bh
)) {
1009 __remove_assoc_queue(bh
);
1011 spin_unlock(&buffer_mapping
->private_lock
);
1017 * Create the appropriate buffers when given a page for data area and
1018 * the size of each buffer.. Use the bh->b_this_page linked list to
1019 * follow the buffers created. Return NULL if unable to create more
1022 * The retry flag is used to differentiate async IO (paging, swapping)
1023 * which may not fail from ordinary buffer allocations.
1025 struct buffer_head
*alloc_page_buffers(struct page
*page
, unsigned long size
,
1028 struct buffer_head
*bh
, *head
;
1034 while ((offset
-= size
) >= 0) {
1035 bh
= alloc_buffer_head(GFP_NOFS
);
1040 bh
->b_this_page
= head
;
1045 atomic_set(&bh
->b_count
, 0);
1048 /* Link the buffer to its page */
1049 set_bh_page(bh
, page
, offset
);
1051 bh
->b_end_io
= NULL
;
1055 * In case anything failed, we just free everything we got.
1061 head
= head
->b_this_page
;
1062 free_buffer_head(bh
);
1067 * Return failure for non-async IO requests. Async IO requests
1068 * are not allowed to fail, so we have to wait until buffer heads
1069 * become available. But we don't want tasks sleeping with
1070 * partially complete buffers, so all were released above.
1075 /* We're _really_ low on memory. Now we just
1076 * wait for old buffer heads to become free due to
1077 * finishing IO. Since this is an async request and
1078 * the reserve list is empty, we're sure there are
1079 * async buffer heads in use.
1084 EXPORT_SYMBOL_GPL(alloc_page_buffers
);
1087 link_dev_buffers(struct page
*page
, struct buffer_head
*head
)
1089 struct buffer_head
*bh
, *tail
;
1094 bh
= bh
->b_this_page
;
1096 tail
->b_this_page
= head
;
1097 attach_page_buffers(page
, head
);
1101 * Initialise the state of a blockdev page's buffers.
1104 init_page_buffers(struct page
*page
, struct block_device
*bdev
,
1105 sector_t block
, int size
)
1107 struct buffer_head
*head
= page_buffers(page
);
1108 struct buffer_head
*bh
= head
;
1109 int uptodate
= PageUptodate(page
);
1112 if (!buffer_mapped(bh
)) {
1113 init_buffer(bh
, NULL
, NULL
);
1115 bh
->b_blocknr
= block
;
1117 set_buffer_uptodate(bh
);
1118 set_buffer_mapped(bh
);
1121 bh
= bh
->b_this_page
;
1122 } while (bh
!= head
);
1126 * Create the page-cache page that contains the requested block.
1128 * This is user purely for blockdev mappings.
1130 static struct page
*
1131 grow_dev_page(struct block_device
*bdev
, sector_t block
,
1132 pgoff_t index
, int size
)
1134 struct inode
*inode
= bdev
->bd_inode
;
1136 struct buffer_head
*bh
;
1138 page
= find_or_create_page(inode
->i_mapping
, index
, GFP_NOFS
);
1142 if (!PageLocked(page
))
1145 if (page_has_buffers(page
)) {
1146 bh
= page_buffers(page
);
1147 if (bh
->b_size
== size
) {
1148 init_page_buffers(page
, bdev
, block
, size
);
1151 if (!try_to_free_buffers(page
))
1156 * Allocate some buffers for this page
1158 bh
= alloc_page_buffers(page
, size
, 0);
1163 * Link the page to the buffers and initialise them. Take the
1164 * lock to be atomic wrt __find_get_block(), which does not
1165 * run under the page lock.
1167 spin_lock(&inode
->i_mapping
->private_lock
);
1168 link_dev_buffers(page
, bh
);
1169 init_page_buffers(page
, bdev
, block
, size
);
1170 spin_unlock(&inode
->i_mapping
->private_lock
);
1176 page_cache_release(page
);
1181 * Create buffers for the specified block device block's page. If
1182 * that page was dirty, the buffers are set dirty also.
1184 * Except that's a bug. Attaching dirty buffers to a dirty
1185 * blockdev's page can result in filesystem corruption, because
1186 * some of those buffers may be aliases of filesystem data.
1187 * grow_dev_page() will go BUG() if this happens.
1190 grow_buffers(struct block_device
*bdev
, sector_t block
, int size
)
1199 } while ((size
<< sizebits
) < PAGE_SIZE
);
1201 index
= block
>> sizebits
;
1202 block
= index
<< sizebits
;
1204 /* Create a page with the proper size buffers.. */
1205 page
= grow_dev_page(bdev
, block
, index
, size
);
1209 page_cache_release(page
);
1213 struct buffer_head
*
1214 __getblk_slow(struct block_device
*bdev
, sector_t block
, int size
)
1216 /* Size must be multiple of hard sectorsize */
1217 if (unlikely(size
& (bdev_hardsect_size(bdev
)-1) ||
1218 (size
< 512 || size
> PAGE_SIZE
))) {
1219 printk(KERN_ERR
"getblk(): invalid block size %d requested\n",
1221 printk(KERN_ERR
"hardsect size: %d\n",
1222 bdev_hardsect_size(bdev
));
1229 struct buffer_head
* bh
;
1231 bh
= __find_get_block(bdev
, block
, size
);
1235 if (!grow_buffers(bdev
, block
, size
))
1241 * The relationship between dirty buffers and dirty pages:
1243 * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1244 * the page is tagged dirty in its radix tree.
1246 * At all times, the dirtiness of the buffers represents the dirtiness of
1247 * subsections of the page. If the page has buffers, the page dirty bit is
1248 * merely a hint about the true dirty state.
1250 * When a page is set dirty in its entirety, all its buffers are marked dirty
1251 * (if the page has buffers).
1253 * When a buffer is marked dirty, its page is dirtied, but the page's other
1256 * Also. When blockdev buffers are explicitly read with bread(), they
1257 * individually become uptodate. But their backing page remains not
1258 * uptodate - even if all of its buffers are uptodate. A subsequent
1259 * block_read_full_page() against that page will discover all the uptodate
1260 * buffers, will set the page uptodate and will perform no I/O.
1264 * mark_buffer_dirty - mark a buffer_head as needing writeout
1265 * @bh: the buffer_head to mark dirty
1267 * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1268 * backing page dirty, then tag the page as dirty in its address_space's radix
1269 * tree and then attach the address_space's inode to its superblock's dirty
1272 * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock,
1273 * mapping->tree_lock and the global inode_lock.
1275 void fastcall
mark_buffer_dirty(struct buffer_head
*bh
)
1277 if (!buffer_dirty(bh
) && !test_set_buffer_dirty(bh
))
1278 __set_page_dirty_nobuffers(bh
->b_page
);
1282 * Decrement a buffer_head's reference count. If all buffers against a page
1283 * have zero reference count, are clean and unlocked, and if the page is clean
1284 * and unlocked then try_to_free_buffers() may strip the buffers from the page
1285 * in preparation for freeing it (sometimes, rarely, buffers are removed from
1286 * a page but it ends up not being freed, and buffers may later be reattached).
1288 void __brelse(struct buffer_head
* buf
)
1290 if (atomic_read(&buf
->b_count
)) {
1294 printk(KERN_ERR
"VFS: brelse: Trying to free free buffer\n");
1299 * bforget() is like brelse(), except it discards any
1300 * potentially dirty data.
1302 void __bforget(struct buffer_head
*bh
)
1304 clear_buffer_dirty(bh
);
1305 if (!list_empty(&bh
->b_assoc_buffers
)) {
1306 struct address_space
*buffer_mapping
= bh
->b_page
->mapping
;
1308 spin_lock(&buffer_mapping
->private_lock
);
1309 list_del_init(&bh
->b_assoc_buffers
);
1310 spin_unlock(&buffer_mapping
->private_lock
);
1315 static struct buffer_head
*__bread_slow(struct buffer_head
*bh
)
1318 if (buffer_uptodate(bh
)) {
1323 bh
->b_end_io
= end_buffer_read_sync
;
1324 submit_bh(READ
, bh
);
1326 if (buffer_uptodate(bh
))
1334 * Per-cpu buffer LRU implementation. To reduce the cost of __find_get_block().
1335 * The bhs[] array is sorted - newest buffer is at bhs[0]. Buffers have their
1336 * refcount elevated by one when they're in an LRU. A buffer can only appear
1337 * once in a particular CPU's LRU. A single buffer can be present in multiple
1338 * CPU's LRUs at the same time.
1340 * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1341 * sb_find_get_block().
1343 * The LRUs themselves only need locking against invalidate_bh_lrus. We use
1344 * a local interrupt disable for that.
1347 #define BH_LRU_SIZE 8
1350 struct buffer_head
*bhs
[BH_LRU_SIZE
];
1353 static DEFINE_PER_CPU(struct bh_lru
, bh_lrus
) = {{ NULL
}};
1356 #define bh_lru_lock() local_irq_disable()
1357 #define bh_lru_unlock() local_irq_enable()
1359 #define bh_lru_lock() preempt_disable()
1360 #define bh_lru_unlock() preempt_enable()
1363 static inline void check_irqs_on(void)
1365 #ifdef irqs_disabled
1366 BUG_ON(irqs_disabled());
1371 * The LRU management algorithm is dopey-but-simple. Sorry.
1373 static void bh_lru_install(struct buffer_head
*bh
)
1375 struct buffer_head
*evictee
= NULL
;
1380 lru
= &__get_cpu_var(bh_lrus
);
1381 if (lru
->bhs
[0] != bh
) {
1382 struct buffer_head
*bhs
[BH_LRU_SIZE
];
1388 for (in
= 0; in
< BH_LRU_SIZE
; in
++) {
1389 struct buffer_head
*bh2
= lru
->bhs
[in
];
1394 if (out
>= BH_LRU_SIZE
) {
1395 BUG_ON(evictee
!= NULL
);
1402 while (out
< BH_LRU_SIZE
)
1404 memcpy(lru
->bhs
, bhs
, sizeof(bhs
));
1413 * Look up the bh in this cpu's LRU. If it's there, move it to the head.
1415 static inline struct buffer_head
*
1416 lookup_bh_lru(struct block_device
*bdev
, sector_t block
, int size
)
1418 struct buffer_head
*ret
= NULL
;
1424 lru
= &__get_cpu_var(bh_lrus
);
1425 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
1426 struct buffer_head
*bh
= lru
->bhs
[i
];
1428 if (bh
&& bh
->b_bdev
== bdev
&&
1429 bh
->b_blocknr
== block
&& bh
->b_size
== size
) {
1432 lru
->bhs
[i
] = lru
->bhs
[i
- 1];
1447 * Perform a pagecache lookup for the matching buffer. If it's there, refresh
1448 * it in the LRU and mark it as accessed. If it is not present then return
1451 struct buffer_head
*
1452 __find_get_block(struct block_device
*bdev
, sector_t block
, int size
)
1454 struct buffer_head
*bh
= lookup_bh_lru(bdev
, block
, size
);
1457 bh
= __find_get_block_slow(bdev
, block
, size
);
1465 EXPORT_SYMBOL(__find_get_block
);
1468 * __getblk will locate (and, if necessary, create) the buffer_head
1469 * which corresponds to the passed block_device, block and size. The
1470 * returned buffer has its reference count incremented.
1472 * __getblk() cannot fail - it just keeps trying. If you pass it an
1473 * illegal block number, __getblk() will happily return a buffer_head
1474 * which represents the non-existent block. Very weird.
1476 * __getblk() will lock up the machine if grow_dev_page's try_to_free_buffers()
1477 * attempt is failing. FIXME, perhaps?
1479 struct buffer_head
*
1480 __getblk(struct block_device
*bdev
, sector_t block
, int size
)
1482 struct buffer_head
*bh
= __find_get_block(bdev
, block
, size
);
1486 bh
= __getblk_slow(bdev
, block
, size
);
1489 EXPORT_SYMBOL(__getblk
);
1492 * Do async read-ahead on a buffer..
1494 void __breadahead(struct block_device
*bdev
, sector_t block
, int size
)
1496 struct buffer_head
*bh
= __getblk(bdev
, block
, size
);
1497 ll_rw_block(READA
, 1, &bh
);
1500 EXPORT_SYMBOL(__breadahead
);
1503 * __bread() - reads a specified block and returns the bh
1504 * @bdev: the block_device to read from
1505 * @block: number of block
1506 * @size: size (in bytes) to read
1508 * Reads a specified block, and returns buffer head that contains it.
1509 * It returns NULL if the block was unreadable.
1511 struct buffer_head
*
1512 __bread(struct block_device
*bdev
, sector_t block
, int size
)
1514 struct buffer_head
*bh
= __getblk(bdev
, block
, size
);
1516 if (!buffer_uptodate(bh
))
1517 bh
= __bread_slow(bh
);
1520 EXPORT_SYMBOL(__bread
);
1523 * invalidate_bh_lrus() is called rarely - but not only at unmount.
1524 * This doesn't race because it runs in each cpu either in irq
1525 * or with preempt disabled.
1527 static void invalidate_bh_lru(void *arg
)
1529 struct bh_lru
*b
= &get_cpu_var(bh_lrus
);
1532 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
1536 put_cpu_var(bh_lrus
);
1539 static void invalidate_bh_lrus(void)
1541 on_each_cpu(invalidate_bh_lru
, NULL
, 1, 1);
1544 void set_bh_page(struct buffer_head
*bh
,
1545 struct page
*page
, unsigned long offset
)
1548 if (offset
>= PAGE_SIZE
)
1550 if (PageHighMem(page
))
1552 * This catches illegal uses and preserves the offset:
1554 bh
->b_data
= (char *)(0 + offset
);
1556 bh
->b_data
= page_address(page
) + offset
;
1558 EXPORT_SYMBOL(set_bh_page
);
1561 * Called when truncating a buffer on a page completely.
1563 static inline void discard_buffer(struct buffer_head
* bh
)
1566 clear_buffer_dirty(bh
);
1568 clear_buffer_mapped(bh
);
1569 clear_buffer_req(bh
);
1570 clear_buffer_new(bh
);
1571 clear_buffer_delay(bh
);
1576 * try_to_release_page() - release old fs-specific metadata on a page
1578 * @page: the page which the kernel is trying to free
1579 * @gfp_mask: memory allocation flags (and I/O mode)
1581 * The address_space is to try to release any data against the page
1582 * (presumably at page->private). If the release was successful, return `1'.
1583 * Otherwise return zero.
1585 * The @gfp_mask argument specifies whether I/O may be performed to release
1586 * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
1588 * NOTE: @gfp_mask may go away, and this function may become non-blocking.
1590 int try_to_release_page(struct page
*page
, int gfp_mask
)
1592 struct address_space
* const mapping
= page
->mapping
;
1594 BUG_ON(!PageLocked(page
));
1595 if (PageWriteback(page
))
1598 if (mapping
&& mapping
->a_ops
->releasepage
)
1599 return mapping
->a_ops
->releasepage(page
, gfp_mask
);
1600 return try_to_free_buffers(page
);
1602 EXPORT_SYMBOL(try_to_release_page
);
1605 * block_invalidatepage - invalidate part of all of a buffer-backed page
1607 * @page: the page which is affected
1608 * @offset: the index of the truncation point
1610 * block_invalidatepage() is called when all or part of the page has become
1611 * invalidatedby a truncate operation.
1613 * block_invalidatepage() does not have to release all buffers, but it must
1614 * ensure that no dirty buffer is left outside @offset and that no I/O
1615 * is underway against any of the blocks which are outside the truncation
1616 * point. Because the caller is about to free (and possibly reuse) those
1619 int block_invalidatepage(struct page
*page
, unsigned long offset
)
1621 struct buffer_head
*head
, *bh
, *next
;
1622 unsigned int curr_off
= 0;
1625 BUG_ON(!PageLocked(page
));
1626 if (!page_has_buffers(page
))
1629 head
= page_buffers(page
);
1632 unsigned int next_off
= curr_off
+ bh
->b_size
;
1633 next
= bh
->b_this_page
;
1636 * is this block fully invalidated?
1638 if (offset
<= curr_off
)
1640 curr_off
= next_off
;
1642 } while (bh
!= head
);
1645 * We release buffers only if the entire page is being invalidated.
1646 * The get_block cached value has been unconditionally invalidated,
1647 * so real IO is not possible anymore.
1650 ret
= try_to_release_page(page
, 0);
1654 EXPORT_SYMBOL(block_invalidatepage
);
1657 * We attach and possibly dirty the buffers atomically wrt
1658 * __set_page_dirty_buffers() via private_lock. try_to_free_buffers
1659 * is already excluded via the page lock.
1661 void create_empty_buffers(struct page
*page
,
1662 unsigned long blocksize
, unsigned long b_state
)
1664 struct buffer_head
*bh
, *head
, *tail
;
1666 head
= alloc_page_buffers(page
, blocksize
, 1);
1669 bh
->b_state
|= b_state
;
1671 bh
= bh
->b_this_page
;
1673 tail
->b_this_page
= head
;
1675 spin_lock(&page
->mapping
->private_lock
);
1676 if (PageUptodate(page
) || PageDirty(page
)) {
1679 if (PageDirty(page
))
1680 set_buffer_dirty(bh
);
1681 if (PageUptodate(page
))
1682 set_buffer_uptodate(bh
);
1683 bh
= bh
->b_this_page
;
1684 } while (bh
!= head
);
1686 attach_page_buffers(page
, head
);
1687 spin_unlock(&page
->mapping
->private_lock
);
1689 EXPORT_SYMBOL(create_empty_buffers
);
1692 * We are taking a block for data and we don't want any output from any
1693 * buffer-cache aliases starting from return from that function and
1694 * until the moment when something will explicitly mark the buffer
1695 * dirty (hopefully that will not happen until we will free that block ;-)
1696 * We don't even need to mark it not-uptodate - nobody can expect
1697 * anything from a newly allocated buffer anyway. We used to used
1698 * unmap_buffer() for such invalidation, but that was wrong. We definitely
1699 * don't want to mark the alias unmapped, for example - it would confuse
1700 * anyone who might pick it with bread() afterwards...
1702 * Also.. Note that bforget() doesn't lock the buffer. So there can
1703 * be writeout I/O going on against recently-freed buffers. We don't
1704 * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1705 * only if we really need to. That happens here.
1707 void unmap_underlying_metadata(struct block_device
*bdev
, sector_t block
)
1709 struct buffer_head
*old_bh
;
1713 old_bh
= __find_get_block_slow(bdev
, block
, 0);
1715 clear_buffer_dirty(old_bh
);
1716 wait_on_buffer(old_bh
);
1717 clear_buffer_req(old_bh
);
1721 EXPORT_SYMBOL(unmap_underlying_metadata
);
1724 * NOTE! All mapped/uptodate combinations are valid:
1726 * Mapped Uptodate Meaning
1728 * No No "unknown" - must do get_block()
1729 * No Yes "hole" - zero-filled
1730 * Yes No "allocated" - allocated on disk, not read in
1731 * Yes Yes "valid" - allocated and up-to-date in memory.
1733 * "Dirty" is valid only with the last case (mapped+uptodate).
1737 * While block_write_full_page is writing back the dirty buffers under
1738 * the page lock, whoever dirtied the buffers may decide to clean them
1739 * again at any time. We handle that by only looking at the buffer
1740 * state inside lock_buffer().
1742 * If block_write_full_page() is called for regular writeback
1743 * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1744 * locked buffer. This only can happen if someone has written the buffer
1745 * directly, with submit_bh(). At the address_space level PageWriteback
1746 * prevents this contention from occurring.
1748 static int __block_write_full_page(struct inode
*inode
, struct page
*page
,
1749 get_block_t
*get_block
, struct writeback_control
*wbc
)
1753 sector_t last_block
;
1754 struct buffer_head
*bh
, *head
, *last_bh
= NULL
;
1755 int nr_underway
= 0;
1757 BUG_ON(!PageLocked(page
));
1759 last_block
= (i_size_read(inode
) - 1) >> inode
->i_blkbits
;
1761 if (!page_has_buffers(page
)) {
1762 create_empty_buffers(page
, 1 << inode
->i_blkbits
,
1763 (1 << BH_Dirty
)|(1 << BH_Uptodate
));
1767 * Be very careful. We have no exclusion from __set_page_dirty_buffers
1768 * here, and the (potentially unmapped) buffers may become dirty at
1769 * any time. If a buffer becomes dirty here after we've inspected it
1770 * then we just miss that fact, and the page stays dirty.
1772 * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1773 * handle that here by just cleaning them.
1776 block
= page
->index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
1777 head
= page_buffers(page
);
1781 * Get all the dirty buffers mapped to disk addresses and
1782 * handle any aliases from the underlying blockdev's mapping.
1785 if (block
> last_block
) {
1787 * mapped buffers outside i_size will occur, because
1788 * this page can be outside i_size when there is a
1789 * truncate in progress.
1792 * The buffer was zeroed by block_write_full_page()
1794 clear_buffer_dirty(bh
);
1795 set_buffer_uptodate(bh
);
1796 } else if (!buffer_mapped(bh
) && buffer_dirty(bh
)) {
1797 err
= get_block(inode
, block
, bh
, 1);
1800 if (buffer_new(bh
)) {
1801 /* blockdev mappings never come here */
1802 clear_buffer_new(bh
);
1803 unmap_underlying_metadata(bh
->b_bdev
,
1807 bh
= bh
->b_this_page
;
1809 } while (bh
!= head
);
1812 if (!buffer_mapped(bh
))
1815 * If it's a fully non-blocking write attempt and we cannot
1816 * lock the buffer then redirty the page. Note that this can
1817 * potentially cause a busy-wait loop from pdflush and kswapd
1818 * activity, but those code paths have their own higher-level
1821 if (wbc
->sync_mode
!= WB_SYNC_NONE
|| !wbc
->nonblocking
) {
1823 } else if (test_set_buffer_locked(bh
)) {
1824 redirty_page_for_writepage(wbc
, page
);
1827 if (test_clear_buffer_dirty(bh
)) {
1828 mark_buffer_async_write(bh
);
1833 } while ((bh
= bh
->b_this_page
) != head
);
1836 * The page and its buffers are protected by PageWriteback(), so we can
1837 * drop the bh refcounts early.
1839 BUG_ON(PageWriteback(page
));
1840 set_page_writeback(page
);
1843 struct buffer_head
*next
= bh
->b_this_page
;
1844 if (buffer_async_write(bh
)) {
1845 submit_bh(WRITE
, bh
);
1851 } while (bh
!= head
);
1857 if (nr_underway
== 0) {
1859 * The page was marked dirty, but the buffers were
1860 * clean. Someone wrote them back by hand with
1861 * ll_rw_block/submit_bh. A rare case.
1865 if (!buffer_uptodate(bh
)) {
1869 bh
= bh
->b_this_page
;
1870 } while (bh
!= head
);
1872 SetPageUptodate(page
);
1873 end_page_writeback(page
);
1875 * The page and buffer_heads can be released at any time from
1878 wbc
->pages_skipped
++; /* We didn't write this page */
1884 * ENOSPC, or some other error. We may already have added some
1885 * blocks to the file, so we need to write these out to avoid
1886 * exposing stale data.
1887 * The page is currently locked and not marked for writeback
1890 /* Recovery: lock and submit the mapped buffers */
1892 if (buffer_mapped(bh
) && buffer_dirty(bh
)) {
1894 mark_buffer_async_write(bh
);
1898 * The buffer may have been set dirty during
1899 * attachment to a dirty page.
1901 clear_buffer_dirty(bh
);
1903 } while ((bh
= bh
->b_this_page
) != head
);
1905 BUG_ON(PageWriteback(page
));
1906 set_page_writeback(page
);
1909 struct buffer_head
*next
= bh
->b_this_page
;
1910 if (buffer_async_write(bh
)) {
1911 clear_buffer_dirty(bh
);
1912 submit_bh(WRITE
, bh
);
1918 } while (bh
!= head
);
1923 static int __block_prepare_write(struct inode
*inode
, struct page
*page
,
1924 unsigned from
, unsigned to
, get_block_t
*get_block
)
1926 unsigned block_start
, block_end
;
1929 unsigned blocksize
, bbits
;
1930 struct buffer_head
*bh
, *head
, *wait
[2], **wait_bh
=wait
;
1932 BUG_ON(!PageLocked(page
));
1933 BUG_ON(from
> PAGE_CACHE_SIZE
);
1934 BUG_ON(to
> PAGE_CACHE_SIZE
);
1937 blocksize
= 1 << inode
->i_blkbits
;
1938 if (!page_has_buffers(page
))
1939 create_empty_buffers(page
, blocksize
, 0);
1940 head
= page_buffers(page
);
1942 bbits
= inode
->i_blkbits
;
1943 block
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- bbits
);
1945 for(bh
= head
, block_start
= 0; bh
!= head
|| !block_start
;
1946 block
++, block_start
=block_end
, bh
= bh
->b_this_page
) {
1947 block_end
= block_start
+ blocksize
;
1948 if (block_end
<= from
|| block_start
>= to
) {
1949 if (PageUptodate(page
)) {
1950 if (!buffer_uptodate(bh
))
1951 set_buffer_uptodate(bh
);
1956 clear_buffer_new(bh
);
1957 if (!buffer_mapped(bh
)) {
1958 err
= get_block(inode
, block
, bh
, 1);
1961 if (buffer_new(bh
)) {
1962 clear_buffer_new(bh
);
1963 unmap_underlying_metadata(bh
->b_bdev
,
1965 if (PageUptodate(page
)) {
1966 set_buffer_uptodate(bh
);
1969 if (block_end
> to
|| block_start
< from
) {
1972 kaddr
= kmap_atomic(page
, KM_USER0
);
1976 if (block_start
< from
)
1977 memset(kaddr
+block_start
,
1978 0, from
-block_start
);
1979 flush_dcache_page(page
);
1980 kunmap_atomic(kaddr
, KM_USER0
);
1985 if (PageUptodate(page
)) {
1986 if (!buffer_uptodate(bh
))
1987 set_buffer_uptodate(bh
);
1990 if (!buffer_uptodate(bh
) && !buffer_delay(bh
) &&
1991 (block_start
< from
|| block_end
> to
)) {
1992 ll_rw_block(READ
, 1, &bh
);
1997 * If we issued read requests - let them complete.
1999 while(wait_bh
> wait
) {
2000 wait_on_buffer(*--wait_bh
);
2001 if (!buffer_uptodate(*wait_bh
))
2009 * Zero out any newly allocated blocks to avoid exposing stale
2010 * data. If BH_New is set, we know that the block was newly
2011 * allocated in the above loop.
2016 block_end
= block_start
+blocksize
;
2017 if (block_end
<= from
)
2019 if (block_start
>= to
)
2021 if (buffer_new(bh
)) {
2024 clear_buffer_new(bh
);
2025 kaddr
= kmap_atomic(page
, KM_USER0
);
2026 memset(kaddr
+block_start
, 0, bh
->b_size
);
2027 kunmap_atomic(kaddr
, KM_USER0
);
2028 set_buffer_uptodate(bh
);
2029 mark_buffer_dirty(bh
);
2032 block_start
= block_end
;
2033 bh
= bh
->b_this_page
;
2034 } while (bh
!= head
);
2038 static int __block_commit_write(struct inode
*inode
, struct page
*page
,
2039 unsigned from
, unsigned to
)
2041 unsigned block_start
, block_end
;
2044 struct buffer_head
*bh
, *head
;
2046 blocksize
= 1 << inode
->i_blkbits
;
2048 for(bh
= head
= page_buffers(page
), block_start
= 0;
2049 bh
!= head
|| !block_start
;
2050 block_start
=block_end
, bh
= bh
->b_this_page
) {
2051 block_end
= block_start
+ blocksize
;
2052 if (block_end
<= from
|| block_start
>= to
) {
2053 if (!buffer_uptodate(bh
))
2056 set_buffer_uptodate(bh
);
2057 mark_buffer_dirty(bh
);
2062 * If this is a partial write which happened to make all buffers
2063 * uptodate then we can optimize away a bogus readpage() for
2064 * the next read(). Here we 'discover' whether the page went
2065 * uptodate as a result of this (potentially partial) write.
2068 SetPageUptodate(page
);
2073 * Generic "read page" function for block devices that have the normal
2074 * get_block functionality. This is most of the block device filesystems.
2075 * Reads the page asynchronously --- the unlock_buffer() and
2076 * set/clear_buffer_uptodate() functions propagate buffer state into the
2077 * page struct once IO has completed.
2079 int block_read_full_page(struct page
*page
, get_block_t
*get_block
)
2081 struct inode
*inode
= page
->mapping
->host
;
2082 sector_t iblock
, lblock
;
2083 struct buffer_head
*bh
, *head
, *arr
[MAX_BUF_PER_PAGE
];
2084 unsigned int blocksize
;
2086 int fully_mapped
= 1;
2088 BUG_ON(!PageLocked(page
));
2089 blocksize
= 1 << inode
->i_blkbits
;
2090 if (!page_has_buffers(page
))
2091 create_empty_buffers(page
, blocksize
, 0);
2092 head
= page_buffers(page
);
2094 iblock
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
2095 lblock
= (i_size_read(inode
)+blocksize
-1) >> inode
->i_blkbits
;
2101 if (buffer_uptodate(bh
))
2104 if (!buffer_mapped(bh
)) {
2106 if (iblock
< lblock
) {
2107 if (get_block(inode
, iblock
, bh
, 0))
2110 if (!buffer_mapped(bh
)) {
2111 void *kaddr
= kmap_atomic(page
, KM_USER0
);
2112 memset(kaddr
+ i
* blocksize
, 0, blocksize
);
2113 flush_dcache_page(page
);
2114 kunmap_atomic(kaddr
, KM_USER0
);
2115 set_buffer_uptodate(bh
);
2119 * get_block() might have updated the buffer
2122 if (buffer_uptodate(bh
))
2126 } while (i
++, iblock
++, (bh
= bh
->b_this_page
) != head
);
2129 SetPageMappedToDisk(page
);
2133 * All buffers are uptodate - we can set the page uptodate
2134 * as well. But not if get_block() returned an error.
2136 if (!PageError(page
))
2137 SetPageUptodate(page
);
2142 /* Stage two: lock the buffers */
2143 for (i
= 0; i
< nr
; i
++) {
2146 mark_buffer_async_read(bh
);
2150 * Stage 3: start the IO. Check for uptodateness
2151 * inside the buffer lock in case another process reading
2152 * the underlying blockdev brought it uptodate (the sct fix).
2154 for (i
= 0; i
< nr
; i
++) {
2156 if (buffer_uptodate(bh
))
2157 end_buffer_async_read(bh
, 1);
2159 submit_bh(READ
, bh
);
2164 /* utility function for filesystems that need to do work on expanding
2165 * truncates. Uses prepare/commit_write to allow the filesystem to
2166 * deal with the hole.
2168 int generic_cont_expand(struct inode
*inode
, loff_t size
)
2170 struct address_space
*mapping
= inode
->i_mapping
;
2172 unsigned long index
, offset
, limit
;
2176 limit
= current
->signal
->rlim
[RLIMIT_FSIZE
].rlim_cur
;
2177 if (limit
!= RLIM_INFINITY
&& size
> (loff_t
)limit
) {
2178 send_sig(SIGXFSZ
, current
, 0);
2181 if (size
> inode
->i_sb
->s_maxbytes
)
2184 offset
= (size
& (PAGE_CACHE_SIZE
-1)); /* Within page */
2186 /* ugh. in prepare/commit_write, if from==to==start of block, we
2187 ** skip the prepare. make sure we never send an offset for the start
2190 if ((offset
& (inode
->i_sb
->s_blocksize
- 1)) == 0) {
2193 index
= size
>> PAGE_CACHE_SHIFT
;
2195 page
= grab_cache_page(mapping
, index
);
2198 err
= mapping
->a_ops
->prepare_write(NULL
, page
, offset
, offset
);
2200 err
= mapping
->a_ops
->commit_write(NULL
, page
, offset
, offset
);
2203 page_cache_release(page
);
2211 * For moronic filesystems that do not allow holes in file.
2212 * We may have to extend the file.
2215 int cont_prepare_write(struct page
*page
, unsigned offset
,
2216 unsigned to
, get_block_t
*get_block
, loff_t
*bytes
)
2218 struct address_space
*mapping
= page
->mapping
;
2219 struct inode
*inode
= mapping
->host
;
2220 struct page
*new_page
;
2224 unsigned blocksize
= 1 << inode
->i_blkbits
;
2227 while(page
->index
> (pgpos
= *bytes
>>PAGE_CACHE_SHIFT
)) {
2229 new_page
= grab_cache_page(mapping
, pgpos
);
2232 /* we might sleep */
2233 if (*bytes
>>PAGE_CACHE_SHIFT
!= pgpos
) {
2234 unlock_page(new_page
);
2235 page_cache_release(new_page
);
2238 zerofrom
= *bytes
& ~PAGE_CACHE_MASK
;
2239 if (zerofrom
& (blocksize
-1)) {
2240 *bytes
|= (blocksize
-1);
2243 status
= __block_prepare_write(inode
, new_page
, zerofrom
,
2244 PAGE_CACHE_SIZE
, get_block
);
2247 kaddr
= kmap_atomic(new_page
, KM_USER0
);
2248 memset(kaddr
+zerofrom
, 0, PAGE_CACHE_SIZE
-zerofrom
);
2249 flush_dcache_page(new_page
);
2250 kunmap_atomic(kaddr
, KM_USER0
);
2251 generic_commit_write(NULL
, new_page
, zerofrom
, PAGE_CACHE_SIZE
);
2252 unlock_page(new_page
);
2253 page_cache_release(new_page
);
2256 if (page
->index
< pgpos
) {
2257 /* completely inside the area */
2260 /* page covers the boundary, find the boundary offset */
2261 zerofrom
= *bytes
& ~PAGE_CACHE_MASK
;
2263 /* if we will expand the thing last block will be filled */
2264 if (to
> zerofrom
&& (zerofrom
& (blocksize
-1))) {
2265 *bytes
|= (blocksize
-1);
2269 /* starting below the boundary? Nothing to zero out */
2270 if (offset
<= zerofrom
)
2273 status
= __block_prepare_write(inode
, page
, zerofrom
, to
, get_block
);
2276 if (zerofrom
< offset
) {
2277 kaddr
= kmap_atomic(page
, KM_USER0
);
2278 memset(kaddr
+zerofrom
, 0, offset
-zerofrom
);
2279 flush_dcache_page(page
);
2280 kunmap_atomic(kaddr
, KM_USER0
);
2281 __block_commit_write(inode
, page
, zerofrom
, offset
);
2285 ClearPageUptodate(page
);
2289 ClearPageUptodate(new_page
);
2290 unlock_page(new_page
);
2291 page_cache_release(new_page
);
2296 int block_prepare_write(struct page
*page
, unsigned from
, unsigned to
,
2297 get_block_t
*get_block
)
2299 struct inode
*inode
= page
->mapping
->host
;
2300 int err
= __block_prepare_write(inode
, page
, from
, to
, get_block
);
2302 ClearPageUptodate(page
);
2306 int block_commit_write(struct page
*page
, unsigned from
, unsigned to
)
2308 struct inode
*inode
= page
->mapping
->host
;
2309 __block_commit_write(inode
,page
,from
,to
);
2313 int generic_commit_write(struct file
*file
, struct page
*page
,
2314 unsigned from
, unsigned to
)
2316 struct inode
*inode
= page
->mapping
->host
;
2317 loff_t pos
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
2318 __block_commit_write(inode
,page
,from
,to
);
2320 * No need to use i_size_read() here, the i_size
2321 * cannot change under us because we hold i_sem.
2323 if (pos
> inode
->i_size
) {
2324 i_size_write(inode
, pos
);
2325 mark_inode_dirty(inode
);
2332 * nobh_prepare_write()'s prereads are special: the buffer_heads are freed
2333 * immediately, while under the page lock. So it needs a special end_io
2334 * handler which does not touch the bh after unlocking it.
2336 * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
2337 * a race there is benign: unlock_buffer() only use the bh's address for
2338 * hashing after unlocking the buffer, so it doesn't actually touch the bh
2341 static void end_buffer_read_nobh(struct buffer_head
*bh
, int uptodate
)
2344 set_buffer_uptodate(bh
);
2346 /* This happens, due to failed READA attempts. */
2347 clear_buffer_uptodate(bh
);
2353 * On entry, the page is fully not uptodate.
2354 * On exit the page is fully uptodate in the areas outside (from,to)
2356 int nobh_prepare_write(struct page
*page
, unsigned from
, unsigned to
,
2357 get_block_t
*get_block
)
2359 struct inode
*inode
= page
->mapping
->host
;
2360 const unsigned blkbits
= inode
->i_blkbits
;
2361 const unsigned blocksize
= 1 << blkbits
;
2362 struct buffer_head map_bh
;
2363 struct buffer_head
*read_bh
[MAX_BUF_PER_PAGE
];
2364 unsigned block_in_page
;
2365 unsigned block_start
;
2366 sector_t block_in_file
;
2371 int is_mapped_to_disk
= 1;
2374 if (PageMappedToDisk(page
))
2377 block_in_file
= (sector_t
)page
->index
<< (PAGE_CACHE_SHIFT
- blkbits
);
2378 map_bh
.b_page
= page
;
2381 * We loop across all blocks in the page, whether or not they are
2382 * part of the affected region. This is so we can discover if the
2383 * page is fully mapped-to-disk.
2385 for (block_start
= 0, block_in_page
= 0;
2386 block_start
< PAGE_CACHE_SIZE
;
2387 block_in_page
++, block_start
+= blocksize
) {
2388 unsigned block_end
= block_start
+ blocksize
;
2393 if (block_start
>= to
)
2395 ret
= get_block(inode
, block_in_file
+ block_in_page
,
2399 if (!buffer_mapped(&map_bh
))
2400 is_mapped_to_disk
= 0;
2401 if (buffer_new(&map_bh
))
2402 unmap_underlying_metadata(map_bh
.b_bdev
,
2404 if (PageUptodate(page
))
2406 if (buffer_new(&map_bh
) || !buffer_mapped(&map_bh
)) {
2407 kaddr
= kmap_atomic(page
, KM_USER0
);
2408 if (block_start
< from
) {
2409 memset(kaddr
+block_start
, 0, from
-block_start
);
2412 if (block_end
> to
) {
2413 memset(kaddr
+ to
, 0, block_end
- to
);
2416 flush_dcache_page(page
);
2417 kunmap_atomic(kaddr
, KM_USER0
);
2420 if (buffer_uptodate(&map_bh
))
2421 continue; /* reiserfs does this */
2422 if (block_start
< from
|| block_end
> to
) {
2423 struct buffer_head
*bh
= alloc_buffer_head(GFP_NOFS
);
2429 bh
->b_state
= map_bh
.b_state
;
2430 atomic_set(&bh
->b_count
, 0);
2431 bh
->b_this_page
= NULL
;
2433 bh
->b_blocknr
= map_bh
.b_blocknr
;
2434 bh
->b_size
= blocksize
;
2435 bh
->b_data
= (char *)(long)block_start
;
2436 bh
->b_bdev
= map_bh
.b_bdev
;
2437 bh
->b_private
= NULL
;
2438 read_bh
[nr_reads
++] = bh
;
2443 struct buffer_head
*bh
;
2446 * The page is locked, so these buffers are protected from
2447 * any VM or truncate activity. Hence we don't need to care
2448 * for the buffer_head refcounts.
2450 for (i
= 0; i
< nr_reads
; i
++) {
2453 bh
->b_end_io
= end_buffer_read_nobh
;
2454 submit_bh(READ
, bh
);
2456 for (i
= 0; i
< nr_reads
; i
++) {
2459 if (!buffer_uptodate(bh
))
2461 free_buffer_head(bh
);
2468 if (is_mapped_to_disk
)
2469 SetPageMappedToDisk(page
);
2470 SetPageUptodate(page
);
2473 * Setting the page dirty here isn't necessary for the prepare_write
2474 * function - commit_write will do that. But if/when this function is
2475 * used within the pagefault handler to ensure that all mmapped pages
2476 * have backing space in the filesystem, we will need to dirty the page
2477 * if its contents were altered.
2480 set_page_dirty(page
);
2485 for (i
= 0; i
< nr_reads
; i
++) {
2487 free_buffer_head(read_bh
[i
]);
2491 * Error recovery is pretty slack. Clear the page and mark it dirty
2492 * so we'll later zero out any blocks which _were_ allocated.
2494 kaddr
= kmap_atomic(page
, KM_USER0
);
2495 memset(kaddr
, 0, PAGE_CACHE_SIZE
);
2496 kunmap_atomic(kaddr
, KM_USER0
);
2497 SetPageUptodate(page
);
2498 set_page_dirty(page
);
2501 EXPORT_SYMBOL(nobh_prepare_write
);
2503 int nobh_commit_write(struct file
*file
, struct page
*page
,
2504 unsigned from
, unsigned to
)
2506 struct inode
*inode
= page
->mapping
->host
;
2507 loff_t pos
= ((loff_t
)page
->index
<< PAGE_CACHE_SHIFT
) + to
;
2509 set_page_dirty(page
);
2510 if (pos
> inode
->i_size
) {
2511 i_size_write(inode
, pos
);
2512 mark_inode_dirty(inode
);
2516 EXPORT_SYMBOL(nobh_commit_write
);
2519 * nobh_writepage() - based on block_full_write_page() except
2520 * that it tries to operate without attaching bufferheads to
2523 int nobh_writepage(struct page
*page
, get_block_t
*get_block
,
2524 struct writeback_control
*wbc
)
2526 struct inode
* const inode
= page
->mapping
->host
;
2527 loff_t i_size
= i_size_read(inode
);
2528 const pgoff_t end_index
= i_size
>> PAGE_CACHE_SHIFT
;
2533 /* Is the page fully inside i_size? */
2534 if (page
->index
< end_index
)
2537 /* Is the page fully outside i_size? (truncate in progress) */
2538 offset
= i_size
& (PAGE_CACHE_SIZE
-1);
2539 if (page
->index
>= end_index
+1 || !offset
) {
2541 * The page may have dirty, unmapped buffers. For example,
2542 * they may have been added in ext3_writepage(). Make them
2543 * freeable here, so the page does not leak.
2546 /* Not really sure about this - do we need this ? */
2547 if (page
->mapping
->a_ops
->invalidatepage
)
2548 page
->mapping
->a_ops
->invalidatepage(page
, offset
);
2551 return 0; /* don't care */
2555 * The page straddles i_size. It must be zeroed out on each and every
2556 * writepage invocation because it may be mmapped. "A file is mapped
2557 * in multiples of the page size. For a file that is not a multiple of
2558 * the page size, the remaining memory is zeroed when mapped, and
2559 * writes to that region are not written out to the file."
2561 kaddr
= kmap_atomic(page
, KM_USER0
);
2562 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2563 flush_dcache_page(page
);
2564 kunmap_atomic(kaddr
, KM_USER0
);
2566 ret
= mpage_writepage(page
, get_block
, wbc
);
2568 ret
= __block_write_full_page(inode
, page
, get_block
, wbc
);
2571 EXPORT_SYMBOL(nobh_writepage
);
2574 * This function assumes that ->prepare_write() uses nobh_prepare_write().
2576 int nobh_truncate_page(struct address_space
*mapping
, loff_t from
)
2578 struct inode
*inode
= mapping
->host
;
2579 unsigned blocksize
= 1 << inode
->i_blkbits
;
2580 pgoff_t index
= from
>> PAGE_CACHE_SHIFT
;
2581 unsigned offset
= from
& (PAGE_CACHE_SIZE
-1);
2584 struct address_space_operations
*a_ops
= mapping
->a_ops
;
2588 if ((offset
& (blocksize
- 1)) == 0)
2592 page
= grab_cache_page(mapping
, index
);
2596 to
= (offset
+ blocksize
) & ~(blocksize
- 1);
2597 ret
= a_ops
->prepare_write(NULL
, page
, offset
, to
);
2599 kaddr
= kmap_atomic(page
, KM_USER0
);
2600 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2601 flush_dcache_page(page
);
2602 kunmap_atomic(kaddr
, KM_USER0
);
2603 set_page_dirty(page
);
2606 page_cache_release(page
);
2610 EXPORT_SYMBOL(nobh_truncate_page
);
2612 int block_truncate_page(struct address_space
*mapping
,
2613 loff_t from
, get_block_t
*get_block
)
2615 pgoff_t index
= from
>> PAGE_CACHE_SHIFT
;
2616 unsigned offset
= from
& (PAGE_CACHE_SIZE
-1);
2619 unsigned length
, pos
;
2620 struct inode
*inode
= mapping
->host
;
2622 struct buffer_head
*bh
;
2626 blocksize
= 1 << inode
->i_blkbits
;
2627 length
= offset
& (blocksize
- 1);
2629 /* Block boundary? Nothing to do */
2633 length
= blocksize
- length
;
2634 iblock
= index
<< (PAGE_CACHE_SHIFT
- inode
->i_blkbits
);
2636 page
= grab_cache_page(mapping
, index
);
2641 if (!page_has_buffers(page
))
2642 create_empty_buffers(page
, blocksize
, 0);
2644 /* Find the buffer that contains "offset" */
2645 bh
= page_buffers(page
);
2647 while (offset
>= pos
) {
2648 bh
= bh
->b_this_page
;
2654 if (!buffer_mapped(bh
)) {
2655 err
= get_block(inode
, iblock
, bh
, 0);
2658 /* unmapped? It's a hole - nothing to do */
2659 if (!buffer_mapped(bh
))
2663 /* Ok, it's mapped. Make sure it's up-to-date */
2664 if (PageUptodate(page
))
2665 set_buffer_uptodate(bh
);
2667 if (!buffer_uptodate(bh
) && !buffer_delay(bh
)) {
2669 ll_rw_block(READ
, 1, &bh
);
2671 /* Uhhuh. Read error. Complain and punt. */
2672 if (!buffer_uptodate(bh
))
2676 kaddr
= kmap_atomic(page
, KM_USER0
);
2677 memset(kaddr
+ offset
, 0, length
);
2678 flush_dcache_page(page
);
2679 kunmap_atomic(kaddr
, KM_USER0
);
2681 mark_buffer_dirty(bh
);
2686 page_cache_release(page
);
2692 * The generic ->writepage function for buffer-backed address_spaces
2694 int block_write_full_page(struct page
*page
, get_block_t
*get_block
,
2695 struct writeback_control
*wbc
)
2697 struct inode
* const inode
= page
->mapping
->host
;
2698 loff_t i_size
= i_size_read(inode
);
2699 const pgoff_t end_index
= i_size
>> PAGE_CACHE_SHIFT
;
2703 /* Is the page fully inside i_size? */
2704 if (page
->index
< end_index
)
2705 return __block_write_full_page(inode
, page
, get_block
, wbc
);
2707 /* Is the page fully outside i_size? (truncate in progress) */
2708 offset
= i_size
& (PAGE_CACHE_SIZE
-1);
2709 if (page
->index
>= end_index
+1 || !offset
) {
2711 * The page may have dirty, unmapped buffers. For example,
2712 * they may have been added in ext3_writepage(). Make them
2713 * freeable here, so the page does not leak.
2715 block_invalidatepage(page
, 0);
2717 return 0; /* don't care */
2721 * The page straddles i_size. It must be zeroed out on each and every
2722 * writepage invokation because it may be mmapped. "A file is mapped
2723 * in multiples of the page size. For a file that is not a multiple of
2724 * the page size, the remaining memory is zeroed when mapped, and
2725 * writes to that region are not written out to the file."
2727 kaddr
= kmap_atomic(page
, KM_USER0
);
2728 memset(kaddr
+ offset
, 0, PAGE_CACHE_SIZE
- offset
);
2729 flush_dcache_page(page
);
2730 kunmap_atomic(kaddr
, KM_USER0
);
2731 return __block_write_full_page(inode
, page
, get_block
, wbc
);
2734 sector_t
generic_block_bmap(struct address_space
*mapping
, sector_t block
,
2735 get_block_t
*get_block
)
2737 struct buffer_head tmp
;
2738 struct inode
*inode
= mapping
->host
;
2741 get_block(inode
, block
, &tmp
, 0);
2742 return tmp
.b_blocknr
;
2745 static int end_bio_bh_io_sync(struct bio
*bio
, unsigned int bytes_done
, int err
)
2747 struct buffer_head
*bh
= bio
->bi_private
;
2752 if (err
== -EOPNOTSUPP
) {
2753 set_bit(BIO_EOPNOTSUPP
, &bio
->bi_flags
);
2754 set_bit(BH_Eopnotsupp
, &bh
->b_state
);
2757 bh
->b_end_io(bh
, test_bit(BIO_UPTODATE
, &bio
->bi_flags
));
2762 int submit_bh(int rw
, struct buffer_head
* bh
)
2767 BUG_ON(!buffer_locked(bh
));
2768 BUG_ON(!buffer_mapped(bh
));
2769 BUG_ON(!bh
->b_end_io
);
2771 if (buffer_ordered(bh
) && (rw
== WRITE
))
2775 * Only clear out a write error when rewriting, should this
2776 * include WRITE_SYNC as well?
2778 if (test_set_buffer_req(bh
) && (rw
== WRITE
|| rw
== WRITE_BARRIER
))
2779 clear_buffer_write_io_error(bh
);
2782 * from here on down, it's all bio -- do the initial mapping,
2783 * submit_bio -> generic_make_request may further map this bio around
2785 bio
= bio_alloc(GFP_NOIO
, 1);
2787 bio
->bi_sector
= bh
->b_blocknr
* (bh
->b_size
>> 9);
2788 bio
->bi_bdev
= bh
->b_bdev
;
2789 bio
->bi_io_vec
[0].bv_page
= bh
->b_page
;
2790 bio
->bi_io_vec
[0].bv_len
= bh
->b_size
;
2791 bio
->bi_io_vec
[0].bv_offset
= bh_offset(bh
);
2795 bio
->bi_size
= bh
->b_size
;
2797 bio
->bi_end_io
= end_bio_bh_io_sync
;
2798 bio
->bi_private
= bh
;
2801 submit_bio(rw
, bio
);
2803 if (bio_flagged(bio
, BIO_EOPNOTSUPP
))
2811 * ll_rw_block: low-level access to block devices (DEPRECATED)
2812 * @rw: whether to %READ or %WRITE or maybe %READA (readahead)
2813 * @nr: number of &struct buffer_heads in the array
2814 * @bhs: array of pointers to &struct buffer_head
2816 * ll_rw_block() takes an array of pointers to &struct buffer_heads,
2817 * and requests an I/O operation on them, either a %READ or a %WRITE.
2818 * The third %READA option is described in the documentation for
2819 * generic_make_request() which ll_rw_block() calls.
2821 * This function drops any buffer that it cannot get a lock on (with the
2822 * BH_Lock state bit), any buffer that appears to be clean when doing a
2823 * write request, and any buffer that appears to be up-to-date when doing
2824 * read request. Further it marks as clean buffers that are processed for
2825 * writing (the buffer cache won't assume that they are actually clean until
2826 * the buffer gets unlocked).
2828 * ll_rw_block sets b_end_io to simple completion handler that marks
2829 * the buffer up-to-date (if approriate), unlocks the buffer and wakes
2832 * All of the buffers must be for the same device, and must also be a
2833 * multiple of the current approved size for the device.
2835 void ll_rw_block(int rw
, int nr
, struct buffer_head
*bhs
[])
2839 for (i
= 0; i
< nr
; i
++) {
2840 struct buffer_head
*bh
= bhs
[i
];
2842 if (test_set_buffer_locked(bh
))
2847 if (test_clear_buffer_dirty(bh
)) {
2848 bh
->b_end_io
= end_buffer_write_sync
;
2849 submit_bh(WRITE
, bh
);
2853 if (!buffer_uptodate(bh
)) {
2854 bh
->b_end_io
= end_buffer_read_sync
;
2865 * For a data-integrity writeout, we need to wait upon any in-progress I/O
2866 * and then start new I/O and then wait upon it. The caller must have a ref on
2869 int sync_dirty_buffer(struct buffer_head
*bh
)
2873 WARN_ON(atomic_read(&bh
->b_count
) < 1);
2875 if (test_clear_buffer_dirty(bh
)) {
2877 bh
->b_end_io
= end_buffer_write_sync
;
2878 ret
= submit_bh(WRITE
, bh
);
2880 if (buffer_eopnotsupp(bh
)) {
2881 clear_buffer_eopnotsupp(bh
);
2884 if (!ret
&& !buffer_uptodate(bh
))
2893 * try_to_free_buffers() checks if all the buffers on this particular page
2894 * are unused, and releases them if so.
2896 * Exclusion against try_to_free_buffers may be obtained by either
2897 * locking the page or by holding its mapping's private_lock.
2899 * If the page is dirty but all the buffers are clean then we need to
2900 * be sure to mark the page clean as well. This is because the page
2901 * may be against a block device, and a later reattachment of buffers
2902 * to a dirty page will set *all* buffers dirty. Which would corrupt
2903 * filesystem data on the same device.
2905 * The same applies to regular filesystem pages: if all the buffers are
2906 * clean then we set the page clean and proceed. To do that, we require
2907 * total exclusion from __set_page_dirty_buffers(). That is obtained with
2910 * try_to_free_buffers() is non-blocking.
2912 static inline int buffer_busy(struct buffer_head
*bh
)
2914 return atomic_read(&bh
->b_count
) |
2915 (bh
->b_state
& ((1 << BH_Dirty
) | (1 << BH_Lock
)));
2919 drop_buffers(struct page
*page
, struct buffer_head
**buffers_to_free
)
2921 struct buffer_head
*head
= page_buffers(page
);
2922 struct buffer_head
*bh
;
2926 if (buffer_write_io_error(bh
) && page
->mapping
)
2927 set_bit(AS_EIO
, &page
->mapping
->flags
);
2928 if (buffer_busy(bh
))
2930 bh
= bh
->b_this_page
;
2931 } while (bh
!= head
);
2934 struct buffer_head
*next
= bh
->b_this_page
;
2936 if (!list_empty(&bh
->b_assoc_buffers
))
2937 __remove_assoc_queue(bh
);
2939 } while (bh
!= head
);
2940 *buffers_to_free
= head
;
2941 __clear_page_buffers(page
);
2947 int try_to_free_buffers(struct page
*page
)
2949 struct address_space
* const mapping
= page
->mapping
;
2950 struct buffer_head
*buffers_to_free
= NULL
;
2953 BUG_ON(!PageLocked(page
));
2954 if (PageWriteback(page
))
2957 if (mapping
== NULL
) { /* can this still happen? */
2958 ret
= drop_buffers(page
, &buffers_to_free
);
2962 spin_lock(&mapping
->private_lock
);
2963 ret
= drop_buffers(page
, &buffers_to_free
);
2966 * If the filesystem writes its buffers by hand (eg ext3)
2967 * then we can have clean buffers against a dirty page. We
2968 * clean the page here; otherwise later reattachment of buffers
2969 * could encounter a non-uptodate page, which is unresolvable.
2970 * This only applies in the rare case where try_to_free_buffers
2971 * succeeds but the page is not freed.
2973 clear_page_dirty(page
);
2975 spin_unlock(&mapping
->private_lock
);
2977 if (buffers_to_free
) {
2978 struct buffer_head
*bh
= buffers_to_free
;
2981 struct buffer_head
*next
= bh
->b_this_page
;
2982 free_buffer_head(bh
);
2984 } while (bh
!= buffers_to_free
);
2988 EXPORT_SYMBOL(try_to_free_buffers
);
2990 int block_sync_page(struct page
*page
)
2992 struct address_space
*mapping
;
2995 mapping
= page_mapping(page
);
2997 blk_run_backing_dev(mapping
->backing_dev_info
, page
);
3002 * There are no bdflush tunables left. But distributions are
3003 * still running obsolete flush daemons, so we terminate them here.
3005 * Use of bdflush() is deprecated and will be removed in a future kernel.
3006 * The `pdflush' kernel threads fully replace bdflush daemons and this call.
3008 asmlinkage
long sys_bdflush(int func
, long data
)
3010 static int msg_count
;
3012 if (!capable(CAP_SYS_ADMIN
))
3015 if (msg_count
< 5) {
3018 "warning: process `%s' used the obsolete bdflush"
3019 " system call\n", current
->comm
);
3020 printk(KERN_INFO
"Fix your initscripts?\n");
3029 * Buffer-head allocation
3031 static kmem_cache_t
*bh_cachep
;
3034 * Once the number of bh's in the machine exceeds this level, we start
3035 * stripping them in writeback.
3037 static int max_buffer_heads
;
3039 int buffer_heads_over_limit
;
3041 struct bh_accounting
{
3042 int nr
; /* Number of live bh's */
3043 int ratelimit
; /* Limit cacheline bouncing */
3046 static DEFINE_PER_CPU(struct bh_accounting
, bh_accounting
) = {0, 0};
3048 static void recalc_bh_state(void)
3053 if (__get_cpu_var(bh_accounting
).ratelimit
++ < 4096)
3055 __get_cpu_var(bh_accounting
).ratelimit
= 0;
3057 tot
+= per_cpu(bh_accounting
, i
).nr
;
3058 buffer_heads_over_limit
= (tot
> max_buffer_heads
);
3061 struct buffer_head
*alloc_buffer_head(unsigned int __nocast gfp_flags
)
3063 struct buffer_head
*ret
= kmem_cache_alloc(bh_cachep
, gfp_flags
);
3066 __get_cpu_var(bh_accounting
).nr
++;
3072 EXPORT_SYMBOL(alloc_buffer_head
);
3074 void free_buffer_head(struct buffer_head
*bh
)
3076 BUG_ON(!list_empty(&bh
->b_assoc_buffers
));
3077 kmem_cache_free(bh_cachep
, bh
);
3079 __get_cpu_var(bh_accounting
).nr
--;
3083 EXPORT_SYMBOL(free_buffer_head
);
3086 init_buffer_head(void *data
, kmem_cache_t
*cachep
, unsigned long flags
)
3088 if ((flags
& (SLAB_CTOR_VERIFY
|SLAB_CTOR_CONSTRUCTOR
)) ==
3089 SLAB_CTOR_CONSTRUCTOR
) {
3090 struct buffer_head
* bh
= (struct buffer_head
*)data
;
3092 memset(bh
, 0, sizeof(*bh
));
3093 INIT_LIST_HEAD(&bh
->b_assoc_buffers
);
3097 #ifdef CONFIG_HOTPLUG_CPU
3098 static void buffer_exit_cpu(int cpu
)
3101 struct bh_lru
*b
= &per_cpu(bh_lrus
, cpu
);
3103 for (i
= 0; i
< BH_LRU_SIZE
; i
++) {
3109 static int buffer_cpu_notify(struct notifier_block
*self
,
3110 unsigned long action
, void *hcpu
)
3112 if (action
== CPU_DEAD
)
3113 buffer_exit_cpu((unsigned long)hcpu
);
3116 #endif /* CONFIG_HOTPLUG_CPU */
3118 void __init
buffer_init(void)
3122 bh_cachep
= kmem_cache_create("buffer_head",
3123 sizeof(struct buffer_head
), 0,
3124 SLAB_RECLAIM_ACCOUNT
|SLAB_PANIC
, init_buffer_head
, NULL
);
3127 * Limit the bh occupancy to 10% of ZONE_NORMAL
3129 nrpages
= (nr_free_buffer_pages() * 10) / 100;
3130 max_buffer_heads
= nrpages
* (PAGE_SIZE
/ sizeof(struct buffer_head
));
3131 hotcpu_notifier(buffer_cpu_notify
, 0);
3134 EXPORT_SYMBOL(__bforget
);
3135 EXPORT_SYMBOL(__brelse
);
3136 EXPORT_SYMBOL(__wait_on_buffer
);
3137 EXPORT_SYMBOL(block_commit_write
);
3138 EXPORT_SYMBOL(block_prepare_write
);
3139 EXPORT_SYMBOL(block_read_full_page
);
3140 EXPORT_SYMBOL(block_sync_page
);
3141 EXPORT_SYMBOL(block_truncate_page
);
3142 EXPORT_SYMBOL(block_write_full_page
);
3143 EXPORT_SYMBOL(cont_prepare_write
);
3144 EXPORT_SYMBOL(end_buffer_async_write
);
3145 EXPORT_SYMBOL(end_buffer_read_sync
);
3146 EXPORT_SYMBOL(end_buffer_write_sync
);
3147 EXPORT_SYMBOL(file_fsync
);
3148 EXPORT_SYMBOL(fsync_bdev
);
3149 EXPORT_SYMBOL(generic_block_bmap
);
3150 EXPORT_SYMBOL(generic_commit_write
);
3151 EXPORT_SYMBOL(generic_cont_expand
);
3152 EXPORT_SYMBOL(init_buffer
);
3153 EXPORT_SYMBOL(invalidate_bdev
);
3154 EXPORT_SYMBOL(ll_rw_block
);
3155 EXPORT_SYMBOL(mark_buffer_dirty
);
3156 EXPORT_SYMBOL(submit_bh
);
3157 EXPORT_SYMBOL(sync_dirty_buffer
);
3158 EXPORT_SYMBOL(unlock_buffer
);