Fix congestion_wait() sync/async vs read/write confusion

Commit 1faa16d22877f4839bd433547d770c676d1d964c accidentally broke
the bdi congestion wait queue logic, causing us to wait on congestion
for WRITE (== 1) when we really wanted BLK_RW_ASYNC (== 0) instead.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 7c8ca91bb9ecbacbccc42df711222e7abdc32310..1f118d462acc242990eb7a796b6832d8d90bc442 100644 (file)
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -751,7 +751,7 @@ survive:
 
                        if (retval == -ENOMEM && is_global_init(current)) {
                                up_read(&current->mm->mmap_sem);
-                               congestion_wait(WRITE, HZ/50);
+                               congestion_wait(BLK_RW_ASYNC, HZ/50);
                                goto survive;
                        }
 

diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c
index 83650e00632d80b6837678355fbac3c50c6d9a94..99a506f619b71bcadac21608b5969931441e05cf 100644 (file)
--- a/drivers/block/pktcdvd.c
+++ b/drivers/block/pktcdvd.c
@@ -1372,8 +1372,10 @@ try_next_bio:
        wakeup = (pd->write_congestion_on > 0
                        && pd->bio_queue_size <= pd->write_congestion_off);
        spin_unlock(&pd->lock);
-       if (wakeup)
-               clear_bdi_congested(&pd->disk->queue->backing_dev_info, WRITE);
+       if (wakeup) {
+               clear_bdi_congested(&pd->disk->queue->backing_dev_info,
+                                       BLK_RW_ASYNC);
+       }
 
        pkt->sleep_time = max(PACKET_WAIT_TIME, 1);
        pkt_set_state(pkt, PACKET_WAITING_STATE);
@@ -2592,10 +2594,10 @@ static int pkt_make_request(struct request_queue *q, struct bio *bio)
        spin_lock(&pd->lock);
        if (pd->write_congestion_on > 0
            && pd->bio_queue_size >= pd->write_congestion_on) {
-               set_bdi_congested(&q->backing_dev_info, WRITE);
+               set_bdi_congested(&q->backing_dev_info, BLK_RW_ASYNC);
                do {
                        spin_unlock(&pd->lock);
-                       congestion_wait(WRITE, HZ);
+                       congestion_wait(BLK_RW_ASYNC, HZ);
                        spin_lock(&pd->lock);
                } while(pd->bio_queue_size > pd->write_congestion_off);
        }

diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 9933eb861c7191b34fe43b9ac01deb5e44bd614f..529e2ba505c30055440be381d908c52b3113f44d 100644 (file)
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -776,7 +776,7 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
                 * But don't wait if split was due to the io size restriction
                 */
                if (unlikely(out_of_pages))
-                       congestion_wait(WRITE, HZ/100);
+                       congestion_wait(BLK_RW_ASYNC, HZ/100);
 
                /*
                 * With async crypto it is unsafe to share the crypto context

diff --git a/fs/fat/file.c b/fs/fat/file.c
index b28ea646ff607cfdbe0f1ebda29158b5feadef88..f042b965c95c8a9c6c0dbb21020877372b1a9c21 100644 (file)
--- a/fs/fat/file.c
+++ b/fs/fat/file.c
@@ -134,7 +134,7 @@ static int fat_file_release(struct inode *inode, struct file *filp)
        if ((filp->f_mode & FMODE_WRITE) &&
             MSDOS_SB(inode->i_sb)->options.flush) {
                fat_flush_inodes(inode->i_sb, inode, NULL);
-               congestion_wait(WRITE, HZ/10);
+               congestion_wait(BLK_RW_ASYNC, HZ/10);
        }
        return 0;
 }

diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index f58ecbc416c86043262013a3276af2927862ec11..6484eb75acd6f318c04238a51763b7cf2c9599ea 100644 (file)
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -286,8 +286,8 @@ __releases(&fc->lock)
                }
                if (fc->num_background == FUSE_CONGESTION_THRESHOLD &&
                    fc->connected && fc->bdi_initialized) {
-                       clear_bdi_congested(&fc->bdi, READ);
-                       clear_bdi_congested(&fc->bdi, WRITE);
+                       clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
+                       clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
                }
                fc->num_background--;
                fc->active_background--;
@@ -414,8 +414,8 @@ static void fuse_request_send_nowait_locked(struct fuse_conn *fc,
                fc->blocked = 1;
        if (fc->num_background == FUSE_CONGESTION_THRESHOLD &&
            fc->bdi_initialized) {
-               set_bdi_congested(&fc->bdi, READ);
-               set_bdi_congested(&fc->bdi, WRITE);
+               set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
+               set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
        }
        list_add_tail(&req->list, &fc->bg_queue);
        flush_bg_queue(fc);

diff --git a/fs/nfs/write.c b/fs/nfs/write.c
index ce728829f79a5dce2b238ccbe763631adec71343..0a0a2ff767c318d5924c603a3f67ed7f5ccc9402 100644 (file)
--- a/fs/nfs/write.c
+++ b/fs/nfs/write.c
@@ -202,8 +202,10 @@ static int nfs_set_page_writeback(struct page *page)
                struct nfs_server *nfss = NFS_SERVER(inode);
 
                if (atomic_long_inc_return(&nfss->writeback) >
-                               NFS_CONGESTION_ON_THRESH)
-                       set_bdi_congested(&nfss->backing_dev_info, WRITE);
+                               NFS_CONGESTION_ON_THRESH) {
+                       set_bdi_congested(&nfss->backing_dev_info,
+                                               BLK_RW_ASYNC);
+               }
        }
        return ret;
 }
@@ -215,7 +217,7 @@ static void nfs_end_page_writeback(struct page *page)
 
        end_page_writeback(page);
        if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
-               clear_bdi_congested(&nfss->backing_dev_info, WRITE);
+               clear_bdi_congested(&nfss->backing_dev_info, BLK_RW_ASYNC);
 }
 
 /*

diff --git a/fs/reiserfs/journal.c b/fs/reiserfs/journal.c
index 77f5bb746bf073969140539752ee87dc8f4f0861..90622200b39c0622e0f159d423c929a036d76257 100644 (file)
--- a/fs/reiserfs/journal.c
+++ b/fs/reiserfs/journal.c
@@ -997,7 +997,7 @@ static int reiserfs_async_progress_wait(struct super_block *s)
        DEFINE_WAIT(wait);
        struct reiserfs_journal *j = SB_JOURNAL(s);
        if (atomic_read(&j->j_async_throttle))
-               congestion_wait(WRITE, HZ / 10);
+               congestion_wait(BLK_RW_ASYNC, HZ / 10);
        return 0;
 }
 

diff --git a/fs/xfs/linux-2.6/kmem.c b/fs/xfs/linux-2.6/kmem.c
index 1cd3b55ee3d227d5916c18d95c2dd84a45e5285d..2d3f90afe5f14ee6ef3543d0bd6e01be014952ba 100644 (file)
--- a/fs/xfs/linux-2.6/kmem.c
+++ b/fs/xfs/linux-2.6/kmem.c
@@ -53,7 +53,7 @@ kmem_alloc(size_t size, unsigned int __nocast flags)
                        printk(KERN_ERR "XFS: possible memory allocation "
                                        "deadlock in %s (mode:0x%x)\n",
                                        __func__, lflags);
-               congestion_wait(WRITE, HZ/50);
+               congestion_wait(BLK_RW_ASYNC, HZ/50);
        } while (1);
 }
 
@@ -130,7 +130,7 @@ kmem_zone_alloc(kmem_zone_t *zone, unsigned int __nocast flags)
                        printk(KERN_ERR "XFS: possible memory allocation "
                                        "deadlock in %s (mode:0x%x)\n",
                                        __func__, lflags);
-               congestion_wait(WRITE, HZ/50);
+               congestion_wait(BLK_RW_ASYNC, HZ/50);
        } while (1);
 }
 

diff --git a/fs/xfs/linux-2.6/xfs_buf.c b/fs/xfs/linux-2.6/xfs_buf.c
index 1418b916fc2755306cca61e60c734154d543c82f..0c93c7ef3d187a27dbe6b8196de4aaa08be66447 100644 (file)
--- a/fs/xfs/linux-2.6/xfs_buf.c
+++ b/fs/xfs/linux-2.6/xfs_buf.c
@@ -412,7 +412,7 @@ _xfs_buf_lookup_pages(
 
                        XFS_STATS_INC(xb_page_retries);
                        xfsbufd_wakeup(0, gfp_mask);
-                       congestion_wait(WRITE, HZ/50);
+                       congestion_wait(BLK_RW_ASYNC, HZ/50);
                        goto retry;
                }
 

diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h
index 0ec2c594868e657ad20cfaffc21227bf7ee2e18b..3a52a63c1351e20d3df91aa59b8fff02df2e775b 100644 (file)
--- a/include/linux/backing-dev.h
+++ b/include/linux/backing-dev.h
@@ -229,9 +229,9 @@ static inline int bdi_rw_congested(struct backing_dev_info *bdi)
                                  (1 << BDI_async_congested));
 }
 
-void clear_bdi_congested(struct backing_dev_info *bdi, int rw);
-void set_bdi_congested(struct backing_dev_info *bdi, int rw);
-long congestion_wait(int rw, long timeout);
+void clear_bdi_congested(struct backing_dev_info *bdi, int sync);
+void set_bdi_congested(struct backing_dev_info *bdi, int sync);
+long congestion_wait(int sync, long timeout);
 
 
 static inline bool bdi_cap_writeback_dirty(struct backing_dev_info *bdi)

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 49ae07951d55470bd3d03df392da7ffe5c0c387b..bb3d3997870139293998814762493632329310fe 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -779,18 +779,18 @@ extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
  * congested queues, and wake up anyone who was waiting for requests to be
  * put back.
  */
-static inline void blk_clear_queue_congested(struct request_queue *q, int rw)
+static inline void blk_clear_queue_congested(struct request_queue *q, int sync)
 {
-       clear_bdi_congested(&q->backing_dev_info, rw);
+       clear_bdi_congested(&q->backing_dev_info, sync);
 }
 
 /*
  * A queue has just entered congestion.  Flag that in the queue's VM-visible
  * state flags and increment the global gounter of congested queues.
  */
-static inline void blk_set_queue_congested(struct request_queue *q, int rw)
+static inline void blk_set_queue_congested(struct request_queue *q, int sync)
 {
-       set_bdi_congested(&q->backing_dev_info, rw);
+       set_bdi_congested(&q->backing_dev_info, sync);
 }
 
 extern void blk_start_queue(struct request_queue *q);

diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index 493b468a503541fd65b64872eece6fb7228e36fa..c86edd2442944ec2532bdde3e2087b56519a224d 100644 (file)
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -283,7 +283,6 @@ static wait_queue_head_t congestion_wqh[2] = {
                __WAIT_QUEUE_HEAD_INITIALIZER(congestion_wqh[1])
        };
 
-
 void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
        enum bdi_state bit;
@@ -308,18 +307,18 @@ EXPORT_SYMBOL(set_bdi_congested);
 
 /**
  * congestion_wait - wait for a backing_dev to become uncongested
- * @rw: READ or WRITE
+ * @sync: SYNC or ASYNC IO
  * @timeout: timeout in jiffies
  *
  * Waits for up to @timeout jiffies for a backing_dev (any backing_dev) to exit
  * write congestion.  If no backing_devs are congested then just wait for the
  * next write to be completed.
  */
-long congestion_wait(int rw, long timeout)
+long congestion_wait(int sync, long timeout)
 {
        long ret;
        DEFINE_WAIT(wait);
-       wait_queue_head_t *wqh = &congestion_wqh[rw];
+       wait_queue_head_t *wqh = &congestion_wqh[sync];
 
        prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
        ret = io_schedule_timeout(timeout);

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index e2fa20dadf408e2d38e6b5a4a61ed4fb9febd3a1..e717964cb5a0cc2e73ddc520a5d76d726cb6593a 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1973,7 +1973,7 @@ try_to_free:
                if (!progress) {
                        nr_retries--;
                        /* maybe some writeback is necessary */
-                       congestion_wait(WRITE, HZ/10);
+                       congestion_wait(BLK_RW_ASYNC, HZ/10);
                }
 
        }

diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 7687879253b929e996dce647dbacab55ef91ff48..81627ebcd313fcd5ee3545a0996d2802fab36085 100644 (file)
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -575,7 +575,7 @@ static void balance_dirty_pages(struct address_space *mapping)
                if (pages_written >= write_chunk)
                        break;          /* We've done our duty */
 
-               congestion_wait(WRITE, HZ/10);
+               congestion_wait(BLK_RW_ASYNC, HZ/10);
        }
 
        if (bdi_nr_reclaimable + bdi_nr_writeback < bdi_thresh &&
@@ -669,7 +669,7 @@ void throttle_vm_writeout(gfp_t gfp_mask)
                 if (global_page_state(NR_UNSTABLE_NFS) +
                        global_page_state(NR_WRITEBACK) <= dirty_thresh)
                                break;
-                congestion_wait(WRITE, HZ/10);
+                congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                /*
                 * The caller might hold locks which can prevent IO completion
@@ -715,7 +715,7 @@ static void background_writeout(unsigned long _min_pages)
                if (wbc.nr_to_write > 0 || wbc.pages_skipped > 0) {
                        /* Wrote less than expected */
                        if (wbc.encountered_congestion || wbc.more_io)
-                               congestion_wait(WRITE, HZ/10);
+                               congestion_wait(BLK_RW_ASYNC, HZ/10);
                        else
                                break;
                }
@@ -787,7 +787,7 @@ static void wb_kupdate(unsigned long arg)
                writeback_inodes(&wbc);
                if (wbc.nr_to_write > 0) {
                        if (wbc.encountered_congestion || wbc.more_io)
-                               congestion_wait(WRITE, HZ/10);
+                               congestion_wait(BLK_RW_ASYNC, HZ/10);
                        else
                                break;  /* All the old data is written */
                }

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index ad7cd1c56b07c330d611a067a0749f9cabd440b0..a35eeab2724cc3e392d664bc24ed4fe2d9a46630 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1666,7 +1666,7 @@ __alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
                        preferred_zone, migratetype);
 
                if (!page && gfp_mask & __GFP_NOFAIL)
-                       congestion_wait(WRITE, HZ/50);
+                       congestion_wait(BLK_RW_ASYNC, HZ/50);
        } while (!page && (gfp_mask & __GFP_NOFAIL));
 
        return page;
@@ -1831,7 +1831,7 @@ rebalance:
        pages_reclaimed += did_some_progress;
        if (should_alloc_retry(gfp_mask, order, pages_reclaimed)) {
                /* Wait for some write requests to complete then retry */
-               congestion_wait(WRITE, HZ/50);
+               congestion_wait(BLK_RW_ASYNC, HZ/50);
                goto rebalance;
        }
 

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 54155268dfcae49634dc016bb168845600a866a0..dea7abd310980daea1fa6c5a0c850a972fe37a21 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1104,7 +1104,7 @@ static unsigned long shrink_inactive_list(unsigned long max_scan,
                 */
                if (nr_freed < nr_taken && !current_is_kswapd() &&
                    lumpy_reclaim) {
-                       congestion_wait(WRITE, HZ/10);
+                       congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                        /*
                         * The attempt at page out may have made some
@@ -1721,7 +1721,7 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
 
                /* Take a nap, wait for some writeback to complete */
                if (sc->nr_scanned && priority < DEF_PRIORITY - 2)
-                       congestion_wait(WRITE, HZ/10);
+                       congestion_wait(BLK_RW_ASYNC, HZ/10);
        }
        /* top priority shrink_zones still had more to do? don't OOM, then */
        if (!sc->all_unreclaimable && scanning_global_lru(sc))
@@ -1960,7 +1960,7 @@ loop_again:
                 * another pass across the zones.
                 */
                if (total_scanned && priority < DEF_PRIORITY - 2)
-                       congestion_wait(WRITE, HZ/10);
+                       congestion_wait(BLK_RW_ASYNC, HZ/10);
 
                /*
                 * We do this so kswapd doesn't build up large priorities for
@@ -2233,7 +2233,7 @@ unsigned long shrink_all_memory(unsigned long nr_pages)
                                goto out;
 
                        if (sc.nr_scanned && prio < DEF_PRIORITY - 2)
-                               congestion_wait(WRITE, HZ / 10);
+                               congestion_wait(BLK_RW_ASYNC, HZ / 10);
                }
        }