vhost: log dirty page correctly

[mirror_ubuntu-bionic-kernel.git] / drivers / vhost / net.c

diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index c7bdeb6556469efb93e2a6a7e742da3a37ad7e69..4a7a0d77dfba6b2ae7fc05dca373935fce250cac 100644 (file)
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -373,13 +373,10 @@ static inline unsigned long busy_clock(void)
        return local_clock() >> 10;
 }
 
-static bool vhost_can_busy_poll(struct vhost_dev *dev,
-                               unsigned long endtime)
+static bool vhost_can_busy_poll(unsigned long endtime)
 {
-       return likely(!need_resched()) &&
-              likely(!time_after(busy_clock(), endtime)) &&
-              likely(!signal_pending(current)) &&
-              !vhost_has_work(dev);
+       return likely(!need_resched() && !time_after(busy_clock(), endtime) &&
+                     !signal_pending(current));
 }
 
 static void vhost_net_disable_vq(struct vhost_net *n,
@@ -411,7 +408,8 @@ static int vhost_net_enable_vq(struct vhost_net *n,
 static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
                                    struct vhost_virtqueue *vq,
                                    struct iovec iov[], unsigned int iov_size,
-                                   unsigned int *out_num, unsigned int *in_num)
+                                   unsigned int *out_num, unsigned int *in_num,
+                                   bool *busyloop_intr)
 {
        unsigned long uninitialized_var(endtime);
        int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
@@ -420,9 +418,15 @@ static int vhost_net_tx_get_vq_desc(struct vhost_net *net,
        if (r == vq->num && vq->busyloop_timeout) {
                preempt_disable();
                endtime = busy_clock() + vq->busyloop_timeout;
-               while (vhost_can_busy_poll(vq->dev, endtime) &&
-                      vhost_vq_avail_empty(vq->dev, vq))
+               while (vhost_can_busy_poll(endtime)) {
+                       if (vhost_has_work(vq->dev)) {
+                               *busyloop_intr = true;
+                               break;
+                       }
+                       if (!vhost_vq_avail_empty(vq->dev, vq))
+                               break;
                        cpu_relax();
+               }
                preempt_enable();
                r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),
                                      out_num, in_num, NULL, NULL);
@@ -477,20 +481,24 @@ static void handle_tx(struct vhost_net *net)
        zcopy = nvq->ubufs;
 
        for (;;) {
+               bool busyloop_intr;
+
                /* Release DMAs done buffers first */
                if (zcopy)
                        vhost_zerocopy_signal_used(net, vq);
 
-
+               busyloop_intr = false;
                head = vhost_net_tx_get_vq_desc(net, vq, vq->iov,
                                                ARRAY_SIZE(vq->iov),
-                                               &out, &in);
+                                               &out, &in, &busyloop_intr);
                /* On error, stop handling until the next kick. */
                if (unlikely(head < 0))
                        break;
                /* Nothing new?  Wait for eventfd to tell us they refilled. */
                if (head == vq->num) {
-                       if (unlikely(vhost_enable_notify(&net->dev, vq))) {
+                       if (unlikely(busyloop_intr)) {
+                               vhost_poll_queue(&vq->poll);
+                       } else if (unlikely(vhost_enable_notify(&net->dev, vq))) {
                                vhost_disable_notify(&net->dev, vq);
                                continue;
                        }
@@ -618,13 +626,14 @@ static int vhost_net_rx_peek_head_len(struct vhost_net *net, struct sock *sk)
 
        if (!len && vq->busyloop_timeout) {
                /* Both tx vq and rx socket were polled here */
-               mutex_lock(&vq->mutex);
+               mutex_lock_nested(&vq->mutex, 1);
                vhost_disable_notify(&net->dev, vq);
 
                preempt_disable();
                endtime = busy_clock() + vq->busyloop_timeout;
 
-               while (vhost_can_busy_poll(&net->dev, endtime) &&
+               while (vhost_can_busy_poll(endtime) &&
+                      !vhost_has_work(&net->dev) &&
                       !sk_has_rx_data(sk) &&
                       vhost_vq_avail_empty(&net->dev, vq))
                        cpu_relax();
@@ -751,7 +760,7 @@ static void handle_rx(struct vhost_net *net)
        struct iov_iter fixup;
        __virtio16 num_buffers;
 
-       mutex_lock(&vq->mutex);
+       mutex_lock_nested(&vq->mutex, 0);
        sock = vq->private_data;
        if (!sock)
                goto out;
@@ -847,7 +856,8 @@ static void handle_rx(struct vhost_net *net)
                vhost_add_used_and_signal_n(&net->dev, vq, vq->heads,
                                            headcount);
                if (unlikely(vq_log))
-                       vhost_log_write(vq, vq_log, log, vhost_len);
+                       vhost_log_write(vq, vq_log, log, vhost_len,
+                                       vq->iov, in);
                total_len += vhost_len;
                if (unlikely(total_len >= VHOST_NET_WEIGHT)) {
                        vhost_poll_queue(&vq->poll);
@@ -1182,7 +1192,8 @@ err_used:
        if (ubufs)
                vhost_net_ubuf_put_wait_and_free(ubufs);
 err_ubufs:
-       sockfd_put(sock);
+       if (sock)
+               sockfd_put(sock);
 err_vq:
        mutex_unlock(&vq->mutex);
 err:
@@ -1208,6 +1219,7 @@ static long vhost_net_reset_owner(struct vhost_net *n)
        }
        vhost_net_stop(n, &tx_sock, &rx_sock);
        vhost_net_flush(n);
+       vhost_dev_stop(&n->dev);
        vhost_dev_reset_owner(&n->dev, umem);
        vhost_net_vq_reset(n);
 done: