[mirror_ubuntu-jammy-kernel.git] / drivers / net / ethernet / google / gve / gve_rx.c

// SPDX-License-Identifier: (GPL-2.0 OR MIT)
/* Google virtual Ethernet (gve) driver
 *
 * Copyright (C) 2015-2021 Google, Inc.
 */

#include "gve.h"
#include "gve_adminq.h"
#include "gve_utils.h"
#include <linux/etherdevice.h>

static void gve_rx_free_buffer(struct device *dev,
			       struct gve_rx_slot_page_info *page_info,
			       union gve_rx_data_slot *data_slot)
{
	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
				      GVE_DATA_SLOT_ADDR_PAGE_MASK);

	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
}

static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
{
	if (rx->data.raw_addressing) {
		u32 slots = rx->mask + 1;
		int i;

		for (i = 0; i < slots; i++)
			gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
					   &rx->data.data_ring[i]);
	} else {
		gve_unassign_qpl(priv, rx->data.qpl->id);
		rx->data.qpl = NULL;
	}
	kvfree(rx->data.page_info);
	rx->data.page_info = NULL;
}

static void gve_rx_free_ring(struct gve_priv *priv, int idx)
{
	struct gve_rx_ring *rx = &priv->rx[idx];
	struct device *dev = &priv->pdev->dev;
	u32 slots = rx->mask + 1;
	size_t bytes;

	gve_rx_remove_from_block(priv, idx);

	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
	rx->desc.desc_ring = NULL;

	dma_free_coherent(dev, sizeof(*rx->q_resources),
			  rx->q_resources, rx->q_resources_bus);
	rx->q_resources = NULL;

	gve_rx_unfill_pages(priv, rx);

	bytes = sizeof(*rx->data.data_ring) * slots;
	dma_free_coherent(dev, bytes, rx->data.data_ring,
			  rx->data.data_bus);
	rx->data.data_ring = NULL;
	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
}

static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
			     dma_addr_t addr, struct page *page, __be64 *slot_addr)
{
	page_info->page = page;
	page_info->page_offset = 0;
	page_info->page_address = page_address(page);
	*slot_addr = cpu_to_be64(addr);
}

static int gve_rx_alloc_buffer(struct gve_priv *priv, struct device *dev,
			       struct gve_rx_slot_page_info *page_info,
			       union gve_rx_data_slot *data_slot)
{
	struct page *page;
	dma_addr_t dma;
	int err;

	err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE);
	if (err)
		return err;

	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
	return 0;
}

static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
{
	struct gve_priv *priv = rx->gve;
	u32 slots;
	int err;
	int i;

	/* Allocate one page per Rx queue slot. Each page is split into two
	 * packet buffers, when possible we "page flip" between the two.
	 */
	slots = rx->mask + 1;

	rx->data.page_info = kvzalloc(slots *
				      sizeof(*rx->data.page_info), GFP_KERNEL);
	if (!rx->data.page_info)
		return -ENOMEM;

	if (!rx->data.raw_addressing) {
		rx->data.qpl = gve_assign_rx_qpl(priv);
		if (!rx->data.qpl) {
			kvfree(rx->data.page_info);
			rx->data.page_info = NULL;
			return -ENOMEM;
		}
	}
	for (i = 0; i < slots; i++) {
		if (!rx->data.raw_addressing) {
			struct page *page = rx->data.qpl->pages[i];
			dma_addr_t addr = i * PAGE_SIZE;

			gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
					    &rx->data.data_ring[i].qpl_offset);
			continue;
		}
		err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
					  &rx->data.data_ring[i]);
		if (err)
			goto alloc_err;
	}

	return slots;
alloc_err:
	while (i--)
		gve_rx_free_buffer(&priv->pdev->dev,
				   &rx->data.page_info[i],
				   &rx->data.data_ring[i]);
	return err;
}

static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
{
	struct gve_rx_ring *rx = &priv->rx[idx];
	struct device *hdev = &priv->pdev->dev;
	u32 slots, npages;
	int filled_pages;
	size_t bytes;
	int err;

	netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
	/* Make sure everything is zeroed to start with */
	memset(rx, 0, sizeof(*rx));

	rx->gve = priv;
	rx->q_num = idx;

	slots = priv->rx_data_slot_cnt;
	rx->mask = slots - 1;
	rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;

	/* alloc rx data ring */
	bytes = sizeof(*rx->data.data_ring) * slots;
	rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
						&rx->data.data_bus,
						GFP_KERNEL);
	if (!rx->data.data_ring)
		return -ENOMEM;
	filled_pages = gve_prefill_rx_pages(rx);
	if (filled_pages < 0) {
		err = -ENOMEM;
		goto abort_with_slots;
	}
	rx->fill_cnt = filled_pages;
	/* Ensure data ring slots (packet buffers) are visible. */
	dma_wmb();

	/* Alloc gve_queue_resources */
	rx->q_resources =
		dma_alloc_coherent(hdev,
				   sizeof(*rx->q_resources),
				   &rx->q_resources_bus,
				   GFP_KERNEL);
	if (!rx->q_resources) {
		err = -ENOMEM;
		goto abort_filled;
	}
	netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
		  (unsigned long)rx->data.data_bus);

	/* alloc rx desc ring */
	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	npages = bytes / PAGE_SIZE;
	if (npages * PAGE_SIZE != bytes) {
		err = -EIO;
		goto abort_with_q_resources;
	}

	rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
						GFP_KERNEL);
	if (!rx->desc.desc_ring) {
		err = -ENOMEM;
		goto abort_with_q_resources;
	}
	rx->cnt = 0;
	rx->db_threshold = priv->rx_desc_cnt / 2;
	rx->desc.seqno = 1;
	gve_rx_add_to_block(priv, idx);

	return 0;

abort_with_q_resources:
	dma_free_coherent(hdev, sizeof(*rx->q_resources),
			  rx->q_resources, rx->q_resources_bus);
	rx->q_resources = NULL;
abort_filled:
	gve_rx_unfill_pages(priv, rx);
abort_with_slots:
	bytes = sizeof(*rx->data.data_ring) * slots;
	dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
	rx->data.data_ring = NULL;

	return err;
}

int gve_rx_alloc_rings(struct gve_priv *priv)
{
	int err = 0;
	int i;

	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
		err = gve_rx_alloc_ring(priv, i);
		if (err) {
			netif_err(priv, drv, priv->dev,
				  "Failed to alloc rx ring=%d: err=%d\n",
				  i, err);
			break;
		}
	}
	/* Unallocate if there was an error */
	if (err) {
		int j;

		for (j = 0; j < i; j++)
			gve_rx_free_ring(priv, j);
	}
	return err;
}

void gve_rx_free_rings_gqi(struct gve_priv *priv)
{
	int i;

	for (i = 0; i < priv->rx_cfg.num_queues; i++)
		gve_rx_free_ring(priv, i);
}

void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx)
{
	u32 db_idx = be32_to_cpu(rx->q_resources->db_index);

	iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
}

static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
{
	if (likely(pkt_flags & (GVE_RXF_TCP | GVE_RXF_UDP)))
		return PKT_HASH_TYPE_L4;
	if (pkt_flags & (GVE_RXF_IPV4 | GVE_RXF_IPV6))
		return PKT_HASH_TYPE_L3;
	return PKT_HASH_TYPE_L2;
}

static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
					struct gve_rx_slot_page_info *page_info,
					u16 len)
{
	struct sk_buff *skb = napi_get_frags(napi);

	if (unlikely(!skb))
		return NULL;

	skb_add_rx_frag(skb, 0, page_info->page,
			page_info->page_offset +
			GVE_RX_PAD, len, PAGE_SIZE / 2);

	return skb;
}

static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
{
	const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);

	/* "flip" to other packet buffer on this page */
	page_info->page_offset ^= PAGE_SIZE / 2;
	*(slot_addr) ^= offset;
}

static bool gve_rx_can_flip_buffers(struct net_device *netdev)
{
	return PAGE_SIZE == 4096
		? netdev->mtu + GVE_RX_PAD + ETH_HLEN <= PAGE_SIZE / 2 : false;
}

static int gve_rx_can_recycle_buffer(struct page *page)
{
	int pagecount = page_count(page);

	/* This page is not being used by any SKBs - reuse */
	if (pagecount == 1)
		return 1;
	/* This page is still being used by an SKB - we can't reuse */
	else if (pagecount >= 2)
		return 0;
	WARN(pagecount < 1, "Pagecount should never be < 1");
	return -1;
}

static struct sk_buff *
gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
		      struct gve_rx_slot_page_info *page_info, u16 len,
		      struct napi_struct *napi,
		      union gve_rx_data_slot *data_slot)
{
	struct sk_buff *skb;

	skb = gve_rx_add_frags(napi, page_info, len);
	if (!skb)
		return NULL;

	/* Optimistically stop the kernel from freeing the page by increasing
	 * the page bias. We will check the refcount in refill to determine if
	 * we need to alloc a new page.
	 */
	get_page(page_info->page);

	return skb;
}

static struct sk_buff *
gve_rx_qpl(struct device *dev, struct net_device *netdev,
	   struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
	   u16 len, struct napi_struct *napi,
	   union gve_rx_data_slot *data_slot)
{
	struct sk_buff *skb;

	/* if raw_addressing mode is not enabled gvnic can only receive into
	 * registered segments. If the buffer can't be recycled, our only
	 * choice is to copy the data out of it so that we can return it to the
	 * device.
	 */
	if (page_info->can_flip) {
		skb = gve_rx_add_frags(napi, page_info, len);
		/* No point in recycling if we didn't get the skb */
		if (skb) {
			/* Make sure that the page isn't freed. */
			get_page(page_info->page);
			gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
		}
	} else {
		skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD);
		if (skb) {
			u64_stats_update_begin(&rx->statss);
			rx->rx_copied_pkt++;
			u64_stats_update_end(&rx->statss);
		}
	}
	return skb;
}

static bool gve_rx(struct gve_rx_ring *rx, struct gve_rx_desc *rx_desc,
		   netdev_features_t feat, u32 idx)
{
	struct gve_rx_slot_page_info *page_info;
	struct gve_priv *priv = rx->gve;
	struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
	struct net_device *dev = priv->dev;
	union gve_rx_data_slot *data_slot;
	struct sk_buff *skb = NULL;
	dma_addr_t page_bus;
	u16 len;

	/* drop this packet */
	if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR)) {
		u64_stats_update_begin(&rx->statss);
		rx->rx_desc_err_dropped_pkt++;
		u64_stats_update_end(&rx->statss);
		return false;
	}

	len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
	page_info = &rx->data.page_info[idx];

	data_slot = &rx->data.data_ring[idx];
	page_bus = (rx->data.raw_addressing) ?
			be64_to_cpu(data_slot->addr) & GVE_DATA_SLOT_ADDR_PAGE_MASK :
			rx->data.qpl->page_buses[idx];
	dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
				PAGE_SIZE, DMA_FROM_DEVICE);

	if (len <= priv->rx_copybreak) {
		/* Just copy small packets */
		skb = gve_rx_copy(dev, napi, page_info, len, GVE_RX_PAD);
		u64_stats_update_begin(&rx->statss);
		rx->rx_copied_pkt++;
		rx->rx_copybreak_pkt++;
		u64_stats_update_end(&rx->statss);
	} else {
		u8 can_flip = gve_rx_can_flip_buffers(dev);
		int recycle = 0;

		if (can_flip) {
			recycle = gve_rx_can_recycle_buffer(page_info->page);
			if (recycle < 0) {
				if (!rx->data.raw_addressing)
					gve_schedule_reset(priv);
				return false;
			}
		}

		page_info->can_flip = can_flip && recycle;
		if (rx->data.raw_addressing) {
			skb = gve_rx_raw_addressing(&priv->pdev->dev, dev,
						    page_info, len, napi,
						    data_slot);
		} else {
			skb = gve_rx_qpl(&priv->pdev->dev, dev, rx,
					 page_info, len, napi, data_slot);
		}
	}

	if (!skb) {
		u64_stats_update_begin(&rx->statss);
		rx->rx_skb_alloc_fail++;
		u64_stats_update_end(&rx->statss);
		return false;
	}

	if (likely(feat & NETIF_F_RXCSUM)) {
		/* NIC passes up the partial sum */
		if (rx_desc->csum)
			skb->ip_summed = CHECKSUM_COMPLETE;
		else
			skb->ip_summed = CHECKSUM_NONE;
		skb->csum = csum_unfold(rx_desc->csum);
	}

	/* parse flags & pass relevant info up */
	if (likely(feat & NETIF_F_RXHASH) &&
	    gve_needs_rss(rx_desc->flags_seq))
		skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
			     gve_rss_type(rx_desc->flags_seq));

	if (skb_is_nonlinear(skb))
		napi_gro_frags(napi);
	else
		napi_gro_receive(napi, skb);
	return true;
}

static bool gve_rx_work_pending(struct gve_rx_ring *rx)
{
	struct gve_rx_desc *desc;
	__be16 flags_seq;
	u32 next_idx;

	next_idx = rx->cnt & rx->mask;
	desc = rx->desc.desc_ring + next_idx;

	flags_seq = desc->flags_seq;
	/* Make sure we have synchronized the seq no with the device */
	smp_rmb();

	return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
}

static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
{
	int refill_target = rx->mask + 1;
	u32 fill_cnt = rx->fill_cnt;

	while (fill_cnt - rx->cnt < refill_target) {
		struct gve_rx_slot_page_info *page_info;
		u32 idx = fill_cnt & rx->mask;

		page_info = &rx->data.page_info[idx];
		if (page_info->can_flip) {
			/* The other half of the page is free because it was
			 * free when we processed the descriptor. Flip to it.
			 */
			union gve_rx_data_slot *data_slot =
						&rx->data.data_ring[idx];

			gve_rx_flip_buff(page_info, &data_slot->addr);
			page_info->can_flip = 0;
		} else {
			/* It is possible that the networking stack has already
			 * finished processing all outstanding packets in the buffer
			 * and it can be reused.
			 * Flipping is unnecessary here - if the networking stack still
			 * owns half the page it is impossible to tell which half. Either
			 * the whole page is free or it needs to be replaced.
			 */
			int recycle = gve_rx_can_recycle_buffer(page_info->page);

			if (recycle < 0) {
				if (!rx->data.raw_addressing)
					gve_schedule_reset(priv);
				return false;
			}
			if (!recycle) {
				/* We can't reuse the buffer - alloc a new one*/
				union gve_rx_data_slot *data_slot =
						&rx->data.data_ring[idx];
				struct device *dev = &priv->pdev->dev;

				gve_rx_free_buffer(dev, page_info, data_slot);
				page_info->page = NULL;
				if (gve_rx_alloc_buffer(priv, dev, page_info, data_slot))
					break;
			}
		}
		fill_cnt++;
	}
	rx->fill_cnt = fill_cnt;
	return true;
}

bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
		       netdev_features_t feat)
{
	struct gve_priv *priv = rx->gve;
	u32 work_done = 0, packets = 0;
	struct gve_rx_desc *desc;
	u32 cnt = rx->cnt;
	u32 idx = cnt & rx->mask;
	u64 bytes = 0;

	desc = rx->desc.desc_ring + idx;
	while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
	       work_done < budget) {
		bool dropped;

		netif_info(priv, rx_status, priv->dev,
			   "[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
			   rx->q_num, idx, desc, desc->flags_seq);
		netif_info(priv, rx_status, priv->dev,
			   "[%d] seqno=%d rx->desc.seqno=%d\n",
			   rx->q_num, GVE_SEQNO(desc->flags_seq),
			   rx->desc.seqno);
		dropped = !gve_rx(rx, desc, feat, idx);
		if (!dropped) {
			bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
			packets++;
		}
		cnt++;
		idx = cnt & rx->mask;
		desc = rx->desc.desc_ring + idx;
		rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
		work_done++;
	}

	if (!work_done && rx->fill_cnt - cnt > rx->db_threshold)
		return false;

	u64_stats_update_begin(&rx->statss);
	rx->rpackets += packets;
	rx->rbytes += bytes;
	u64_stats_update_end(&rx->statss);
	rx->cnt = cnt;

	/* restock ring slots */
	if (!rx->data.raw_addressing) {
		/* In QPL mode buffs are refilled as the desc are processed */
		rx->fill_cnt += work_done;
	} else if (rx->fill_cnt - cnt <= rx->db_threshold) {
		/* In raw addressing mode buffs are only refilled if the avail
		 * falls below a threshold.
		 */
		if (!gve_rx_refill_buffers(priv, rx))
			return false;

		/* If we were not able to completely refill buffers, we'll want
		 * to schedule this queue for work again to refill buffers.
		 */
		if (rx->fill_cnt - cnt <= rx->db_threshold) {
			gve_rx_write_doorbell(priv, rx);
			return true;
		}
	}

	gve_rx_write_doorbell(priv, rx);
	return gve_rx_work_pending(rx);
}

bool gve_rx_poll(struct gve_notify_block *block, int budget)
{
	struct gve_rx_ring *rx = block->rx;
	netdev_features_t feat;
	bool repoll = false;

	feat = block->napi.dev->features;

	/* If budget is 0, do all the work */
	if (budget == 0)
		budget = INT_MAX;

	if (budget > 0)
		repoll |= gve_clean_rx_done(rx, budget, feat);
	else
		repoll |= gve_rx_work_pending(rx);
	return repoll;
}
Commit	Line	Data
f5cedc84 CS	1	// SPDX-License-Identifier: (GPL-2.0 OR MIT)
	2	/* Google virtual Ethernet (gve) driver
	3	*
dbdaa675	4	* Copyright (C) 2015-2021 Google, Inc.
f5cedc84 CS	5	*/
	6
	7	#include "gve.h"
	8	#include "gve_adminq.h"
dbdaa675	9	#include "gve_utils.h"
f5cedc84 CS	10	#include <linux/etherdevice.h>
f5cedc84 CS	11
ede3fcf5 CS	12	static void gve_rx_free_buffer(struct device *dev,
	13	struct gve_rx_slot_page_info *page_info,
	14	union gve_rx_data_slot *data_slot)
	15	{
	16	dma_addr_t dma = (dma_addr_t)(be64_to_cpu(data_slot->addr) &
	17	GVE_DATA_SLOT_ADDR_PAGE_MASK);
	18
	19	gve_free_page(dev, page_info->page, dma, DMA_FROM_DEVICE);
	20	}
	21
	22	static void gve_rx_unfill_pages(struct gve_priv priv, struct gve_rx_ring rx)
	23	{
	24	if (rx->data.raw_addressing) {
	25	u32 slots = rx->mask + 1;
	26	int i;
	27
	28	for (i = 0; i < slots; i++)
	29	gve_rx_free_buffer(&priv->pdev->dev, &rx->data.page_info[i],
	30	&rx->data.data_ring[i]);
	31	} else {
	32	gve_unassign_qpl(priv, rx->data.qpl->id);
	33	rx->data.qpl = NULL;
	34	}
	35	kvfree(rx->data.page_info);
	36	rx->data.page_info = NULL;
	37	}
	38
f5cedc84 CS	39	static void gve_rx_free_ring(struct gve_priv *priv, int idx)
	40	{
	41	struct gve_rx_ring *rx = &priv->rx[idx];
	42	struct device *dev = &priv->pdev->dev;
ede3fcf5	43	u32 slots = rx->mask + 1;
f5cedc84	44	size_t bytes;
f5cedc84 CS	45
	46	gve_rx_remove_from_block(priv, idx);
	47
	48	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	49	dma_free_coherent(dev, bytes, rx->desc.desc_ring, rx->desc.bus);
	50	rx->desc.desc_ring = NULL;
	51
	52	dma_free_coherent(dev, sizeof(*rx->q_resources),
	53	rx->q_resources, rx->q_resources_bus);
	54	rx->q_resources = NULL;
	55
ede3fcf5	56	gve_rx_unfill_pages(priv, rx);
f5cedc84	57
f5cedc84 CS	58	bytes = sizeof(rx->data.data_ring) slots;
	59	dma_free_coherent(dev, bytes, rx->data.data_ring,
	60	rx->data.data_bus);
	61	rx->data.data_ring = NULL;
	62	netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
	63	}
	64
	65	static void gve_setup_rx_buffer(struct gve_rx_slot_page_info *page_info,
ede3fcf5	66	dma_addr_t addr, struct page page, __be64 slot_addr)
f5cedc84 CS	67	{
	68	page_info->page = page;
	69	page_info->page_offset = 0;
	70	page_info->page_address = page_address(page);
ede3fcf5 CS	71	*slot_addr = cpu_to_be64(addr);
	72	}
	73
	74	static int gve_rx_alloc_buffer(struct gve_priv priv, struct device dev,
	75	struct gve_rx_slot_page_info *page_info,
	76	union gve_rx_data_slot *data_slot)
	77	{
	78	struct page *page;
	79	dma_addr_t dma;
	80	int err;
	81
	82	err = gve_alloc_page(priv, dev, &page, &dma, DMA_FROM_DEVICE);
	83	if (err)
	84	return err;
	85
	86	gve_setup_rx_buffer(page_info, dma, page, &data_slot->addr);
	87	return 0;
f5cedc84 CS	88	}
	89
	90	static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
	91	{
	92	struct gve_priv *priv = rx->gve;
	93	u32 slots;
ede3fcf5	94	int err;
f5cedc84 CS	95	int i;
	96
	97	/* Allocate one page per Rx queue slot. Each page is split into two
	98	* packet buffers, when possible we "page flip" between the two.
	99	*/
438b43bd	100	slots = rx->mask + 1;
f5cedc84 CS	101
	102	rx->data.page_info = kvzalloc(slots *
	103	sizeof(*rx->data.page_info), GFP_KERNEL);
	104	if (!rx->data.page_info)
	105	return -ENOMEM;
	106
d4b111fd	107	if (!rx->data.raw_addressing) {
ede3fcf5	108	rx->data.qpl = gve_assign_rx_qpl(priv);
d4b111fd CS	109	if (!rx->data.qpl) {
	110	kvfree(rx->data.page_info);
	111	rx->data.page_info = NULL;
	112	return -ENOMEM;
	113	}
	114	}
f5cedc84	115	for (i = 0; i < slots; i++) {
ede3fcf5 CS	116	if (!rx->data.raw_addressing) {
	117	struct page *page = rx->data.qpl->pages[i];
	118	dma_addr_t addr = i * PAGE_SIZE;
f5cedc84	119
ede3fcf5 CS	120	gve_setup_rx_buffer(&rx->data.page_info[i], addr, page,
	121	&rx->data.data_ring[i].qpl_offset);
	122	continue;
	123	}
	124	err = gve_rx_alloc_buffer(priv, &priv->pdev->dev, &rx->data.page_info[i],
	125	&rx->data.data_ring[i]);
	126	if (err)
	127	goto alloc_err;
f5cedc84 CS	128	}
	129
	130	return slots;
ede3fcf5 CS	131	alloc_err:
	132	while (i--)
	133	gve_rx_free_buffer(&priv->pdev->dev,
	134	&rx->data.page_info[i],
	135	&rx->data.data_ring[i]);
	136	return err;
f5cedc84 CS	137	}
f5cedc84 CS	138
f5cedc84 CS	139	static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
	140	{
	141	struct gve_rx_ring *rx = &priv->rx[idx];
	142	struct device *hdev = &priv->pdev->dev;
	143	u32 slots, npages;
	144	int filled_pages;
	145	size_t bytes;
	146	int err;
	147
	148	netif_dbg(priv, drv, priv->dev, "allocating rx ring\n");
	149	/* Make sure everything is zeroed to start with */
	150	memset(rx, 0, sizeof(*rx));
	151
	152	rx->gve = priv;
	153	rx->q_num = idx;
	154
ede3fcf5	155	slots = priv->rx_data_slot_cnt;
438b43bd	156	rx->mask = slots - 1;
a5886ef4	157	rx->data.raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT;
f5cedc84 CS	158
	159	/* alloc rx data ring */
	160	bytes = sizeof(rx->data.data_ring) slots;
	161	rx->data.data_ring = dma_alloc_coherent(hdev, bytes,
	162	&rx->data.data_bus,
	163	GFP_KERNEL);
	164	if (!rx->data.data_ring)
	165	return -ENOMEM;
	166	filled_pages = gve_prefill_rx_pages(rx);
	167	if (filled_pages < 0) {
	168	err = -ENOMEM;
	169	goto abort_with_slots;
	170	}
438b43bd	171	rx->fill_cnt = filled_pages;
f5cedc84 CS	172	/* Ensure data ring slots (packet buffers) are visible. */
	173	dma_wmb();
	174
	175	/* Alloc gve_queue_resources */
	176	rx->q_resources =
	177	dma_alloc_coherent(hdev,
	178	sizeof(*rx->q_resources),
	179	&rx->q_resources_bus,
	180	GFP_KERNEL);
	181	if (!rx->q_resources) {
	182	err = -ENOMEM;
	183	goto abort_filled;
	184	}
	185	netif_dbg(priv, drv, priv->dev, "rx[%d]->data.data_bus=%lx\n", idx,
	186	(unsigned long)rx->data.data_bus);
	187
	188	/* alloc rx desc ring */
	189	bytes = sizeof(struct gve_rx_desc) * priv->rx_desc_cnt;
	190	npages = bytes / PAGE_SIZE;
	191	if (npages * PAGE_SIZE != bytes) {
	192	err = -EIO;
	193	goto abort_with_q_resources;
	194	}
	195
	196	rx->desc.desc_ring = dma_alloc_coherent(hdev, bytes, &rx->desc.bus,
	197	GFP_KERNEL);
	198	if (!rx->desc.desc_ring) {
	199	err = -ENOMEM;
	200	goto abort_with_q_resources;
	201	}
438b43bd	202	rx->cnt = 0;
ede3fcf5	203	rx->db_threshold = priv->rx_desc_cnt / 2;
f5cedc84 CS	204	rx->desc.seqno = 1;
	205	gve_rx_add_to_block(priv, idx);
	206
	207	return 0;
	208
	209	abort_with_q_resources:
	210	dma_free_coherent(hdev, sizeof(*rx->q_resources),
	211	rx->q_resources, rx->q_resources_bus);
	212	rx->q_resources = NULL;
	213	abort_filled:
ede3fcf5	214	gve_rx_unfill_pages(priv, rx);
f5cedc84 CS	215	abort_with_slots:
	216	bytes = sizeof(rx->data.data_ring) slots;
	217	dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
	218	rx->data.data_ring = NULL;
	219
	220	return err;
	221	}
	222
	223	int gve_rx_alloc_rings(struct gve_priv *priv)
	224	{
	225	int err = 0;
	226	int i;
	227
	228	for (i = 0; i < priv->rx_cfg.num_queues; i++) {
	229	err = gve_rx_alloc_ring(priv, i);
	230	if (err) {
	231	netif_err(priv, drv, priv->dev,
	232	"Failed to alloc rx ring=%d: err=%d\n",
	233	i, err);
	234	break;
	235	}
	236	}
	237	/* Unallocate if there was an error */
	238	if (err) {
	239	int j;
	240
	241	for (j = 0; j < i; j++)
	242	gve_rx_free_ring(priv, j);
	243	}
	244	return err;
	245	}
	246
9c1a59a2	247	void gve_rx_free_rings_gqi(struct gve_priv *priv)
f5cedc84 CS	248	{
	249	int i;
	250
	251	for (i = 0; i < priv->rx_cfg.num_queues; i++)
	252	gve_rx_free_ring(priv, i);
	253	}
	254
	255	void gve_rx_write_doorbell(struct gve_priv priv, struct gve_rx_ring rx)
	256	{
	257	u32 db_idx = be32_to_cpu(rx->q_resources->db_index);
	258
438b43bd	259	iowrite32be(rx->fill_cnt, &priv->db_bar2[db_idx]);
f5cedc84 CS	260	}
	261
	262	static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
	263	{
	264	if (likely(pkt_flags & (GVE_RXF_TCP \| GVE_RXF_UDP)))
	265	return PKT_HASH_TYPE_L4;
	266	if (pkt_flags & (GVE_RXF_IPV4 \| GVE_RXF_IPV6))
	267	return PKT_HASH_TYPE_L3;
	268	return PKT_HASH_TYPE_L2;
	269	}
	270
ede3fcf5	271	static struct sk_buff gve_rx_add_frags(struct napi_struct napi,
f5cedc84 CS	272	struct gve_rx_slot_page_info *page_info,
	273	u16 len)
	274	{
	275	struct sk_buff *skb = napi_get_frags(napi);
	276
	277	if (unlikely(!skb))
	278	return NULL;
	279
	280	skb_add_rx_frag(skb, 0, page_info->page,
920fb451	281	page_info->page_offset +
f5cedc84 CS	282	GVE_RX_PAD, len, PAGE_SIZE / 2);
	283
	284	return skb;
	285	}
	286
ede3fcf5	287	static void gve_rx_flip_buff(struct gve_rx_slot_page_info page_info, __be64 slot_addr)
f5cedc84	288	{
ede3fcf5	289	const __be64 offset = cpu_to_be64(PAGE_SIZE / 2);
f5cedc84	290
ede3fcf5	291	/* "flip" to other packet buffer on this page */
920fb451	292	page_info->page_offset ^= PAGE_SIZE / 2;
ede3fcf5	293	*(slot_addr) ^= offset;
f5cedc84 CS	294	}
f5cedc84 CS	295
02b0e0c1 DA	296	static bool gve_rx_can_flip_buffers(struct net_device *netdev)
	297	{
	298	return PAGE_SIZE == 4096
	299	? netdev->mtu + GVE_RX_PAD + ETH_HLEN <= PAGE_SIZE / 2 : false;
	300	}
	301
	302	static int gve_rx_can_recycle_buffer(struct page *page)
	303	{
	304	int pagecount = page_count(page);
	305
	306	/* This page is not being used by any SKBs - reuse */
	307	if (pagecount == 1)
	308	return 1;
	309	/* This page is still being used by an SKB - we can't reuse */
	310	else if (pagecount >= 2)
	311	return 0;
	312	WARN(pagecount < 1, "Pagecount should never be < 1");
	313	return -1;
	314	}
	315
	316	static struct sk_buff *
	317	gve_rx_raw_addressing(struct device dev, struct net_device netdev,
	318	struct gve_rx_slot_page_info *page_info, u16 len,
	319	struct napi_struct *napi,
	320	union gve_rx_data_slot *data_slot)
	321	{
	322	struct sk_buff *skb;
	323
	324	skb = gve_rx_add_frags(napi, page_info, len);
	325	if (!skb)
	326	return NULL;
	327
	328	/* Optimistically stop the kernel from freeing the page by increasing
	329	* the page bias. We will check the refcount in refill to determine if
	330	* we need to alloc a new page.
	331	*/
	332	get_page(page_info->page);
	333
	334	return skb;
	335	}
	336
	337	static struct sk_buff *
	338	gve_rx_qpl(struct device dev, struct net_device netdev,
	339	struct gve_rx_ring rx, struct gve_rx_slot_page_info page_info,
	340	u16 len, struct napi_struct *napi,
	341	union gve_rx_data_slot *data_slot)
	342	{
	343	struct sk_buff *skb;
	344
	345	/* if raw_addressing mode is not enabled gvnic can only receive into
	346	* registered segments. If the buffer can't be recycled, our only
	347	* choice is to copy the data out of it so that we can return it to the
	348	* device.
	349	*/
	350	if (page_info->can_flip) {
	351	skb = gve_rx_add_frags(napi, page_info, len);
	352	/* No point in recycling if we didn't get the skb */
	353	if (skb) {
	354	/* Make sure that the page isn't freed. */
	355	get_page(page_info->page);
	356	gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
	357	}
	358	} else {
35f9b2f4	359	skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD);
02b0e0c1 DA	360	if (skb) {
	361	u64_stats_update_begin(&rx->statss);
	362	rx->rx_copied_pkt++;
	363	u64_stats_update_end(&rx->statss);
	364	}
	365	}
	366	return skb;
	367	}
	368
f5cedc84	369	static bool gve_rx(struct gve_rx_ring rx, struct gve_rx_desc rx_desc,
438b43bd	370	netdev_features_t feat, u32 idx)
f5cedc84 CS	371	{
	372	struct gve_rx_slot_page_info *page_info;
	373	struct gve_priv *priv = rx->gve;
	374	struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
	375	struct net_device *dev = priv->dev;
ede3fcf5 CS	376	union gve_rx_data_slot *data_slot;
	377	struct sk_buff *skb = NULL;
	378	dma_addr_t page_bus;
f5cedc84	379	u16 len;
f5cedc84 CS	380
f5cedc84 CS	381	/* drop this packet */
433e274b KZ	382	if (unlikely(rx_desc->flags_seq & GVE_RXF_ERR)) {
	383	u64_stats_update_begin(&rx->statss);
	384	rx->rx_desc_err_dropped_pkt++;
	385	u64_stats_update_end(&rx->statss);
ede3fcf5	386	return false;
433e274b	387	}
f5cedc84 CS	388
f5cedc84 CS	389	len = be16_to_cpu(rx_desc->len) - GVE_RX_PAD;
f5cedc84 CS	390	page_info = &rx->data.page_info[idx];
f5cedc84 CS	391
ede3fcf5 CS	392	data_slot = &rx->data.data_ring[idx];
	393	page_bus = (rx->data.raw_addressing) ?
	394	be64_to_cpu(data_slot->addr) & GVE_DATA_SLOT_ADDR_PAGE_MASK :
	395	rx->data.qpl->page_buses[idx];
	396	dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
	397	PAGE_SIZE, DMA_FROM_DEVICE);
f5cedc84	398
02b0e0c1 DA	399	if (len <= priv->rx_copybreak) {
02b0e0c1 DA	400	/* Just copy small packets */
35f9b2f4	401	skb = gve_rx_copy(dev, napi, page_info, len, GVE_RX_PAD);
02b0e0c1 DA	402	u64_stats_update_begin(&rx->statss);
	403	rx->rx_copied_pkt++;
	404	rx->rx_copybreak_pkt++;
	405	u64_stats_update_end(&rx->statss);
	406	} else {
	407	u8 can_flip = gve_rx_can_flip_buffers(dev);
	408	int recycle = 0;
	409
	410	if (can_flip) {
	411	recycle = gve_rx_can_recycle_buffer(page_info->page);
	412	if (recycle < 0) {
	413	if (!rx->data.raw_addressing)
	414	gve_schedule_reset(priv);
ede3fcf5	415	return false;
433e274b	416	}
02b0e0c1 DA	417	}
	418
	419	page_info->can_flip = can_flip && recycle;
	420	if (rx->data.raw_addressing) {
	421	skb = gve_rx_raw_addressing(&priv->pdev->dev, dev,
	422	page_info, len, napi,
	423	data_slot);
0287f9ed	424	} else {
02b0e0c1 DA	425	skb = gve_rx_qpl(&priv->pdev->dev, dev, rx,
02b0e0c1 DA	426	page_info, len, napi, data_slot);
0287f9ed	427	}
f5cedc84	428	}
f5cedc84	429
433e274b KZ	430	if (!skb) {
	431	u64_stats_update_begin(&rx->statss);
	432	rx->rx_skb_alloc_fail++;
	433	u64_stats_update_end(&rx->statss);
ede3fcf5	434	return false;
433e274b	435	}
f5cedc84	436
f5cedc84 CS	437	if (likely(feat & NETIF_F_RXCSUM)) {
	438	/* NIC passes up the partial sum */
	439	if (rx_desc->csum)
	440	skb->ip_summed = CHECKSUM_COMPLETE;
	441	else
	442	skb->ip_summed = CHECKSUM_NONE;
	443	skb->csum = csum_unfold(rx_desc->csum);
	444	}
	445
	446	/* parse flags & pass relevant info up */
	447	if (likely(feat & NETIF_F_RXHASH) &&
	448	gve_needs_rss(rx_desc->flags_seq))
	449	skb_set_hash(skb, be32_to_cpu(rx_desc->rss_hash),
	450	gve_rss_type(rx_desc->flags_seq));
	451
	452	if (skb_is_nonlinear(skb))
	453	napi_gro_frags(napi);
	454	else
	455	napi_gro_receive(napi, skb);
	456	return true;
	457	}
	458
	459	static bool gve_rx_work_pending(struct gve_rx_ring *rx)
	460	{
	461	struct gve_rx_desc *desc;
	462	__be16 flags_seq;
	463	u32 next_idx;
	464
438b43bd	465	next_idx = rx->cnt & rx->mask;
f5cedc84 CS	466	desc = rx->desc.desc_ring + next_idx;
	467
	468	flags_seq = desc->flags_seq;
	469	/* Make sure we have synchronized the seq no with the device */
	470	smp_rmb();
	471
	472	return (GVE_SEQNO(flags_seq) == rx->desc.seqno);
	473	}
	474
ede3fcf5 CS	475	static bool gve_rx_refill_buffers(struct gve_priv priv, struct gve_rx_ring rx)
	476	{
	477	int refill_target = rx->mask + 1;
	478	u32 fill_cnt = rx->fill_cnt;
	479
	480	while (fill_cnt - rx->cnt < refill_target) {
	481	struct gve_rx_slot_page_info *page_info;
ede3fcf5 CS	482	u32 idx = fill_cnt & rx->mask;
	483
	484	page_info = &rx->data.page_info[idx];
02b0e0c1 DA	485	if (page_info->can_flip) {
	486	/* The other half of the page is free because it was
	487	* free when we processed the descriptor. Flip to it.
	488	*/
	489	union gve_rx_data_slot *data_slot =
	490	&rx->data.data_ring[idx];
	491
	492	gve_rx_flip_buff(page_info, &data_slot->addr);
	493	page_info->can_flip = 0;
	494	} else {
	495	/* It is possible that the networking stack has already
	496	* finished processing all outstanding packets in the buffer
	497	* and it can be reused.
	498	* Flipping is unnecessary here - if the networking stack still
	499	* owns half the page it is impossible to tell which half. Either
	500	* the whole page is free or it needs to be replaced.
	501	*/
	502	int recycle = gve_rx_can_recycle_buffer(page_info->page);
	503
	504	if (recycle < 0) {
	505	if (!rx->data.raw_addressing)
	506	gve_schedule_reset(priv);
	507	return false;
	508	}
	509	if (!recycle) {
	510	/* We can't reuse the buffer - alloc a new one*/
	511	union gve_rx_data_slot *data_slot =
	512	&rx->data.data_ring[idx];
	513	struct device *dev = &priv->pdev->dev;
	514
	515	gve_rx_free_buffer(dev, page_info, data_slot);
	516	page_info->page = NULL;
	517	if (gve_rx_alloc_buffer(priv, dev, page_info, data_slot))
	518	break;
	519	}
ede3fcf5 CS	520	}
	521	fill_cnt++;
	522	}
	523	rx->fill_cnt = fill_cnt;
	524	return true;
	525	}
	526
f5cedc84 CS	527	bool gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
	528	netdev_features_t feat)
	529	{
	530	struct gve_priv *priv = rx->gve;
ede3fcf5	531	u32 work_done = 0, packets = 0;
f5cedc84	532	struct gve_rx_desc *desc;
438b43bd CS	533	u32 cnt = rx->cnt;
438b43bd CS	534	u32 idx = cnt & rx->mask;
f5cedc84 CS	535	u64 bytes = 0;
	536
	537	desc = rx->desc.desc_ring + idx;
	538	while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
	539	work_done < budget) {
ede3fcf5 CS	540	bool dropped;
ede3fcf5 CS	541
f5cedc84 CS	542	netif_info(priv, rx_status, priv->dev,
	543	"[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
	544	rx->q_num, idx, desc, desc->flags_seq);
	545	netif_info(priv, rx_status, priv->dev,
	546	"[%d] seqno=%d rx->desc.seqno=%d\n",
	547	rx->q_num, GVE_SEQNO(desc->flags_seq),
	548	rx->desc.seqno);
ede3fcf5 CS	549	dropped = !gve_rx(rx, desc, feat, idx);
	550	if (!dropped) {
	551	bytes += be16_to_cpu(desc->len) - GVE_RX_PAD;
	552	packets++;
	553	}
f5cedc84	554	cnt++;
438b43bd	555	idx = cnt & rx->mask;
f5cedc84 CS	556	desc = rx->desc.desc_ring + idx;
	557	rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
	558	work_done++;
	559	}
	560
ede3fcf5	561	if (!work_done && rx->fill_cnt - cnt > rx->db_threshold)
f5cedc84 CS	562	return false;
	563
	564	u64_stats_update_begin(&rx->statss);
ede3fcf5	565	rx->rpackets += packets;
f5cedc84 CS	566	rx->rbytes += bytes;
f5cedc84 CS	567	u64_stats_update_end(&rx->statss);
438b43bd	568	rx->cnt = cnt;
ede3fcf5 CS	569
	570	/* restock ring slots */
	571	if (!rx->data.raw_addressing) {
	572	/* In QPL mode buffs are refilled as the desc are processed */
	573	rx->fill_cnt += work_done;
	574	} else if (rx->fill_cnt - cnt <= rx->db_threshold) {
	575	/* In raw addressing mode buffs are only refilled if the avail
	576	* falls below a threshold.
	577	*/
	578	if (!gve_rx_refill_buffers(priv, rx))
	579	return false;
	580
	581	/* If we were not able to completely refill buffers, we'll want
	582	* to schedule this queue for work again to refill buffers.
	583	*/
	584	if (rx->fill_cnt - cnt <= rx->db_threshold) {
	585	gve_rx_write_doorbell(priv, rx);
	586	return true;
	587	}
	588	}
f5cedc84	589
f5cedc84 CS	590	gve_rx_write_doorbell(priv, rx);
	591	return gve_rx_work_pending(rx);
	592	}
	593
	594	bool gve_rx_poll(struct gve_notify_block *block, int budget)
	595	{
	596	struct gve_rx_ring *rx = block->rx;
	597	netdev_features_t feat;
	598	bool repoll = false;
	599
	600	feat = block->napi.dev->features;
	601
	602	/* If budget is 0, do all the work */
	603	if (budget == 0)
	604	budget = INT_MAX;
	605
	606	if (budget > 0)
	607	repoll \|= gve_clean_rx_done(rx, budget, feat);
	608	else
	609	repoll \|= gve_rx_work_pending(rx);
	610	return repoll;
	611	}