[mirror_ubuntu-bionic-kernel.git] / drivers / staging / android / ion / ion_system_heap.c

/*
 * drivers/staging/android/ion/ion_system_heap.c
 *
 * Copyright (C) 2011 Google, Inc.
 *
 * This software is licensed under the terms of the GNU General Public
 * License version 2, as published by the Free Software Foundation, and
 * may be copied, distributed, and modified under those terms.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 */

#include <asm/page.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/highmem.h>
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/seq_file.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include "ion.h"
#include "ion_priv.h"

static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN |
				     __GFP_NORETRY) & ~__GFP_WAIT;
static gfp_t low_order_gfp_flags  = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN);
static const unsigned int orders[] = {8, 4, 0};
static const int num_orders = ARRAY_SIZE(orders);
static int order_to_index(unsigned int order)
{
	int i;

	for (i = 0; i < num_orders; i++)
		if (order == orders[i])
			return i;
	BUG();
	return -1;
}

static inline unsigned int order_to_size(int order)
{
	return PAGE_SIZE << order;
}

struct ion_system_heap {
	struct ion_heap heap;
	struct ion_page_pool **pools;
};

static struct page *alloc_buffer_page(struct ion_system_heap *heap,
				      struct ion_buffer *buffer,
				      unsigned long order)
{
	bool cached = ion_buffer_cached(buffer);
	struct ion_page_pool *pool = heap->pools[order_to_index(order)];
	struct page *page;

	if (!cached) {
		page = ion_page_pool_alloc(pool);
	} else {
		gfp_t gfp_flags = low_order_gfp_flags;

		if (order > 4)
			gfp_flags = high_order_gfp_flags;
		page = alloc_pages(gfp_flags | __GFP_COMP, order);
		if (!page)
			return NULL;
		ion_pages_sync_for_device(NULL, page, PAGE_SIZE << order,
						DMA_BIDIRECTIONAL);
	}

	return page;
}

static void free_buffer_page(struct ion_system_heap *heap,
			     struct ion_buffer *buffer, struct page *page)
{
	unsigned int order = compound_order(page);
	bool cached = ion_buffer_cached(buffer);

	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) {
		struct ion_page_pool *pool = heap->pools[order_to_index(order)];

		ion_page_pool_free(pool, page);
	} else {
		__free_pages(page, order);
	}
}


static struct page *alloc_largest_available(struct ion_system_heap *heap,
					    struct ion_buffer *buffer,
					    unsigned long size,
					    unsigned int max_order)
{
	struct page *page;
	int i;

	for (i = 0; i < num_orders; i++) {
		if (size < order_to_size(orders[i]))
			continue;
		if (max_order < orders[i])
			continue;

		page = alloc_buffer_page(heap, buffer, orders[i]);
		if (!page)
			continue;

		return page;
	}

	return NULL;
}

static int ion_system_heap_allocate(struct ion_heap *heap,
				     struct ion_buffer *buffer,
				     unsigned long size, unsigned long align,
				     unsigned long flags)
{
	struct ion_system_heap *sys_heap = container_of(heap,
							struct ion_system_heap,
							heap);
	struct sg_table *table;
	struct scatterlist *sg;
	struct list_head pages;
	struct page *page, *tmp_page;
	int i = 0;
	unsigned long size_remaining = PAGE_ALIGN(size);
	unsigned int max_order = orders[0];

	if (align > PAGE_SIZE)
		return -EINVAL;

	if (size / PAGE_SIZE > totalram_pages / 2)
		return -ENOMEM;

	INIT_LIST_HEAD(&pages);
	while (size_remaining > 0) {
		page = alloc_largest_available(sys_heap, buffer, size_remaining,
						max_order);
		if (!page)
			goto free_pages;
		list_add_tail(&page->lru, &pages);
		size_remaining -= PAGE_SIZE << compound_order(page);
		max_order = compound_order(page);
		i++;
	}
	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
	if (!table)
		goto free_pages;

	if (sg_alloc_table(table, i, GFP_KERNEL))
		goto free_table;

	sg = table->sgl;
	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
		sg_set_page(sg, page, PAGE_SIZE << compound_order(page), 0);
		sg = sg_next(sg);
		list_del(&page->lru);
	}

	buffer->priv_virt = table;
	return 0;

free_table:
	kfree(table);
free_pages:
	list_for_each_entry_safe(page, tmp_page, &pages, lru)
		free_buffer_page(sys_heap, buffer, page);
	return -ENOMEM;
}

static void ion_system_heap_free(struct ion_buffer *buffer)
{
	struct ion_system_heap *sys_heap = container_of(buffer->heap,
							struct ion_system_heap,
							heap);
	struct sg_table *table = buffer->sg_table;
	bool cached = ion_buffer_cached(buffer);
	struct scatterlist *sg;
	int i;

	/* uncached pages come from the page pools, zero them before returning
	   for security purposes (other allocations are zerod at alloc time */
	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
		ion_heap_buffer_zero(buffer);

	for_each_sg(table->sgl, sg, table->nents, i)
		free_buffer_page(sys_heap, buffer, sg_page(sg));
	sg_free_table(table);
	kfree(table);
}

static struct sg_table *ion_system_heap_map_dma(struct ion_heap *heap,
						struct ion_buffer *buffer)
{
	return buffer->priv_virt;
}

static void ion_system_heap_unmap_dma(struct ion_heap *heap,
				      struct ion_buffer *buffer)
{
	return;
}

static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
					int nr_to_scan)
{
	struct ion_system_heap *sys_heap;
	int nr_total = 0;
	int i;

	sys_heap = container_of(heap, struct ion_system_heap, heap);

	for (i = 0; i < num_orders; i++) {
		struct ion_page_pool *pool = sys_heap->pools[i];

		nr_total += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan);
	}

	return nr_total;
}

static struct ion_heap_ops system_heap_ops = {
	.allocate = ion_system_heap_allocate,
	.free = ion_system_heap_free,
	.map_dma = ion_system_heap_map_dma,
	.unmap_dma = ion_system_heap_unmap_dma,
	.map_kernel = ion_heap_map_kernel,
	.unmap_kernel = ion_heap_unmap_kernel,
	.map_user = ion_heap_map_user,
	.shrink = ion_system_heap_shrink,
};

static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,
				      void *unused)
{

	struct ion_system_heap *sys_heap = container_of(heap,
							struct ion_system_heap,
							heap);
	int i;

	for (i = 0; i < num_orders; i++) {
		struct ion_page_pool *pool = sys_heap->pools[i];

		seq_printf(s, "%d order %u highmem pages in pool = %lu total\n",
			   pool->high_count, pool->order,
			   (PAGE_SIZE << pool->order) * pool->high_count);
		seq_printf(s, "%d order %u lowmem pages in pool = %lu total\n",
			   pool->low_count, pool->order,
			   (PAGE_SIZE << pool->order) * pool->low_count);
	}
	return 0;
}

struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused)
{
	struct ion_system_heap *heap;
	int i;

	heap = kzalloc(sizeof(struct ion_system_heap), GFP_KERNEL);
	if (!heap)
		return ERR_PTR(-ENOMEM);
	heap->heap.ops = &system_heap_ops;
	heap->heap.type = ION_HEAP_TYPE_SYSTEM;
	heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
	heap->pools = kzalloc(sizeof(struct ion_page_pool *) * num_orders,
			      GFP_KERNEL);
	if (!heap->pools)
		goto free_heap;
	for (i = 0; i < num_orders; i++) {
		struct ion_page_pool *pool;
		gfp_t gfp_flags = low_order_gfp_flags;

		if (orders[i] > 4)
			gfp_flags = high_order_gfp_flags;
		pool = ion_page_pool_create(gfp_flags, orders[i]);
		if (!pool)
			goto destroy_pools;
		heap->pools[i] = pool;
	}

	heap->heap.debug_show = ion_system_heap_debug_show;
	return &heap->heap;

destroy_pools:
	while (i--)
		ion_page_pool_destroy(heap->pools[i]);
	kfree(heap->pools);
free_heap:
	kfree(heap);
	return ERR_PTR(-ENOMEM);
}

void ion_system_heap_destroy(struct ion_heap *heap)
{
	struct ion_system_heap *sys_heap = container_of(heap,
							struct ion_system_heap,
							heap);
	int i;

	for (i = 0; i < num_orders; i++)
		ion_page_pool_destroy(sys_heap->pools[i]);
	kfree(sys_heap->pools);
	kfree(sys_heap);
}

static int ion_system_contig_heap_allocate(struct ion_heap *heap,
					   struct ion_buffer *buffer,
					   unsigned long len,
					   unsigned long align,
					   unsigned long flags)
{
	int order = get_order(len);
	struct page *page;
	struct sg_table *table;
	unsigned long i;
	int ret;

	if (align > (PAGE_SIZE << order))
		return -EINVAL;

	page = alloc_pages(low_order_gfp_flags, order);
	if (!page)
		return -ENOMEM;

	split_page(page, order);

	len = PAGE_ALIGN(len);
	for (i = len >> PAGE_SHIFT; i < (1 << order); i++)
		__free_page(page + i);

	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
	if (!table) {
		ret = -ENOMEM;
		goto free_pages;
	}

	ret = sg_alloc_table(table, 1, GFP_KERNEL);
	if (ret)
		goto free_table;

	sg_set_page(table->sgl, page, len, 0);

	buffer->priv_virt = table;

	ion_pages_sync_for_device(NULL, page, len, DMA_BIDIRECTIONAL);

	return 0;

free_table:
	kfree(table);
free_pages:
	for (i = 0; i < len >> PAGE_SHIFT; i++)
		__free_page(page + i);

	return ret;
}

static void ion_system_contig_heap_free(struct ion_buffer *buffer)
{
	struct sg_table *table = buffer->priv_virt;
	struct page *page = sg_page(table->sgl);
	unsigned long pages = PAGE_ALIGN(buffer->size) >> PAGE_SHIFT;
	unsigned long i;

	for (i = 0; i < pages; i++)
		__free_page(page + i);
	sg_free_table(table);
	kfree(table);
}

static int ion_system_contig_heap_phys(struct ion_heap *heap,
				       struct ion_buffer *buffer,
				       ion_phys_addr_t *addr, size_t *len)
{
	struct sg_table *table = buffer->priv_virt;
	struct page *page = sg_page(table->sgl);
	*addr = page_to_phys(page);
	*len = buffer->size;
	return 0;
}

static struct sg_table *ion_system_contig_heap_map_dma(struct ion_heap *heap,
						struct ion_buffer *buffer)
{
	return buffer->priv_virt;
}

static void ion_system_contig_heap_unmap_dma(struct ion_heap *heap,
					     struct ion_buffer *buffer)
{
}

static struct ion_heap_ops kmalloc_ops = {
	.allocate = ion_system_contig_heap_allocate,
	.free = ion_system_contig_heap_free,
	.phys = ion_system_contig_heap_phys,
	.map_dma = ion_system_contig_heap_map_dma,
	.unmap_dma = ion_system_contig_heap_unmap_dma,
	.map_kernel = ion_heap_map_kernel,
	.unmap_kernel = ion_heap_unmap_kernel,
	.map_user = ion_heap_map_user,
};

struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused)
{
	struct ion_heap *heap;

	heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
	if (!heap)
		return ERR_PTR(-ENOMEM);
	heap->ops = &kmalloc_ops;
	heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG;
	return heap;
}

void ion_system_contig_heap_destroy(struct ion_heap *heap)
{
	kfree(heap);
}
Commit	Line	Data
c30707be RSZ	1	/*
	2	* drivers/staging/android/ion/ion_system_heap.c
	3	*
	4	* Copyright (C) 2011 Google, Inc.
	5	*
	6	* This software is licensed under the terms of the GNU General Public
	7	* License version 2, as published by the Free Software Foundation, and
	8	* may be copied, distributed, and modified under those terms.
	9	*
	10	* This program is distributed in the hope that it will be useful,
	11	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	13	* GNU General Public License for more details.
	14	*
	15	*/
	16
bd5d6bda RSZ	17	#include <asm/page.h>
bd5d6bda RSZ	18	#include <linux/dma-mapping.h>
c30707be	19	#include <linux/err.h>
bd5d6bda	20	#include <linux/highmem.h>
c30707be RSZ	21	#include <linux/mm.h>
c30707be RSZ	22	#include <linux/scatterlist.h>
45b17a80	23	#include <linux/seq_file.h>
c30707be RSZ	24	#include <linux/slab.h>
	25	#include <linux/vmalloc.h>
	26	#include "ion.h"
	27	#include "ion_priv.h"
	28
f63958d8 CC	29	static gfp_t high_order_gfp_flags = (GFP_HIGHUSER \| __GFP_ZERO \| __GFP_NOWARN \|
	30	__GFP_NORETRY) & ~__GFP_WAIT;
	31	static gfp_t low_order_gfp_flags = (GFP_HIGHUSER \| __GFP_ZERO \| __GFP_NOWARN);
45b17a80 RSZ	32	static const unsigned int orders[] = {8, 4, 0};
	33	static const int num_orders = ARRAY_SIZE(orders);
	34	static int order_to_index(unsigned int order)
	35	{
	36	int i;
10f62861	37
45b17a80 RSZ	38	for (i = 0; i < num_orders; i++)
	39	if (order == orders[i])
	40	return i;
	41	BUG();
	42	return -1;
	43	}
	44
79240748	45	static inline unsigned int order_to_size(int order)
45b17a80 RSZ	46	{
	47	return PAGE_SIZE << order;
	48	}
	49
	50	struct ion_system_heap {
	51	struct ion_heap heap;
	52	struct ion_page_pool **pools;
	53	};
	54
45b17a80 RSZ	55	static struct page alloc_buffer_page(struct ion_system_heap heap,
	56	struct ion_buffer *buffer,
	57	unsigned long order)
	58	{
	59	bool cached = ion_buffer_cached(buffer);
45b17a80 RSZ	60	struct ion_page_pool *pool = heap->pools[order_to_index(order)];
	61	struct page *page;
	62
ee4a4986	63	if (!cached) {
45b17a80	64	page = ion_page_pool_alloc(pool);
ee4a4986 RSZ	65	} else {
	66	gfp_t gfp_flags = low_order_gfp_flags;
	67
	68	if (order > 4)
	69	gfp_flags = high_order_gfp_flags;
bdeb9f1c	70	page = alloc_pages(gfp_flags \| __GFP_COMP, order);
8fae8312	71	if (!page)
f63958d8	72	return NULL;
e946b209 CC	73	ion_pages_sync_for_device(NULL, page, PAGE_SIZE << order,
e946b209 CC	74	DMA_BIDIRECTIONAL);
ee4a4986	75	}
8fae8312	76
45b17a80 RSZ	77	return page;
	78	}
	79
	80	static void free_buffer_page(struct ion_system_heap *heap,
06566f5d	81	struct ion_buffer buffer, struct page page)
45b17a80	82	{
06566f5d	83	unsigned int order = compound_order(page);
45b17a80	84	bool cached = ion_buffer_cached(buffer);
45b17a80	85
53a91c68	86	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) {
45b17a80	87	struct ion_page_pool *pool = heap->pools[order_to_index(order)];
10f62861	88
45b17a80	89	ion_page_pool_free(pool, page);
45b17a80 RSZ	90	} else {
	91	__free_pages(page, order);
	92	}
	93	}
	94
b0599c01	95
7eb88bff HS	96	static struct page alloc_largest_available(struct ion_system_heap heap,
	97	struct ion_buffer *buffer,
	98	unsigned long size,
	99	unsigned int max_order)
bd5d6bda	100	{
bd5d6bda	101	struct page *page;
bd5d6bda RSZ	102	int i;
bd5d6bda RSZ	103
45b17a80 RSZ	104	for (i = 0; i < num_orders; i++) {
45b17a80 RSZ	105	if (size < order_to_size(orders[i]))
bd5d6bda	106	continue;
ba96a2ee RSZ	107	if (max_order < orders[i])
ba96a2ee RSZ	108	continue;
45b17a80 RSZ	109
45b17a80 RSZ	110	page = alloc_buffer_page(heap, buffer, orders[i]);
bd5d6bda RSZ	111	if (!page)
bd5d6bda RSZ	112	continue;
45b17a80	113
7eb88bff	114	return page;
bd5d6bda	115	}
f4ea823b	116
bd5d6bda RSZ	117	return NULL;
	118	}
	119
c30707be RSZ	120	static int ion_system_heap_allocate(struct ion_heap *heap,
	121	struct ion_buffer *buffer,
	122	unsigned long size, unsigned long align,
	123	unsigned long flags)
c30707be	124	{
45b17a80 RSZ	125	struct ion_system_heap *sys_heap = container_of(heap,
	126	struct ion_system_heap,
	127	heap);
4d5ca329 RSZ	128	struct sg_table *table;
4d5ca329 RSZ	129	struct scatterlist *sg;
bd5d6bda	130	struct list_head pages;
7eb88bff	131	struct page page, tmp_page;
13ba7805	132	int i = 0;
c9e8440e	133	unsigned long size_remaining = PAGE_ALIGN(size);
ba96a2ee RSZ	134	unsigned int max_order = orders[0];
ba96a2ee RSZ	135
c13d1df9 CC	136	if (align > PAGE_SIZE)
	137	return -EINVAL;
	138
c9e8440e CC	139	if (size / PAGE_SIZE > totalram_pages / 2)
	140	return -ENOMEM;
	141
bd5d6bda RSZ	142	INIT_LIST_HEAD(&pages);
bd5d6bda RSZ	143	while (size_remaining > 0) {
7eb88bff	144	page = alloc_largest_available(sys_heap, buffer, size_remaining,
e1d855b0	145	max_order);
7eb88bff	146	if (!page)
79240748	147	goto free_pages;
7eb88bff HS	148	list_add_tail(&page->lru, &pages);
	149	size_remaining -= PAGE_SIZE << compound_order(page);
	150	max_order = compound_order(page);
13ba7805	151	i++;
bd5d6bda	152	}
b6152016	153	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
4d5ca329	154	if (!table)
79240748	155	goto free_pages;
bd5d6bda	156
79240748 HS	157	if (sg_alloc_table(table, i, GFP_KERNEL))
79240748 HS	158	goto free_table;
bd5d6bda RSZ	159
bd5d6bda RSZ	160	sg = table->sgl;
7eb88bff	161	list_for_each_entry_safe(page, tmp_page, &pages, lru) {
d10e4ffd	162	sg_set_page(sg, page, PAGE_SIZE << compound_order(page), 0);
c13bd1c4	163	sg = sg_next(sg);
7eb88bff	164	list_del(&page->lru);
c30707be	165	}
bd5d6bda	166
b15934b6 RSZ	167	buffer->priv_virt = table;
b15934b6 RSZ	168	return 0;
79240748 HS	169
79240748 HS	170	free_table:
4d5ca329	171	kfree(table);
79240748	172	free_pages:
7eb88bff	173	list_for_each_entry_safe(page, tmp_page, &pages, lru)
06566f5d	174	free_buffer_page(sys_heap, buffer, page);
b15934b6	175	return -ENOMEM;
c30707be RSZ	176	}
c30707be RSZ	177
f63958d8	178	static void ion_system_heap_free(struct ion_buffer *buffer)
c30707be	179	{
79240748	180	struct ion_system_heap *sys_heap = container_of(buffer->heap,
45b17a80 RSZ	181	struct ion_system_heap,
45b17a80 RSZ	182	heap);
8898227e	183	struct sg_table *table = buffer->sg_table;
0b6b2cde	184	bool cached = ion_buffer_cached(buffer);
45b17a80	185	struct scatterlist *sg;
45b17a80	186	int i;
b15934b6	187
0b6b2cde RSZ	188	/* uncached pages come from the page pools, zero them before returning
0b6b2cde RSZ	189	for security purposes (other allocations are zerod at alloc time */
53a91c68	190	if (!cached && !(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE))
0b6b2cde	191	ion_heap_buffer_zero(buffer);
77cbe828	192
b15934b6	193	for_each_sg(table->sgl, sg, table->nents, i)
06566f5d	194	free_buffer_page(sys_heap, buffer, sg_page(sg));
45b17a80 RSZ	195	sg_free_table(table);
45b17a80 RSZ	196	kfree(table);
c30707be RSZ	197	}
c30707be RSZ	198
f63958d8 CC	199	static struct sg_table ion_system_heap_map_dma(struct ion_heap heap,
f63958d8 CC	200	struct ion_buffer *buffer)
b15934b6 RSZ	201	{
	202	return buffer->priv_virt;
	203	}
	204
f63958d8 CC	205	static void ion_system_heap_unmap_dma(struct ion_heap *heap,
f63958d8 CC	206	struct ion_buffer *buffer)
b15934b6 RSZ	207	{
	208	return;
	209	}
	210
b9daf0b6 CC	211	static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask,
b9daf0b6 CC	212	int nr_to_scan)
b1aced6f	213	{
b9daf0b6	214	struct ion_system_heap *sys_heap;
ea313b5f	215	int nr_total = 0;
b1aced6f JS	216	int i;
b1aced6f JS	217
b9daf0b6	218	sys_heap = container_of(heap, struct ion_system_heap, heap);
ea313b5f RSZ	219
	220	for (i = 0; i < num_orders; i++) {
	221	struct ion_page_pool *pool = sys_heap->pools[i];
10f62861	222
b9daf0b6	223	nr_total += ion_page_pool_shrink(pool, gfp_mask, nr_to_scan);
ea313b5f RSZ	224	}
ea313b5f RSZ	225
b9daf0b6	226	return nr_total;
ea313b5f RSZ	227	}
ea313b5f RSZ	228
b9daf0b6 CC	229	static struct ion_heap_ops system_heap_ops = {
	230	.allocate = ion_system_heap_allocate,
	231	.free = ion_system_heap_free,
	232	.map_dma = ion_system_heap_map_dma,
	233	.unmap_dma = ion_system_heap_unmap_dma,
	234	.map_kernel = ion_heap_map_kernel,
	235	.unmap_kernel = ion_heap_unmap_kernel,
	236	.map_user = ion_heap_map_user,
	237	.shrink = ion_system_heap_shrink,
	238	};
	239
45b17a80 RSZ	240	static int ion_system_heap_debug_show(struct ion_heap heap, struct seq_file s,
	241	void *unused)
	242	{
	243
	244	struct ion_system_heap *sys_heap = container_of(heap,
	245	struct ion_system_heap,
	246	heap);
	247	int i;
10f62861	248
45b17a80 RSZ	249	for (i = 0; i < num_orders; i++) {
45b17a80 RSZ	250	struct ion_page_pool *pool = sys_heap->pools[i];
10f62861	251
0fb9b815 RSZ	252	seq_printf(s, "%d order %u highmem pages in pool = %lu total\n",
0fb9b815 RSZ	253	pool->high_count, pool->order,
79240748	254	(PAGE_SIZE << pool->order) * pool->high_count);
0fb9b815 RSZ	255	seq_printf(s, "%d order %u lowmem pages in pool = %lu total\n",
0fb9b815 RSZ	256	pool->low_count, pool->order,
79240748	257	(PAGE_SIZE << pool->order) * pool->low_count);
45b17a80 RSZ	258	}
	259	return 0;
	260	}
	261
c30707be RSZ	262	struct ion_heap ion_system_heap_create(struct ion_platform_heap unused)
c30707be RSZ	263	{
45b17a80 RSZ	264	struct ion_system_heap *heap;
45b17a80 RSZ	265	int i;
c30707be	266
45b17a80	267	heap = kzalloc(sizeof(struct ion_system_heap), GFP_KERNEL);
c30707be RSZ	268	if (!heap)
c30707be RSZ	269	return ERR_PTR(-ENOMEM);
45b17a80 RSZ	270	heap->heap.ops = &system_heap_ops;
45b17a80 RSZ	271	heap->heap.type = ION_HEAP_TYPE_SYSTEM;
fe2faea7	272	heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE;
45b17a80 RSZ	273	heap->pools = kzalloc(sizeof(struct ion_page_pool ) num_orders,
	274	GFP_KERNEL);
	275	if (!heap->pools)
79240748	276	goto free_heap;
45b17a80 RSZ	277	for (i = 0; i < num_orders; i++) {
45b17a80 RSZ	278	struct ion_page_pool *pool;
ee4a4986 RSZ	279	gfp_t gfp_flags = low_order_gfp_flags;
	280
	281	if (orders[i] > 4)
	282	gfp_flags = high_order_gfp_flags;
	283	pool = ion_page_pool_create(gfp_flags, orders[i]);
45b17a80	284	if (!pool)
79240748	285	goto destroy_pools;
45b17a80 RSZ	286	heap->pools[i] = pool;
45b17a80 RSZ	287	}
ea313b5f	288
45b17a80 RSZ	289	heap->heap.debug_show = ion_system_heap_debug_show;
45b17a80 RSZ	290	return &heap->heap;
79240748 HS	291
	292	destroy_pools:
	293	while (i--)
	294	ion_page_pool_destroy(heap->pools[i]);
45b17a80	295	kfree(heap->pools);
79240748	296	free_heap:
45b17a80 RSZ	297	kfree(heap);
45b17a80 RSZ	298	return ERR_PTR(-ENOMEM);
c30707be RSZ	299	}
	300
	301	void ion_system_heap_destroy(struct ion_heap *heap)
	302	{
45b17a80 RSZ	303	struct ion_system_heap *sys_heap = container_of(heap,
	304	struct ion_system_heap,
	305	heap);
	306	int i;
	307
	308	for (i = 0; i < num_orders; i++)
	309	ion_page_pool_destroy(sys_heap->pools[i]);
	310	kfree(sys_heap->pools);
	311	kfree(sys_heap);
c30707be RSZ	312	}
	313
	314	static int ion_system_contig_heap_allocate(struct ion_heap *heap,
	315	struct ion_buffer *buffer,
	316	unsigned long len,
	317	unsigned long align,
	318	unsigned long flags)
	319	{
c13d1df9	320	int order = get_order(len);
5c6a4705 CC	321	struct page *page;
	322	struct sg_table *table;
	323	unsigned long i;
	324	int ret;
c13d1df9 CC	325
	326	if (align > (PAGE_SIZE << order))
	327	return -EINVAL;
	328
5c6a4705 CC	329	page = alloc_pages(low_order_gfp_flags, order);
5c6a4705 CC	330	if (!page)
c30707be	331	return -ENOMEM;
5c6a4705 CC	332
	333	split_page(page, order);
	334
	335	len = PAGE_ALIGN(len);
	336	for (i = len >> PAGE_SHIFT; i < (1 << order); i++)
	337	__free_page(page + i);
	338
b6152016	339	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
5c6a4705 CC	340	if (!table) {
5c6a4705 CC	341	ret = -ENOMEM;
79240748	342	goto free_pages;
5c6a4705 CC	343	}
	344
	345	ret = sg_alloc_table(table, 1, GFP_KERNEL);
	346	if (ret)
79240748	347	goto free_table;
5c6a4705 CC	348
	349	sg_set_page(table->sgl, page, len, 0);
	350
	351	buffer->priv_virt = table;
	352
	353	ion_pages_sync_for_device(NULL, page, len, DMA_BIDIRECTIONAL);
	354
c30707be	355	return 0;
5c6a4705	356
79240748 HS	357	free_table:
	358	kfree(table);
	359	free_pages:
5c6a4705 CC	360	for (i = 0; i < len >> PAGE_SHIFT; i++)
5c6a4705 CC	361	__free_page(page + i);
79240748	362
5c6a4705	363	return ret;
c30707be RSZ	364	}
c30707be RSZ	365
f63958d8	366	static void ion_system_contig_heap_free(struct ion_buffer *buffer)
c30707be	367	{
5c6a4705 CC	368	struct sg_table *table = buffer->priv_virt;
	369	struct page *page = sg_page(table->sgl);
	370	unsigned long pages = PAGE_ALIGN(buffer->size) >> PAGE_SHIFT;
	371	unsigned long i;
	372
	373	for (i = 0; i < pages; i++)
	374	__free_page(page + i);
	375	sg_free_table(table);
	376	kfree(table);
c30707be RSZ	377	}
	378
	379	static int ion_system_contig_heap_phys(struct ion_heap *heap,
	380	struct ion_buffer *buffer,
	381	ion_phys_addr_t addr, size_t len)
	382	{
5c6a4705 CC	383	struct sg_table *table = buffer->priv_virt;
	384	struct page *page = sg_page(table->sgl);
	385	*addr = page_to_phys(page);
c30707be RSZ	386	*len = buffer->size;
	387	return 0;
	388	}
	389
f63958d8	390	static struct sg_table ion_system_contig_heap_map_dma(struct ion_heap heap,
56a7c185	391	struct ion_buffer *buffer)
c30707be	392	{
5c6a4705	393	return buffer->priv_virt;
c30707be RSZ	394	}
c30707be RSZ	395
f63958d8 CC	396	static void ion_system_contig_heap_unmap_dma(struct ion_heap *heap,
f63958d8 CC	397	struct ion_buffer *buffer)
56a7c185	398	{
56a7c185 RSZ	399	}
56a7c185 RSZ	400
c30707be RSZ	401	static struct ion_heap_ops kmalloc_ops = {
	402	.allocate = ion_system_contig_heap_allocate,
	403	.free = ion_system_contig_heap_free,
	404	.phys = ion_system_contig_heap_phys,
	405	.map_dma = ion_system_contig_heap_map_dma,
56a7c185	406	.unmap_dma = ion_system_contig_heap_unmap_dma,
8898227e RSZ	407	.map_kernel = ion_heap_map_kernel,
8898227e RSZ	408	.unmap_kernel = ion_heap_unmap_kernel,
a82130f4	409	.map_user = ion_heap_map_user,
c30707be RSZ	410	};
	411
	412	struct ion_heap ion_system_contig_heap_create(struct ion_platform_heap unused)
	413	{
	414	struct ion_heap *heap;
	415
	416	heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL);
	417	if (!heap)
	418	return ERR_PTR(-ENOMEM);
	419	heap->ops = &kmalloc_ops;
	420	heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG;
	421	return heap;
	422	}
	423
	424	void ion_system_contig_heap_destroy(struct ion_heap *heap)
	425	{
	426	kfree(heap);
	427	}