unsigned small;
};
-#define NUM_POOLS 4
+#define NUM_POOLS 6
/**
* struct ttm_pool_manager - Holds memory pools for fst allocation
struct ttm_page_pool uc_pool;
struct ttm_page_pool wc_pool_dma32;
struct ttm_page_pool uc_pool_dma32;
+ struct ttm_page_pool wc_pool_huge;
+ struct ttm_page_pool uc_pool_huge;
} ;
};
};
/**
* Select the right pool or requested caching state and ttm flags. */
-static struct ttm_page_pool *ttm_get_pool(int flags,
- enum ttm_caching_state cstate)
+static struct ttm_page_pool *ttm_get_pool(int flags, bool huge,
+ enum ttm_caching_state cstate)
{
int pool_index;
else
pool_index = 0x1;
- if (flags & TTM_PAGE_FLAG_DMA32)
+ if (flags & TTM_PAGE_FLAG_DMA32) {
+ if (huge)
+ return NULL;
pool_index |= 0x2;
+ } else if (huge) {
+ pool_index |= 0x4;
+ }
+
return &_manager->pools[pool_index];
}
* pages returned in pages array.
*/
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
- int ttm_flags, enum ttm_caching_state cstate, unsigned count)
+ int ttm_flags, enum ttm_caching_state cstate,
+ unsigned count, unsigned order)
{
struct page **caching_array;
struct page *p;
int r = 0;
- unsigned i, cpages;
+ unsigned i, j, cpages;
+ unsigned npages = 1 << order;
unsigned max_cpages = min(count,
(unsigned)(PAGE_SIZE/sizeof(struct page *)));
}
for (i = 0, cpages = 0; i < count; ++i) {
- p = alloc_page(gfp_flags);
+ p = alloc_pages(gfp_flags, order);
if (!p) {
pr_err("Unable to get page %u\n", i);
goto out;
}
+ list_add(&p->lru, pages);
+
#ifdef CONFIG_HIGHMEM
/* gfp flags of highmem page should never be dma32 so we
* we should be fine in such case
*/
- if (!PageHighMem(p))
+ if (PageHighMem(p))
+ continue;
+
#endif
- {
- caching_array[cpages++] = p;
+ for (j = 0; j < npages; ++j) {
+ caching_array[cpages++] = p++;
if (cpages == max_cpages) {
r = ttm_set_pages_caching(caching_array,
cpages = 0;
}
}
-
- list_add(&p->lru, pages);
}
if (cpages) {
* Fill the given pool if there aren't enough pages and the requested number of
* pages is small.
*/
-static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
- int ttm_flags, enum ttm_caching_state cstate, unsigned count,
- unsigned long *irq_flags)
+static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool, int ttm_flags,
+ enum ttm_caching_state cstate,
+ unsigned count, unsigned long *irq_flags)
{
struct page *p;
int r;
INIT_LIST_HEAD(&new_pages);
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
- cstate, alloc_size);
+ cstate, alloc_size, 0);
spin_lock_irqsave(&pool->lock, *irq_flags);
if (!r) {
}
/**
- * Cut 'count' number of pages from the pool and put them on the return list.
+ * Allocate pages from the pool and put them on the return list.
*
- * @return count of pages still required to fulfill the request.
+ * @return zero for success or negative error code.
*/
-static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
- struct list_head *pages,
- int ttm_flags,
- enum ttm_caching_state cstate,
- unsigned count)
+static int ttm_page_pool_get_pages(struct ttm_page_pool *pool,
+ struct list_head *pages,
+ int ttm_flags,
+ enum ttm_caching_state cstate,
+ unsigned count, unsigned order)
{
unsigned long irq_flags;
struct list_head *p;
unsigned i;
+ int r = 0;
spin_lock_irqsave(&pool->lock, irq_flags);
- ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
+ if (!order)
+ ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count,
+ &irq_flags);
if (count >= pool->npages) {
/* take all pages from the pool */
count = 0;
out:
spin_unlock_irqrestore(&pool->lock, irq_flags);
- return count;
+
+ /* clear the pages coming from the pool if requested */
+ if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
+ struct page *page;
+
+ list_for_each_entry(page, pages, lru) {
+ if (PageHighMem(page))
+ clear_highpage(page);
+ else
+ clear_page(page_address(page));
+ }
+ }
+
+ /* If pool didn't have enough pages allocate new one. */
+ if (count) {
+ gfp_t gfp_flags = pool->gfp_flags;
+
+ /* set zero flag for page allocation if required */
+ if (ttm_flags & TTM_PAGE_FLAG_ZERO_ALLOC)
+ gfp_flags |= __GFP_ZERO;
+
+ /* ttm_alloc_new_pages doesn't reference pool so we can run
+ * multiple requests in parallel.
+ **/
+ r = ttm_alloc_new_pages(pages, gfp_flags, ttm_flags, cstate,
+ count, order);
+ }
+
+ return r;
}
/* Put all pages in pages list to correct pool to wait for reuse */
static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
+ struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
+#endif
unsigned long irq_flags;
- struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
unsigned i;
if (pool == NULL) {
/* No pool for this memory type so free the pages */
- for (i = 0; i < npages; i++) {
- if (pages[i]) {
- if (page_count(pages[i]) != 1)
- pr_err("Erroneous page count. Leaking pages.\n");
- __free_page(pages[i]);
- pages[i] = NULL;
+ i = 0;
+ while (i < npages) {
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ struct page *p = pages[i];
+#endif
+ unsigned order = 0, j;
+
+ if (!pages[i]) {
+ ++i;
+ continue;
+ }
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ for (j = 0; j < HPAGE_PMD_NR; ++j)
+ if (p++ != pages[i + j])
+ break;
+
+ if (j == HPAGE_PMD_NR)
+ order = HPAGE_PMD_ORDER;
+#endif
+
+ if (page_count(pages[i]) != 1)
+ pr_err("Erroneous page count. Leaking pages.\n");
+ __free_pages(pages[i], order);
+
+ j = 1 << order;
+ while (j) {
+ pages[i++] = NULL;
+ --j;
}
}
return;
}
+ i = 0;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ if (huge) {
+ unsigned max_size, n2free;
+
+ spin_lock_irqsave(&huge->lock, irq_flags);
+ while (i < npages) {
+ struct page *p = pages[i];
+ unsigned j;
+
+ if (!p)
+ break;
+
+ for (j = 0; j < HPAGE_PMD_NR; ++j)
+ if (p++ != pages[i + j])
+ break;
+
+ if (j != HPAGE_PMD_NR)
+ break;
+
+ list_add_tail(&pages[i]->lru, &huge->list);
+
+ for (j = 0; j < HPAGE_PMD_NR; ++j)
+ pages[i++] = NULL;
+ huge->npages++;
+ }
+
+ /* Check that we don't go over the pool limit */
+ max_size = _manager->options.max_size;
+ max_size /= HPAGE_PMD_NR;
+ if (huge->npages > max_size)
+ n2free = huge->npages - max_size;
+ else
+ n2free = 0;
+ spin_unlock_irqrestore(&huge->lock, irq_flags);
+ if (n2free)
+ ttm_page_pool_free(huge, n2free, false);
+ }
+#endif
+
spin_lock_irqsave(&pool->lock, irq_flags);
- for (i = 0; i < npages; i++) {
+ while (i < npages) {
if (pages[i]) {
if (page_count(pages[i]) != 1)
pr_err("Erroneous page count. Leaking pages.\n");
pages[i] = NULL;
pool->npages++;
}
+ ++i;
}
/* Check that we don't go over the pool limit */
npages = 0;
static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
enum ttm_caching_state cstate)
{
- struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
+ struct ttm_page_pool *pool = ttm_get_pool(flags, false, cstate);
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ struct ttm_page_pool *huge = ttm_get_pool(flags, true, cstate);
+#endif
struct list_head plist;
struct page *p = NULL;
- gfp_t gfp_flags = GFP_USER;
unsigned count;
int r;
- /* set zero flag for page allocation if required */
- if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
- gfp_flags |= __GFP_ZERO;
-
/* No pool for cached pages */
if (pool == NULL) {
+ gfp_t gfp_flags = GFP_USER;
+ unsigned i;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ unsigned j;
+#endif
+
+ /* set zero flag for page allocation if required */
+ if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
+ gfp_flags |= __GFP_ZERO;
+
if (flags & TTM_PAGE_FLAG_DMA32)
gfp_flags |= GFP_DMA32;
else
gfp_flags |= GFP_HIGHUSER;
- for (r = 0; r < npages; ++r) {
+ i = 0;
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ while (npages >= HPAGE_PMD_NR) {
+ gfp_t huge_flags = gfp_flags;
+
+ huge_flags |= GFP_TRANSHUGE;
+ huge_flags &= ~__GFP_MOVABLE;
+ huge_flags &= ~__GFP_COMP;
+ p = alloc_pages(huge_flags, HPAGE_PMD_ORDER);
+ if (!p)
+ break;
+
+ for (j = 0; j < HPAGE_PMD_NR; ++j)
+ pages[i++] = p++;
+
+ npages -= HPAGE_PMD_NR;
+ }
+#endif
+
+ while (npages) {
p = alloc_page(gfp_flags);
if (!p) {
return -ENOMEM;
}
- pages[r] = p;
+ pages[i++] = p;
+ --npages;
}
return 0;
}
- /* combine zero flag to pool flags */
- gfp_flags |= pool->gfp_flags;
-
- /* First we take pages from the pool */
- INIT_LIST_HEAD(&plist);
- npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
count = 0;
- list_for_each_entry(p, &plist, lru) {
- pages[count++] = p;
- }
- /* clear the pages coming from the pool if requested */
- if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ if (huge && npages >= HPAGE_PMD_NR) {
+ INIT_LIST_HEAD(&plist);
+ ttm_page_pool_get_pages(huge, &plist, flags, cstate,
+ npages / HPAGE_PMD_NR,
+ HPAGE_PMD_ORDER);
+
list_for_each_entry(p, &plist, lru) {
- if (PageHighMem(p))
- clear_highpage(p);
- else
- clear_page(page_address(p));
+ unsigned j;
+
+ for (j = 0; j < HPAGE_PMD_NR; ++j)
+ pages[count++] = &p[j];
}
}
+#endif
- /* If pool didn't have enough pages allocate new one. */
- if (npages > 0) {
- /* ttm_alloc_new_pages doesn't reference pool so we can run
- * multiple requests in parallel.
- **/
- INIT_LIST_HEAD(&plist);
- r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
- list_for_each_entry(p, &plist, lru) {
- pages[count++] = p;
- }
- if (r) {
- /* If there is any pages in the list put them back to
- * the pool. */
- pr_err("Failed to allocate extra pages for large request\n");
- ttm_put_pages(pages, count, flags, cstate);
- return r;
- }
+ INIT_LIST_HEAD(&plist);
+ r = ttm_page_pool_get_pages(pool, &plist, flags, cstate,
+ npages - count, 0);
+
+ list_for_each_entry(p, &plist, lru)
+ pages[count++] = p;
+
+ if (r) {
+ /* If there is any pages in the list put them back to
+ * the pool.
+ */
+ pr_err("Failed to allocate extra pages for large request\n");
+ ttm_put_pages(pages, count, flags, cstate);
+ return r;
}
return 0;
ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
GFP_USER | GFP_DMA32, "uc dma");
+ ttm_page_pool_init_locked(&_manager->wc_pool_huge,
+ GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP),
+ "wc huge");
+
+ ttm_page_pool_init_locked(&_manager->uc_pool_huge,
+ GFP_TRANSHUGE & ~(__GFP_MOVABLE | __GFP_COMP)
+ , "uc huge");
+
_manager->options.max_size = max_pages;
_manager->options.small = SMALL_ALLOCATION;
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
if (ttm->state != tt_unpopulated)
return 0;
- for (i = 0; i < ttm->num_pages; ++i) {
- ret = ttm_get_pages(&ttm->pages[i], 1,
- ttm->page_flags,
- ttm->caching_state);
- if (ret != 0) {
- ttm_pool_unpopulate(ttm);
- return -ENOMEM;
- }
+ ret = ttm_get_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+ ttm->caching_state);
+ if (unlikely(ret != 0)) {
+ ttm_pool_unpopulate(ttm);
+ return ret;
+ }
+ for (i = 0; i < ttm->num_pages; ++i) {
ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
- false, false);
+ PAGE_SIZE);
if (unlikely(ret != 0)) {
ttm_pool_unpopulate(ttm);
return -ENOMEM;
unsigned i;
for (i = 0; i < ttm->num_pages; ++i) {
- if (ttm->pages[i]) {
- ttm_mem_global_free_page(ttm->glob->mem_glob,
- ttm->pages[i]);
- ttm_put_pages(&ttm->pages[i], 1,
- ttm->page_flags,
- ttm->caching_state);
- }
+ if (!ttm->pages[i])
+ continue;
+
+ ttm_mem_global_free_page(ttm->glob->mem_glob, ttm->pages[i],
+ PAGE_SIZE);
}
+ ttm_put_pages(ttm->pages, ttm->num_pages, ttm->page_flags,
+ ttm->caching_state);
ttm->state = tt_unpopulated;
}
EXPORT_SYMBOL(ttm_pool_unpopulate);
+#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU)
+int ttm_populate_and_map_pages(struct device *dev, struct ttm_dma_tt *tt)
+{
+ unsigned i, j;
+ int r;
+
+ r = ttm_pool_populate(&tt->ttm);
+ if (r)
+ return r;
+
+ for (i = 0; i < tt->ttm.num_pages; ++i) {
+ struct page *p = tt->ttm.pages[i];
+ size_t num_pages = 1;
+
+ for (j = i + 1; j < tt->ttm.num_pages; ++j) {
+ if (++p != tt->ttm.pages[j])
+ break;
+
+ ++num_pages;
+ }
+
+ tt->dma_address[i] = dma_map_page(dev, tt->ttm.pages[i],
+ 0, num_pages * PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, tt->dma_address[i])) {
+ while (i--) {
+ dma_unmap_page(dev, tt->dma_address[i],
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ tt->dma_address[i] = 0;
+ }
+ ttm_pool_unpopulate(&tt->ttm);
+ return -EFAULT;
+ }
+
+ for (j = 1; j < num_pages; ++j) {
+ tt->dma_address[i + 1] = tt->dma_address[i] + PAGE_SIZE;
+ ++i;
+ }
+ }
+ return 0;
+}
+EXPORT_SYMBOL(ttm_populate_and_map_pages);
+
+void ttm_unmap_and_unpopulate_pages(struct device *dev, struct ttm_dma_tt *tt)
+{
+ unsigned i, j;
+
+ for (i = 0; i < tt->ttm.num_pages;) {
+ struct page *p = tt->ttm.pages[i];
+ size_t num_pages = 1;
+
+ if (!tt->dma_address[i] || !tt->ttm.pages[i]) {
+ ++i;
+ continue;
+ }
+
+ for (j = i + 1; j < tt->ttm.num_pages; ++j) {
+ if (++p != tt->ttm.pages[j])
+ break;
+
+ ++num_pages;
+ }
+
+ dma_unmap_page(dev, tt->dma_address[i], num_pages * PAGE_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ i += num_pages;
+ }
+ ttm_pool_unpopulate(&tt->ttm);
+}
+EXPORT_SYMBOL(ttm_unmap_and_unpopulate_pages);
+#endif
+
int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
{
struct ttm_page_pool *p;
seq_printf(m, "No pool allocator running.\n");
return 0;
}
- seq_printf(m, "%6s %12s %13s %8s\n",
+ seq_printf(m, "%7s %12s %13s %8s\n",
h[0], h[1], h[2], h[3]);
for (i = 0; i < NUM_POOLS; ++i) {
p = &_manager->pools[i];
- seq_printf(m, "%6s %12ld %13ld %8d\n",
+ seq_printf(m, "%7s %12ld %13ld %8d\n",
p->name, p->nrefills,
p->nfrees, p->npages);
}
projects / mirror_ubuntu-bionic-kernel.git / blobdiff