[mirror_ubuntu-artful-kernel.git] / include / linux / huge_mm.h

#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
				      struct vm_area_struct *vma,
				      unsigned long address, pmd_t *pmd,
				      unsigned int flags);
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
extern void huge_pmd_set_accessed(struct mm_struct *mm,
				  struct vm_area_struct *vma,
				  unsigned long address, pmd_t *pmd,
				  pmd_t orig_pmd, int dirty);
extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
			       unsigned long address, pmd_t *pmd,
			       pmd_t orig_pmd);
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr);
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);
extern bool move_huge_pmd(struct vm_area_struct *vma,
			 struct vm_area_struct *new_vma,
			 unsigned long old_addr,
			 unsigned long new_addr, unsigned long old_end,
			 pmd_t *old_pmd, pmd_t *new_pmd);
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, pgprot_t newprot,
			int prot_numa);
int vmf_insert_pfn_pmd(struct vm_area_struct *, unsigned long addr, pmd_t *,
			unsigned long pfn, bool write);

enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

extern pmd_t *page_check_address_pmd(struct page *page,
				     struct mm_struct *mm,
				     unsigned long address,
				     spinlock_t **ptl);

#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))

extern bool is_vma_temporary_stack(struct vm_area_struct *vma);

#define transparent_hugepage_enabled(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_FLAG) ||				\
	  (transparent_hugepage_flags &					\
	   (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
	 !is_vma_temporary_stack(__vma))
#define transparent_hugepage_defrag(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||			\
	 (transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&		\
	  (__vma)->vm_flags & VM_HUGEPAGE))
#define transparent_hugepage_use_zero_page()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long transparent_hugepage_flags;

#define split_huge_page_to_list(page, list) BUILD_BUG()
#define split_huge_page(page) BUILD_BUG()
#define split_huge_pmd(__vma, __pmd, __address) BUILD_BUG()

#if HPAGE_PMD_ORDER >= MAX_ORDER
#error "hugepages can't be allocated by the buddy allocator"
#endif
extern int hugepage_madvise(struct vm_area_struct *vma,
			    unsigned long *vm_flags, int advice);
extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
				    unsigned long start,
				    unsigned long end,
				    long adjust_next);
extern bool __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
		spinlock_t **ptl);
/* mmap_sem must be held on entry */
static inline bool pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
		spinlock_t **ptl)
{
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	if (pmd_trans_huge(*pmd))
		return __pmd_trans_huge_lock(pmd, vma, ptl);
	else
		return false;
}
static inline int hpage_nr_pages(struct page *page)
{
	if (unlikely(PageTransHuge(page)))
		return HPAGE_PMD_NR;
	return 1;
}

extern int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
				unsigned long addr, pmd_t pmd, pmd_t *pmdp);

extern struct page *huge_zero_page;

static inline bool is_huge_zero_page(struct page *page)
{
	return ACCESS_ONCE(huge_zero_page) == page;
}

static inline bool is_huge_zero_pmd(pmd_t pmd)
{
	return is_huge_zero_page(pmd_page(pmd));
}

struct page *get_huge_zero_page(void);

#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })

#define hpage_nr_pages(x) 1

#define transparent_hugepage_enabled(__vma) 0

#define transparent_hugepage_flags 0UL
static inline int
split_huge_page_to_list(struct page *page, struct list_head *list)
{
	return 0;
}
static inline int split_huge_page(struct page *page)
{
	return 0;
}
#define split_huge_pmd(__vma, __pmd, __address)	\
	do { } while (0)
static inline int hugepage_madvise(struct vm_area_struct *vma,
				   unsigned long *vm_flags, int advice)
{
	BUG();
	return 0;
}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
}
static inline bool pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma,
		spinlock_t **ptl)
{
	return false;
}

static inline int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
					unsigned long addr, pmd_t pmd, pmd_t *pmdp)
{
	return 0;
}

static inline bool is_huge_zero_page(struct page *page)
{
	return false;
}

#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */
Commit	Line	Data
71e3aac0 AA	1	#ifndef _LINUX_HUGE_MM_H
	2	#define _LINUX_HUGE_MM_H
	3
	4	extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
	5	struct vm_area_struct *vma,
	6	unsigned long address, pmd_t *pmd,
	7	unsigned int flags);
	8	extern int copy_huge_pmd(struct mm_struct dst_mm, struct mm_struct src_mm,
	9	pmd_t dst_pmd, pmd_t src_pmd, unsigned long addr,
	10	struct vm_area_struct *vma);
a1dd450b WD	11	extern void huge_pmd_set_accessed(struct mm_struct *mm,
	12	struct vm_area_struct *vma,
	13	unsigned long address, pmd_t *pmd,
	14	pmd_t orig_pmd, int dirty);
71e3aac0 AA	15	extern int do_huge_pmd_wp_page(struct mm_struct mm, struct vm_area_struct vma,
	16	unsigned long address, pmd_t *pmd,
	17	pmd_t orig_pmd);
b676b293	18	extern struct page follow_trans_huge_pmd(struct vm_area_struct vma,
71e3aac0 AA	19	unsigned long addr,
	20	pmd_t *pmd,
	21	unsigned int flags);
	22	extern int zap_huge_pmd(struct mmu_gather *tlb,
	23	struct vm_area_struct *vma,
f21760b1	24	pmd_t *pmd, unsigned long addr);
0ca1634d JW	25	extern int mincore_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
	26	unsigned long addr, unsigned long end,
	27	unsigned char *vec);
4b471e88	28	extern bool move_huge_pmd(struct vm_area_struct *vma,
37a1c49a AA	29	struct vm_area_struct *new_vma,
	30	unsigned long old_addr,
	31	unsigned long new_addr, unsigned long old_end,
	32	pmd_t old_pmd, pmd_t new_pmd);
cd7548ab	33	extern int change_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
4b10e7d5 MG	34	unsigned long addr, pgprot_t newprot,
4b10e7d5 MG	35	int prot_numa);
5cad465d MW	36	int vmf_insert_pfn_pmd(struct vm_area_struct , unsigned long addr, pmd_t ,
5cad465d MW	37	unsigned long pfn, bool write);
71e3aac0 AA	38
	39	enum transparent_hugepage_flag {
	40	TRANSPARENT_HUGEPAGE_FLAG,
	41	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	42	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	43	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f	44	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
79da5407	45	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
71e3aac0 AA	46	#ifdef CONFIG_DEBUG_VM
	47	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
	48	#endif
	49	};
	50
71e3aac0 AA	51	extern pmd_t page_check_address_pmd(struct page page,
	52	struct mm_struct *mm,
	53	unsigned long address,
117b0791	54	spinlock_t **ptl);
71e3aac0	55
d8c37c48 NH	56	#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
	57	#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
	58
71e3aac0	59	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
fde52796 AK	60	#define HPAGE_PMD_SHIFT PMD_SHIFT
	61	#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
	62	#define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1))
71e3aac0	63
20995974 AS	64	extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
20995974 AS	65
71e3aac0	66	#define transparent_hugepage_enabled(__vma) \
a664b2d8 AA	67	((transparent_hugepage_flags & \
	68	(1<<TRANSPARENT_HUGEPAGE_FLAG) \|\| \
	69	(transparent_hugepage_flags & \
	70	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \
	71	((__vma)->vm_flags & VM_HUGEPAGE))) && \
a7d6e4ec AA	72	!((__vma)->vm_flags & VM_NOHUGEPAGE) && \
a7d6e4ec AA	73	!is_vma_temporary_stack(__vma))
71e3aac0 AA	74	#define transparent_hugepage_defrag(__vma) \
	75	((transparent_hugepage_flags & \
	76	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) \|\| \
	77	(transparent_hugepage_flags & \
	78	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \
	79	(__vma)->vm_flags & VM_HUGEPAGE))
79da5407 KS	80	#define transparent_hugepage_use_zero_page() \
	81	(transparent_hugepage_flags & \
	82	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
71e3aac0 AA	83	#ifdef CONFIG_DEBUG_VM
	84	#define transparent_hugepage_debug_cow() \
	85	(transparent_hugepage_flags & \
	86	(1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
	87	#else /* CONFIG_DEBUG_VM */
	88	#define transparent_hugepage_debug_cow() 0
	89	#endif /* CONFIG_DEBUG_VM */
	90
	91	extern unsigned long transparent_hugepage_flags;
ad0bed24 KS	92
	93	#define split_huge_page_to_list(page, list) BUILD_BUG()
	94	#define split_huge_page(page) BUILD_BUG()
	95	#define split_huge_pmd(__vma, __pmd, __address) BUILD_BUG()
	96
dfa5e237	97	#if HPAGE_PMD_ORDER >= MAX_ORDER
71e3aac0 AA	98	#error "hugepages can't be allocated by the buddy allocator"
71e3aac0 AA	99	#endif
60ab3244 AA	100	extern int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	101	unsigned long *vm_flags, int advice);
e1b9996b	102	extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
94fcc585 AA	103	unsigned long start,
	104	unsigned long end,
	105	long adjust_next);
4b471e88	106	extern bool __pmd_trans_huge_lock(pmd_t pmd, struct vm_area_struct vma,
bf929152	107	spinlock_t **ptl);
025c5b24	108	/* mmap_sem must be held on entry */
4b471e88	109	static inline bool pmd_trans_huge_lock(pmd_t pmd, struct vm_area_struct vma,
bf929152	110	spinlock_t **ptl)
025c5b24	111	{
81d1b09c	112	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
025c5b24	113	if (pmd_trans_huge(*pmd))
bf929152	114	return __pmd_trans_huge_lock(pmd, vma, ptl);
025c5b24	115	else
4b471e88	116	return false;
025c5b24	117	}
2c888cfb RR	118	static inline int hpage_nr_pages(struct page *page)
	119	{
	120	if (unlikely(PageTransHuge(page)))
	121	return HPAGE_PMD_NR;
	122	return 1;
	123	}
d10e63f2	124
4daae3b4 MG	125	extern int do_huge_pmd_numa_page(struct mm_struct mm, struct vm_area_struct vma,
4daae3b4 MG	126	unsigned long addr, pmd_t pmd, pmd_t *pmdp);
d10e63f2	127
56873f43 WY	128	extern struct page *huge_zero_page;
	129
	130	static inline bool is_huge_zero_page(struct page *page)
	131	{
	132	return ACCESS_ONCE(huge_zero_page) == page;
	133	}
	134
fc437044 MW	135	static inline bool is_huge_zero_pmd(pmd_t pmd)
	136	{
	137	return is_huge_zero_page(pmd_page(pmd));
	138	}
	139
	140	struct page *get_huge_zero_page(void);
fc437044	141
71e3aac0	142	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
d8c37c48 NH	143	#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
	144	#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
	145	#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
71e3aac0	146
2c888cfb RR	147	#define hpage_nr_pages(x) 1
2c888cfb RR	148
71e3aac0 AA	149	#define transparent_hugepage_enabled(__vma) 0
	150
	151	#define transparent_hugepage_flags 0UL
5bc7b8ac SL	152	static inline int
	153	split_huge_page_to_list(struct page page, struct list_head list)
	154	{
	155	return 0;
	156	}
71e3aac0 AA	157	static inline int split_huge_page(struct page *page)
	158	{
	159	return 0;
	160	}
78ddc534	161	#define split_huge_pmd(__vma, __pmd, __address) \
e180377f	162	do { } while (0)
60ab3244 AA	163	static inline int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	164	unsigned long *vm_flags, int advice)
0af4e98b AA	165	{
	166	BUG();
	167	return 0;
	168	}
94fcc585 AA	169	static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
	170	unsigned long start,
	171	unsigned long end,
	172	long adjust_next)
	173	{
	174	}
4b471e88	175	static inline bool pmd_trans_huge_lock(pmd_t pmd, struct vm_area_struct vma,
bf929152	176	spinlock_t **ptl)
025c5b24	177	{
4b471e88	178	return false;
025c5b24	179	}
d10e63f2	180
4daae3b4 MG	181	static inline int do_huge_pmd_numa_page(struct mm_struct mm, struct vm_area_struct vma,
4daae3b4 MG	182	unsigned long addr, pmd_t pmd, pmd_t *pmdp)
d10e63f2	183	{
4daae3b4	184	return 0;
d10e63f2 MG	185	}
d10e63f2 MG	186
56873f43 WY	187	static inline bool is_huge_zero_page(struct page *page)
	188	{
	189	return false;
	190	}
	191
71e3aac0 AA	192	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	193
	194	#endif /* _LINUX_HUGE_MM_H */