[mirror_ubuntu-hirsute-kernel.git] / include / linux / huge_mm.h

#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

extern int do_huge_pmd_anonymous_page(struct fault_env *fe);
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
extern void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd);
extern int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd);
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
extern int madvise_free_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr, unsigned long next);
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr);
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);
extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
			 unsigned long new_addr, unsigned long old_end,
			 pmd_t *old_pmd, pmd_t *new_pmd);
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, pgprot_t newprot,
			int prot_numa);
int vmf_insert_pfn_pmd(struct vm_area_struct *, unsigned long addr, pmd_t *,
			pfn_t pfn, bool write);
enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

extern struct kobj_attribute shmem_enabled_attr;

#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd, int flags);

#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))

extern bool is_vma_temporary_stack(struct vm_area_struct *vma);

#define transparent_hugepage_enabled(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_FLAG) ||				\
	  (transparent_hugepage_flags &					\
	   (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
	   ((__vma)->vm_flags & VM_HUGEPAGE))) &&			\
	 !((__vma)->vm_flags & VM_NOHUGEPAGE) &&			\
	 !is_vma_temporary_stack(__vma))
#define transparent_hugepage_use_zero_page()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long transparent_hugepage_flags;

extern void prep_transhuge_page(struct page *page);
extern void free_transhuge_page(struct page *page);

int split_huge_page_to_list(struct page *page, struct list_head *list);
static inline int split_huge_page(struct page *page)
{
	return split_huge_page_to_list(page, NULL);
}
void deferred_split_huge_page(struct page *page);

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
		unsigned long address, bool freeze, struct page *page);

#define split_huge_pmd(__vma, __pmd, __address)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		if (pmd_trans_huge(*____pmd)				\
					|| pmd_devmap(*____pmd))	\
			__split_huge_pmd(__vma, __pmd, __address,	\
						false, NULL);		\
	}  while (0)


void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
		bool freeze, struct page *page);

extern int hugepage_madvise(struct vm_area_struct *vma,
			    unsigned long *vm_flags, int advice);
extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
				    unsigned long start,
				    unsigned long end,
				    long adjust_next);
extern spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma);
/* mmap_sem must be held on entry */
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
{
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	if (pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
		return __pmd_trans_huge_lock(pmd, vma);
	else
		return NULL;
}
static inline int hpage_nr_pages(struct page *page)
{
	if (unlikely(PageTransHuge(page)))
		return HPAGE_PMD_NR;
	return 1;
}

extern int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd);

extern struct page *huge_zero_page;

static inline bool is_huge_zero_page(struct page *page)
{
	return ACCESS_ONCE(huge_zero_page) == page;
}

static inline bool is_huge_zero_pmd(pmd_t pmd)
{
	return is_huge_zero_page(pmd_page(pmd));
}

struct page *get_huge_zero_page(void);
void put_huge_zero_page(void);

#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))

#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })

#define hpage_nr_pages(x) 1

#define transparent_hugepage_enabled(__vma) 0

static inline void prep_transhuge_page(struct page *page) {}

#define transparent_hugepage_flags 0UL
static inline int
split_huge_page_to_list(struct page *page, struct list_head *list)
{
	return 0;
}
static inline int split_huge_page(struct page *page)
{
	return 0;
}
static inline void deferred_split_huge_page(struct page *page) {}
#define split_huge_pmd(__vma, __pmd, __address)	\
	do { } while (0)

static inline void split_huge_pmd_address(struct vm_area_struct *vma,
		unsigned long address, bool freeze, struct page *page) {}

static inline int hugepage_madvise(struct vm_area_struct *vma,
				   unsigned long *vm_flags, int advice)
{
	BUG();
	return 0;
}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
}
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
{
	return NULL;
}

static inline int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd)
{
	return 0;
}

static inline bool is_huge_zero_page(struct page *page)
{
	return false;
}

static inline void put_huge_zero_page(void)
{
	BUILD_BUG();
}

static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
		unsigned long addr, pmd_t *pmd, int flags)
{
	return NULL;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */
Commit	Line	Data
71e3aac0 AA	1	#ifndef _LINUX_HUGE_MM_H
	2	#define _LINUX_HUGE_MM_H
	3
bae473a4	4	extern int do_huge_pmd_anonymous_page(struct fault_env *fe);
71e3aac0 AA	5	extern int copy_huge_pmd(struct mm_struct dst_mm, struct mm_struct src_mm,
	6	pmd_t dst_pmd, pmd_t src_pmd, unsigned long addr,
	7	struct vm_area_struct *vma);
bae473a4 KS	8	extern void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd);
bae473a4 KS	9	extern int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd);
b676b293	10	extern struct page follow_trans_huge_pmd(struct vm_area_struct vma,
71e3aac0 AA	11	unsigned long addr,
	12	pmd_t *pmd,
	13	unsigned int flags);
b8d3c4c3 MK	14	extern int madvise_free_huge_pmd(struct mmu_gather *tlb,
	15	struct vm_area_struct *vma,
	16	pmd_t *pmd, unsigned long addr, unsigned long next);
71e3aac0 AA	17	extern int zap_huge_pmd(struct mmu_gather *tlb,
71e3aac0 AA	18	struct vm_area_struct *vma,
f21760b1	19	pmd_t *pmd, unsigned long addr);
0ca1634d JW	20	extern int mincore_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
	21	unsigned long addr, unsigned long end,
	22	unsigned char *vec);
bf8616d5	23	extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
37a1c49a AA	24	unsigned long new_addr, unsigned long old_end,
37a1c49a AA	25	pmd_t old_pmd, pmd_t new_pmd);
cd7548ab	26	extern int change_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
4b10e7d5 MG	27	unsigned long addr, pgprot_t newprot,
4b10e7d5 MG	28	int prot_numa);
5cad465d	29	int vmf_insert_pfn_pmd(struct vm_area_struct , unsigned long addr, pmd_t ,
f25748e3	30	pfn_t pfn, bool write);
71e3aac0 AA	31	enum transparent_hugepage_flag {
	32	TRANSPARENT_HUGEPAGE_FLAG,
	33	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
444eb2a4 MG	34	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
444eb2a4 MG	35	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
71e3aac0	36	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f	37	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
79da5407	38	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
71e3aac0 AA	39	#ifdef CONFIG_DEBUG_VM
	40	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
	41	#endif
	42	};
	43
5a6e75f8 KS	44	extern struct kobj_attribute shmem_enabled_attr;
5a6e75f8 KS	45
d8c37c48 NH	46	#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
	47	#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
	48
71e3aac0	49	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3565fce3 DW	50	struct page follow_devmap_pmd(struct vm_area_struct vma, unsigned long addr,
	51	pmd_t *pmd, int flags);
	52
fde52796 AK	53	#define HPAGE_PMD_SHIFT PMD_SHIFT
	54	#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
	55	#define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1))
71e3aac0	56
20995974 AS	57	extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
20995974 AS	58
71e3aac0	59	#define transparent_hugepage_enabled(__vma) \
a664b2d8 AA	60	((transparent_hugepage_flags & \
	61	(1<<TRANSPARENT_HUGEPAGE_FLAG) \|\| \
	62	(transparent_hugepage_flags & \
	63	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \
	64	((__vma)->vm_flags & VM_HUGEPAGE))) && \
a7d6e4ec AA	65	!((__vma)->vm_flags & VM_NOHUGEPAGE) && \
a7d6e4ec AA	66	!is_vma_temporary_stack(__vma))
79da5407 KS	67	#define transparent_hugepage_use_zero_page() \
	68	(transparent_hugepage_flags & \
	69	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
71e3aac0 AA	70	#ifdef CONFIG_DEBUG_VM
	71	#define transparent_hugepage_debug_cow() \
	72	(transparent_hugepage_flags & \
	73	(1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
	74	#else /* CONFIG_DEBUG_VM */
	75	#define transparent_hugepage_debug_cow() 0
	76	#endif /* CONFIG_DEBUG_VM */
	77
	78	extern unsigned long transparent_hugepage_flags;
ad0bed24	79
9a982250 KS	80	extern void prep_transhuge_page(struct page *page);
	81	extern void free_transhuge_page(struct page *page);
	82
e9b61f19 KS	83	int split_huge_page_to_list(struct page page, struct list_head list);
	84	static inline int split_huge_page(struct page *page)
	85	{
	86	return split_huge_page_to_list(page, NULL);
	87	}
9a982250	88	void deferred_split_huge_page(struct page *page);
eef1b3ba KS	89
eef1b3ba KS	90	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
33f4751e	91	unsigned long address, bool freeze, struct page *page);
eef1b3ba KS	92
	93	#define split_huge_pmd(__vma, __pmd, __address) \
	94	do { \
	95	pmd_t *____pmd = (__pmd); \
5c7fb56e DW	96	if (pmd_trans_huge(*____pmd) \
5c7fb56e DW	97	\|\| pmd_devmap(*____pmd)) \
fec89c10	98	__split_huge_pmd(__vma, __pmd, __address, \
33f4751e	99	false, NULL); \
eef1b3ba	100	} while (0)
ad0bed24	101
2a52bcbc	102
fec89c10 KS	103	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
fec89c10 KS	104	bool freeze, struct page *page);
2a52bcbc	105
60ab3244 AA	106	extern int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	107	unsigned long *vm_flags, int advice);
e1b9996b	108	extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
94fcc585 AA	109	unsigned long start,
	110	unsigned long end,
	111	long adjust_next);
b6ec57f4 KS	112	extern spinlock_t __pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	113	struct vm_area_struct *vma);
025c5b24	114	/* mmap_sem must be held on entry */
b6ec57f4 KS	115	static inline spinlock_t pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	116	struct vm_area_struct *vma)
025c5b24	117	{
81d1b09c	118	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
5c7fb56e	119	if (pmd_trans_huge(pmd) \|\| pmd_devmap(pmd))
b6ec57f4	120	return __pmd_trans_huge_lock(pmd, vma);
025c5b24	121	else
969e8d7e	122	return NULL;
025c5b24	123	}
2c888cfb RR	124	static inline int hpage_nr_pages(struct page *page)
	125	{
	126	if (unlikely(PageTransHuge(page)))
	127	return HPAGE_PMD_NR;
	128	return 1;
	129	}
d10e63f2	130
bae473a4	131	extern int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd);
d10e63f2	132
56873f43 WY	133	extern struct page *huge_zero_page;
	134
	135	static inline bool is_huge_zero_page(struct page *page)
	136	{
	137	return ACCESS_ONCE(huge_zero_page) == page;
	138	}
	139
fc437044 MW	140	static inline bool is_huge_zero_pmd(pmd_t pmd)
	141	{
	142	return is_huge_zero_page(pmd_page(pmd));
	143	}
	144
	145	struct page *get_huge_zero_page(void);
aa88b68c	146	void put_huge_zero_page(void);
fc437044	147
10102459 KS	148	#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
10102459 KS	149
71e3aac0	150	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
d8c37c48 NH	151	#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
	152	#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
	153	#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
71e3aac0	154
2c888cfb RR	155	#define hpage_nr_pages(x) 1
2c888cfb RR	156
71e3aac0 AA	157	#define transparent_hugepage_enabled(__vma) 0
71e3aac0 AA	158
800d8c63 KS	159	static inline void prep_transhuge_page(struct page *page) {}
800d8c63 KS	160
71e3aac0	161	#define transparent_hugepage_flags 0UL
5bc7b8ac SL	162	static inline int
	163	split_huge_page_to_list(struct page page, struct list_head list)
	164	{
	165	return 0;
	166	}
71e3aac0 AA	167	static inline int split_huge_page(struct page *page)
	168	{
	169	return 0;
	170	}
9a982250	171	static inline void deferred_split_huge_page(struct page *page) {}
78ddc534	172	#define split_huge_pmd(__vma, __pmd, __address) \
e180377f	173	do { } while (0)
2a52bcbc KS	174
2a52bcbc KS	175	static inline void split_huge_pmd_address(struct vm_area_struct *vma,
fec89c10	176	unsigned long address, bool freeze, struct page *page) {}
2a52bcbc	177
60ab3244 AA	178	static inline int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	179	unsigned long *vm_flags, int advice)
0af4e98b AA	180	{
	181	BUG();
	182	return 0;
	183	}
94fcc585 AA	184	static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
	185	unsigned long start,
	186	unsigned long end,
	187	long adjust_next)
	188	{
	189	}
b6ec57f4 KS	190	static inline spinlock_t pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	191	struct vm_area_struct *vma)
025c5b24	192	{
b6ec57f4	193	return NULL;
025c5b24	194	}
d10e63f2	195
bae473a4	196	static inline int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t orig_pmd)
d10e63f2	197	{
4daae3b4	198	return 0;
d10e63f2 MG	199	}
d10e63f2 MG	200
56873f43 WY	201	static inline bool is_huge_zero_page(struct page *page)
	202	{
	203	return false;
	204	}
	205
aa88b68c KS	206	static inline void put_huge_zero_page(void)
	207	{
	208	BUILD_BUG();
	209	}
3565fce3 DW	210
	211	static inline struct page follow_devmap_pmd(struct vm_area_struct vma,
	212	unsigned long addr, pmd_t *pmd, int flags)
	213	{
	214	return NULL;
	215	}
71e3aac0 AA	216	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	217
	218	#endif /* _LINUX_HUGE_MM_H */