[mirror_ubuntu-hirsute-kernel.git] / include / linux / huge_mm.h

#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

#include <linux/sched/coredump.h>

#include <linux/fs.h> /* only for vma_is_dax() */

extern int do_huge_pmd_anonymous_page(struct vm_fault *vmf);
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
extern void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd);
extern int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pud_t *dst_pud, pud_t *src_pud, unsigned long addr,
			 struct vm_area_struct *vma);

#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
extern void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
#else
static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
{
}
#endif

extern int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd);
extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
extern bool madvise_free_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr, unsigned long next);
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd, unsigned long addr);
extern int zap_huge_pud(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pud_t *pud, unsigned long addr);
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);
extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
			 unsigned long new_addr, unsigned long old_end,
			 pmd_t *old_pmd, pmd_t *new_pmd, bool *need_flush);
extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, pgprot_t newprot,
			int prot_numa);
int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
			pmd_t *pmd, pfn_t pfn, bool write);
int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
			pud_t *pud, pfn_t pfn, bool write);
enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

struct kobject;
struct kobj_attribute;

extern ssize_t single_hugepage_flag_store(struct kobject *kobj,
				 struct kobj_attribute *attr,
				 const char *buf, size_t count,
				 enum transparent_hugepage_flag flag);
extern ssize_t single_hugepage_flag_show(struct kobject *kobj,
				struct kobj_attribute *attr, char *buf,
				enum transparent_hugepage_flag flag);
extern struct kobj_attribute shmem_enabled_attr;

#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_PMD_SHIFT PMD_SHIFT
#define HPAGE_PMD_SIZE	((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PMD_MASK	(~(HPAGE_PMD_SIZE - 1))

#define HPAGE_PUD_SHIFT PUD_SHIFT
#define HPAGE_PUD_SIZE	((1UL) << HPAGE_PUD_SHIFT)
#define HPAGE_PUD_MASK	(~(HPAGE_PUD_SIZE - 1))

extern bool is_vma_temporary_stack(struct vm_area_struct *vma);

extern unsigned long transparent_hugepage_flags;

static inline bool transparent_hugepage_enabled(struct vm_area_struct *vma)
{
	if (vma->vm_flags & VM_NOHUGEPAGE)
		return false;

	if (is_vma_temporary_stack(vma))
		return false;

	if (test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
		return false;

	if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG))
		return true;

	if (vma_is_dax(vma))
		return true;

	if (transparent_hugepage_flags &
				(1 << TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG))
		return !!(vma->vm_flags & VM_HUGEPAGE);

	return false;
}

#define transparent_hugepage_use_zero_page()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long thp_get_unmapped_area(struct file *filp,
		unsigned long addr, unsigned long len, unsigned long pgoff,
		unsigned long flags);

extern void prep_transhuge_page(struct page *page);
extern void free_transhuge_page(struct page *page);

bool can_split_huge_page(struct page *page, int *pextra_pins);
int split_huge_page_to_list(struct page *page, struct list_head *list);
static inline int split_huge_page(struct page *page)
{
	return split_huge_page_to_list(page, NULL);
}
void deferred_split_huge_page(struct page *page);

void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
		unsigned long address, bool freeze, struct page *page);

#define split_huge_pmd(__vma, __pmd, __address)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		if (is_swap_pmd(*____pmd) || pmd_trans_huge(*____pmd)	\
					|| pmd_devmap(*____pmd))	\
			__split_huge_pmd(__vma, __pmd, __address,	\
						false, NULL);		\
	}  while (0)


void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
		bool freeze, struct page *page);

void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud,
		unsigned long address);

#define split_huge_pud(__vma, __pud, __address)				\
	do {								\
		pud_t *____pud = (__pud);				\
		if (pud_trans_huge(*____pud)				\
					|| pud_devmap(*____pud))	\
			__split_huge_pud(__vma, __pud, __address);	\
	}  while (0)

extern int hugepage_madvise(struct vm_area_struct *vma,
			    unsigned long *vm_flags, int advice);
extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
				    unsigned long start,
				    unsigned long end,
				    long adjust_next);
extern spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma);
extern spinlock_t *__pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma);

static inline int is_swap_pmd(pmd_t pmd)
{
	return !pmd_none(pmd) && !pmd_present(pmd);
}

/* mmap_sem must be held on entry */
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
{
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd))
		return __pmd_trans_huge_lock(pmd, vma);
	else
		return NULL;
}
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma)
{
	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	if (pud_trans_huge(*pud) || pud_devmap(*pud))
		return __pud_trans_huge_lock(pud, vma);
	else
		return NULL;
}
static inline int hpage_nr_pages(struct page *page)
{
	if (unlikely(PageTransHuge(page)))
		return HPAGE_PMD_NR;
	return 1;
}

struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
		pmd_t *pmd, int flags);
struct page *follow_devmap_pud(struct vm_area_struct *vma, unsigned long addr,
		pud_t *pud, int flags);

extern int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd);

extern struct page *huge_zero_page;

static inline bool is_huge_zero_page(struct page *page)
{
	return READ_ONCE(huge_zero_page) == page;
}

static inline bool is_huge_zero_pmd(pmd_t pmd)
{
	return is_huge_zero_page(pmd_page(pmd));
}

static inline bool is_huge_zero_pud(pud_t pud)
{
	return false;
}

struct page *mm_get_huge_zero_page(struct mm_struct *mm);
void mm_put_huge_zero_page(struct mm_struct *mm);

#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))

static inline bool thp_migration_supported(void)
{
	return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
}

#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })

#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })

#define hpage_nr_pages(x) 1

static inline bool transparent_hugepage_enabled(struct vm_area_struct *vma)
{
	return false;
}

static inline void prep_transhuge_page(struct page *page) {}

#define transparent_hugepage_flags 0UL

#define thp_get_unmapped_area	NULL

static inline bool
can_split_huge_page(struct page *page, int *pextra_pins)
{
	BUILD_BUG();
	return false;
}
static inline int
split_huge_page_to_list(struct page *page, struct list_head *list)
{
	return 0;
}
static inline int split_huge_page(struct page *page)
{
	return 0;
}
static inline void deferred_split_huge_page(struct page *page) {}
#define split_huge_pmd(__vma, __pmd, __address)	\
	do { } while (0)

static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
		unsigned long address, bool freeze, struct page *page) {}
static inline void split_huge_pmd_address(struct vm_area_struct *vma,
		unsigned long address, bool freeze, struct page *page) {}

#define split_huge_pud(__vma, __pmd, __address)	\
	do { } while (0)

static inline int hugepage_madvise(struct vm_area_struct *vma,
				   unsigned long *vm_flags, int advice)
{
	BUG();
	return 0;
}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
					 unsigned long start,
					 unsigned long end,
					 long adjust_next)
{
}
static inline int is_swap_pmd(pmd_t pmd)
{
	return 0;
}
static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd,
		struct vm_area_struct *vma)
{
	return NULL;
}
static inline spinlock_t *pud_trans_huge_lock(pud_t *pud,
		struct vm_area_struct *vma)
{
	return NULL;
}

static inline int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd)
{
	return 0;
}

static inline bool is_huge_zero_page(struct page *page)
{
	return false;
}

static inline bool is_huge_zero_pud(pud_t pud)
{
	return false;
}

static inline void mm_put_huge_zero_page(struct mm_struct *mm)
{
	return;
}

static inline struct page *follow_devmap_pmd(struct vm_area_struct *vma,
		unsigned long addr, pmd_t *pmd, int flags)
{
	return NULL;
}

static inline struct page *follow_devmap_pud(struct vm_area_struct *vma,
		unsigned long addr, pud_t *pud, int flags)
{
	return NULL;
}

static inline bool thp_migration_supported(void)
{
	return false;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */
Commit	Line	Data
71e3aac0 AA	1	#ifndef _LINUX_HUGE_MM_H
	2	#define _LINUX_HUGE_MM_H
	3
16981d76 DW	4	#include <linux/sched/coredump.h>
16981d76 DW	5
baabda26 DW	6	#include <linux/fs.h> /* only for vma_is_dax() */
baabda26 DW	7
82b0f8c3	8	extern int do_huge_pmd_anonymous_page(struct vm_fault *vmf);
71e3aac0 AA	9	extern int copy_huge_pmd(struct mm_struct dst_mm, struct mm_struct src_mm,
	10	pmd_t dst_pmd, pmd_t src_pmd, unsigned long addr,
	11	struct vm_area_struct *vma);
82b0f8c3	12	extern void huge_pmd_set_accessed(struct vm_fault *vmf, pmd_t orig_pmd);
a00cc7d9 MW	13	extern int copy_huge_pud(struct mm_struct dst_mm, struct mm_struct src_mm,
	14	pud_t dst_pud, pud_t src_pud, unsigned long addr,
	15	struct vm_area_struct *vma);
	16
	17	#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
	18	extern void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);
	19	#else
	20	static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud)
	21	{
	22	}
	23	#endif
	24
82b0f8c3	25	extern int do_huge_pmd_wp_page(struct vm_fault *vmf, pmd_t orig_pmd);
b676b293	26	extern struct page follow_trans_huge_pmd(struct vm_area_struct vma,
71e3aac0 AA	27	unsigned long addr,
	28	pmd_t *pmd,
	29	unsigned int flags);
319904ad	30	extern bool madvise_free_huge_pmd(struct mmu_gather *tlb,
b8d3c4c3 MK	31	struct vm_area_struct *vma,
b8d3c4c3 MK	32	pmd_t *pmd, unsigned long addr, unsigned long next);
71e3aac0 AA	33	extern int zap_huge_pmd(struct mmu_gather *tlb,
71e3aac0 AA	34	struct vm_area_struct *vma,
f21760b1	35	pmd_t *pmd, unsigned long addr);
a00cc7d9 MW	36	extern int zap_huge_pud(struct mmu_gather *tlb,
	37	struct vm_area_struct *vma,
	38	pud_t *pud, unsigned long addr);
0ca1634d JW	39	extern int mincore_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
	40	unsigned long addr, unsigned long end,
	41	unsigned char *vec);
bf8616d5	42	extern bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
37a1c49a	43	unsigned long new_addr, unsigned long old_end,
5d190420	44	pmd_t old_pmd, pmd_t new_pmd, bool *need_flush);
cd7548ab	45	extern int change_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
4b10e7d5 MG	46	unsigned long addr, pgprot_t newprot,
4b10e7d5 MG	47	int prot_numa);
a00cc7d9 MW	48	int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
	49	pmd_t *pmd, pfn_t pfn, bool write);
	50	int vmf_insert_pfn_pud(struct vm_area_struct *vma, unsigned long addr,
	51	pud_t *pud, pfn_t pfn, bool write);
71e3aac0 AA	52	enum transparent_hugepage_flag {
	53	TRANSPARENT_HUGEPAGE_FLAG,
	54	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
444eb2a4 MG	55	TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
444eb2a4 MG	56	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
21440d7e	57	TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
71e3aac0	58	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f	59	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
79da5407	60	TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,
71e3aac0 AA	61	#ifdef CONFIG_DEBUG_VM
	62	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
	63	#endif
	64	};
	65
b46e756f KS	66	struct kobject;
	67	struct kobj_attribute;
	68
	69	extern ssize_t single_hugepage_flag_store(struct kobject *kobj,
	70	struct kobj_attribute *attr,
	71	const char *buf, size_t count,
	72	enum transparent_hugepage_flag flag);
	73	extern ssize_t single_hugepage_flag_show(struct kobject *kobj,
	74	struct kobj_attribute attr, char buf,
	75	enum transparent_hugepage_flag flag);
5a6e75f8 KS	76	extern struct kobj_attribute shmem_enabled_attr;
5a6e75f8 KS	77
d8c37c48 NH	78	#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
	79	#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
	80
71e3aac0	81	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
fde52796 AK	82	#define HPAGE_PMD_SHIFT PMD_SHIFT
	83	#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
	84	#define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1))
71e3aac0	85
a00cc7d9 MW	86	#define HPAGE_PUD_SHIFT PUD_SHIFT
	87	#define HPAGE_PUD_SIZE ((1UL) << HPAGE_PUD_SHIFT)
	88	#define HPAGE_PUD_MASK (~(HPAGE_PUD_SIZE - 1))
	89
20995974 AS	90	extern bool is_vma_temporary_stack(struct vm_area_struct *vma);
20995974 AS	91
16981d76 DW	92	extern unsigned long transparent_hugepage_flags;
	93
	94	static inline bool transparent_hugepage_enabled(struct vm_area_struct *vma)
	95	{
	96	if (vma->vm_flags & VM_NOHUGEPAGE)
	97	return false;
	98
	99	if (is_vma_temporary_stack(vma))
	100	return false;
	101
	102	if (test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
	103	return false;
	104
	105	if (transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG))
	106	return true;
	107
baabda26 DW	108	if (vma_is_dax(vma))
	109	return true;
	110
16981d76 DW	111	if (transparent_hugepage_flags &
	112	(1 << TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG))
	113	return !!(vma->vm_flags & VM_HUGEPAGE);
	114
	115	return false;
	116	}
	117
79da5407 KS	118	#define transparent_hugepage_use_zero_page() \
	119	(transparent_hugepage_flags & \
	120	(1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
71e3aac0 AA	121	#ifdef CONFIG_DEBUG_VM
	122	#define transparent_hugepage_debug_cow() \
	123	(transparent_hugepage_flags & \
	124	(1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
	125	#else /* CONFIG_DEBUG_VM */
	126	#define transparent_hugepage_debug_cow() 0
	127	#endif /* CONFIG_DEBUG_VM */
	128
74d2fad1 TK	129	extern unsigned long thp_get_unmapped_area(struct file *filp,
	130	unsigned long addr, unsigned long len, unsigned long pgoff,
	131	unsigned long flags);
	132
9a982250 KS	133	extern void prep_transhuge_page(struct page *page);
	134	extern void free_transhuge_page(struct page *page);
	135
b8f593cd	136	bool can_split_huge_page(struct page page, int pextra_pins);
e9b61f19 KS	137	int split_huge_page_to_list(struct page page, struct list_head list);
	138	static inline int split_huge_page(struct page *page)
	139	{
	140	return split_huge_page_to_list(page, NULL);
	141	}
9a982250	142	void deferred_split_huge_page(struct page *page);
eef1b3ba KS	143
eef1b3ba KS	144	void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
33f4751e	145	unsigned long address, bool freeze, struct page *page);
eef1b3ba KS	146
	147	#define split_huge_pmd(__vma, __pmd, __address) \
	148	do { \
	149	pmd_t *____pmd = (__pmd); \
84c3fc4e	150	if (is_swap_pmd(____pmd) \|\| pmd_trans_huge(____pmd) \
5c7fb56e	151	\|\| pmd_devmap(*____pmd)) \
fec89c10	152	__split_huge_pmd(__vma, __pmd, __address, \
33f4751e	153	false, NULL); \
eef1b3ba	154	} while (0)
ad0bed24	155
2a52bcbc	156
fec89c10 KS	157	void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
fec89c10 KS	158	bool freeze, struct page *page);
2a52bcbc	159
a00cc7d9 MW	160	void __split_huge_pud(struct vm_area_struct vma, pud_t pud,
	161	unsigned long address);
	162
	163	#define split_huge_pud(__vma, __pud, __address) \
	164	do { \
	165	pud_t *____pud = (__pud); \
	166	if (pud_trans_huge(*____pud) \
	167	\|\| pud_devmap(*____pud)) \
	168	__split_huge_pud(__vma, __pud, __address); \
	169	} while (0)
	170
60ab3244 AA	171	extern int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	172	unsigned long *vm_flags, int advice);
e1b9996b	173	extern void vma_adjust_trans_huge(struct vm_area_struct *vma,
94fcc585 AA	174	unsigned long start,
	175	unsigned long end,
	176	long adjust_next);
b6ec57f4 KS	177	extern spinlock_t __pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	178	struct vm_area_struct *vma);
a00cc7d9 MW	179	extern spinlock_t __pud_trans_huge_lock(pud_t pud,
a00cc7d9 MW	180	struct vm_area_struct *vma);
84c3fc4e ZY	181
	182	static inline int is_swap_pmd(pmd_t pmd)
	183	{
	184	return !pmd_none(pmd) && !pmd_present(pmd);
	185	}
	186
025c5b24	187	/* mmap_sem must be held on entry */
b6ec57f4 KS	188	static inline spinlock_t pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	189	struct vm_area_struct *vma)
025c5b24	190	{
81d1b09c	191	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
84c3fc4e	192	if (is_swap_pmd(pmd) \|\| pmd_trans_huge(pmd) \|\| pmd_devmap(*pmd))
b6ec57f4	193	return __pmd_trans_huge_lock(pmd, vma);
025c5b24	194	else
969e8d7e	195	return NULL;
025c5b24	196	}
a00cc7d9 MW	197	static inline spinlock_t pud_trans_huge_lock(pud_t pud,
	198	struct vm_area_struct *vma)
	199	{
	200	VM_BUG_ON_VMA(!rwsem_is_locked(&vma->vm_mm->mmap_sem), vma);
	201	if (pud_trans_huge(pud) \|\| pud_devmap(pud))
	202	return __pud_trans_huge_lock(pud, vma);
	203	else
	204	return NULL;
	205	}
2c888cfb RR	206	static inline int hpage_nr_pages(struct page *page)
	207	{
	208	if (unlikely(PageTransHuge(page)))
	209	return HPAGE_PMD_NR;
	210	return 1;
	211	}
d10e63f2	212
a00cc7d9 MW	213	struct page follow_devmap_pmd(struct vm_area_struct vma, unsigned long addr,
	214	pmd_t *pmd, int flags);
	215	struct page follow_devmap_pud(struct vm_area_struct vma, unsigned long addr,
	216	pud_t *pud, int flags);
	217
82b0f8c3	218	extern int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd);
d10e63f2	219
56873f43 WY	220	extern struct page *huge_zero_page;
	221
	222	static inline bool is_huge_zero_page(struct page *page)
	223	{
6aa7de05	224	return READ_ONCE(huge_zero_page) == page;
56873f43 WY	225	}
56873f43 WY	226
fc437044 MW	227	static inline bool is_huge_zero_pmd(pmd_t pmd)
	228	{
	229	return is_huge_zero_page(pmd_page(pmd));
	230	}
	231
a00cc7d9 MW	232	static inline bool is_huge_zero_pud(pud_t pud)
	233	{
	234	return false;
	235	}
	236
6fcb52a5 AL	237	struct page mm_get_huge_zero_page(struct mm_struct mm);
6fcb52a5 AL	238	void mm_put_huge_zero_page(struct mm_struct *mm);
fc437044	239
10102459 KS	240	#define mk_huge_pmd(page, prot) pmd_mkhuge(mk_pmd(page, prot))
10102459 KS	241
9c670ea3 NH	242	static inline bool thp_migration_supported(void)
	243	{
	244	return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);
	245	}
	246
71e3aac0	247	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
d8c37c48 NH	248	#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })
	249	#define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; })
	250	#define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
71e3aac0	251
a00cc7d9 MW	252	#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })
	253	#define HPAGE_PUD_MASK ({ BUILD_BUG(); 0; })
	254	#define HPAGE_PUD_SIZE ({ BUILD_BUG(); 0; })
	255
2c888cfb RR	256	#define hpage_nr_pages(x) 1
2c888cfb RR	257
16981d76 DW	258	static inline bool transparent_hugepage_enabled(struct vm_area_struct *vma)
	259	{
	260	return false;
	261	}
71e3aac0	262
800d8c63 KS	263	static inline void prep_transhuge_page(struct page *page) {}
800d8c63 KS	264
71e3aac0	265	#define transparent_hugepage_flags 0UL
74d2fad1 TK	266
	267	#define thp_get_unmapped_area NULL
	268
b8f593cd HY	269	static inline bool
	270	can_split_huge_page(struct page page, int pextra_pins)
	271	{
	272	BUILD_BUG();
	273	return false;
	274	}
5bc7b8ac SL	275	static inline int
	276	split_huge_page_to_list(struct page page, struct list_head list)
	277	{
	278	return 0;
	279	}
71e3aac0 AA	280	static inline int split_huge_page(struct page *page)
	281	{
	282	return 0;
	283	}
9a982250	284	static inline void deferred_split_huge_page(struct page *page) {}
78ddc534	285	#define split_huge_pmd(__vma, __pmd, __address) \
e180377f	286	do { } while (0)
2a52bcbc	287
fd60775a DR	288	static inline void __split_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
fd60775a DR	289	unsigned long address, bool freeze, struct page *page) {}
2a52bcbc	290	static inline void split_huge_pmd_address(struct vm_area_struct *vma,
fec89c10	291	unsigned long address, bool freeze, struct page *page) {}
2a52bcbc	292
a00cc7d9 MW	293	#define split_huge_pud(__vma, __pmd, __address) \
	294	do { } while (0)
	295
60ab3244 AA	296	static inline int hugepage_madvise(struct vm_area_struct *vma,
60ab3244 AA	297	unsigned long *vm_flags, int advice)
0af4e98b AA	298	{
	299	BUG();
	300	return 0;
	301	}
94fcc585 AA	302	static inline void vma_adjust_trans_huge(struct vm_area_struct *vma,
	303	unsigned long start,
	304	unsigned long end,
	305	long adjust_next)
	306	{
	307	}
84c3fc4e ZY	308	static inline int is_swap_pmd(pmd_t pmd)
	309	{
	310	return 0;
	311	}
b6ec57f4 KS	312	static inline spinlock_t pmd_trans_huge_lock(pmd_t pmd,
b6ec57f4 KS	313	struct vm_area_struct *vma)
025c5b24	314	{
b6ec57f4	315	return NULL;
025c5b24	316	}
a00cc7d9 MW	317	static inline spinlock_t pud_trans_huge_lock(pud_t pud,
	318	struct vm_area_struct *vma)
	319	{
	320	return NULL;
	321	}
d10e63f2	322
82b0f8c3	323	static inline int do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t orig_pmd)
d10e63f2	324	{
4daae3b4	325	return 0;
d10e63f2 MG	326	}
d10e63f2 MG	327
56873f43 WY	328	static inline bool is_huge_zero_page(struct page *page)
	329	{
	330	return false;
	331	}
	332
a00cc7d9 MW	333	static inline bool is_huge_zero_pud(pud_t pud)
	334	{
	335	return false;
	336	}
	337
6fcb52a5	338	static inline void mm_put_huge_zero_page(struct mm_struct *mm)
aa88b68c	339	{
6fcb52a5	340	return;
aa88b68c	341	}
3565fce3 DW	342
	343	static inline struct page follow_devmap_pmd(struct vm_area_struct vma,
	344	unsigned long addr, pmd_t *pmd, int flags)
	345	{
	346	return NULL;
	347	}
a00cc7d9 MW	348
	349	static inline struct page follow_devmap_pud(struct vm_area_struct vma,
	350	unsigned long addr, pud_t *pud, int flags)
	351	{
	352	return NULL;
	353	}
9c670ea3 NH	354
	355	static inline bool thp_migration_supported(void)
	356	{
	357	return false;
	358	}
71e3aac0 AA	359	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	360
	361	#endif /* _LINUX_HUGE_MM_H */