[mirror_ubuntu-hirsute-kernel.git] / include / linux / huge_mm.h

#ifndef _LINUX_HUGE_MM_H
#define _LINUX_HUGE_MM_H

extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
				      struct vm_area_struct *vma,
				      unsigned long address, pmd_t *pmd,
				      unsigned int flags);
extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
			 struct vm_area_struct *vma);
extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
			       unsigned long address, pmd_t *pmd,
			       pmd_t orig_pmd);
extern pgtable_t get_pmd_huge_pte(struct mm_struct *mm);
extern struct page *follow_trans_huge_pmd(struct mm_struct *mm,
					  unsigned long addr,
					  pmd_t *pmd,
					  unsigned int flags);
extern int zap_huge_pmd(struct mmu_gather *tlb,
			struct vm_area_struct *vma,
			pmd_t *pmd);
extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
			unsigned long addr, unsigned long end,
			unsigned char *vec);

enum transparent_hugepage_flag {
	TRANSPARENT_HUGEPAGE_FLAG,
	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
#ifdef CONFIG_DEBUG_VM
	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
#endif
};

enum page_check_address_pmd_flag {
	PAGE_CHECK_ADDRESS_PMD_FLAG,
	PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
	PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
};
extern pmd_t *page_check_address_pmd(struct page *page,
				     struct mm_struct *mm,
				     unsigned long address,
				     enum page_check_address_pmd_flag flag);

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
#define HPAGE_PMD_SHIFT HPAGE_SHIFT
#define HPAGE_PMD_MASK HPAGE_MASK
#define HPAGE_PMD_SIZE HPAGE_SIZE

#define transparent_hugepage_enabled(__vma)				\
	(transparent_hugepage_flags & (1<<TRANSPARENT_HUGEPAGE_FLAG) ||	\
	 (transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) &&			\
	  (__vma)->vm_flags & VM_HUGEPAGE))
#define transparent_hugepage_defrag(__vma)				\
	((transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) ||			\
	 (transparent_hugepage_flags &					\
	  (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) &&		\
	  (__vma)->vm_flags & VM_HUGEPAGE))
#ifdef CONFIG_DEBUG_VM
#define transparent_hugepage_debug_cow()				\
	(transparent_hugepage_flags &					\
	 (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
#else /* CONFIG_DEBUG_VM */
#define transparent_hugepage_debug_cow() 0
#endif /* CONFIG_DEBUG_VM */

extern unsigned long transparent_hugepage_flags;
extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
			  pmd_t *dst_pmd, pmd_t *src_pmd,
			  struct vm_area_struct *vma,
			  unsigned long addr, unsigned long end);
extern int handle_pte_fault(struct mm_struct *mm,
			    struct vm_area_struct *vma, unsigned long address,
			    pte_t *pte, pmd_t *pmd, unsigned int flags);
extern int split_huge_page(struct page *page);
extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
#define split_huge_page_pmd(__mm, __pmd)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		if (unlikely(pmd_trans_huge(*____pmd)))			\
			__split_huge_page_pmd(__mm, ____pmd);		\
	}  while (0)
#define wait_split_huge_page(__anon_vma, __pmd)				\
	do {								\
		pmd_t *____pmd = (__pmd);				\
		spin_unlock_wait(&(__anon_vma)->root->lock);		\
		/*							\
		 * spin_unlock_wait() is just a loop in C and so the	\
		 * CPU can reorder anything around it.			\
		 */							\
		smp_mb();						\
		BUG_ON(pmd_trans_splitting(*____pmd) ||			\
		       pmd_trans_huge(*____pmd));			\
	} while (0)
#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
#if HPAGE_PMD_ORDER > MAX_ORDER
#error "hugepages can't be allocated by the buddy allocator"
#endif
extern int hugepage_madvise(unsigned long *vm_flags);
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
#define HPAGE_PMD_SHIFT ({ BUG(); 0; })
#define HPAGE_PMD_MASK ({ BUG(); 0; })
#define HPAGE_PMD_SIZE ({ BUG(); 0; })

#define transparent_hugepage_enabled(__vma) 0

#define transparent_hugepage_flags 0UL
static inline int split_huge_page(struct page *page)
{
	return 0;
}
#define split_huge_page_pmd(__mm, __pmd)	\
	do { } while (0)
#define wait_split_huge_page(__anon_vma, __pmd)	\
	do { } while (0)
static inline int hugepage_madvise(unsigned long *vm_flags)
{
	BUG();
	return 0;
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */

#endif /* _LINUX_HUGE_MM_H */
Commit	Line	Data
71e3aac0 AA	1	#ifndef _LINUX_HUGE_MM_H
	2	#define _LINUX_HUGE_MM_H
	3
	4	extern int do_huge_pmd_anonymous_page(struct mm_struct *mm,
	5	struct vm_area_struct *vma,
	6	unsigned long address, pmd_t *pmd,
	7	unsigned int flags);
	8	extern int copy_huge_pmd(struct mm_struct dst_mm, struct mm_struct src_mm,
	9	pmd_t dst_pmd, pmd_t src_pmd, unsigned long addr,
	10	struct vm_area_struct *vma);
	11	extern int do_huge_pmd_wp_page(struct mm_struct mm, struct vm_area_struct vma,
	12	unsigned long address, pmd_t *pmd,
	13	pmd_t orig_pmd);
	14	extern pgtable_t get_pmd_huge_pte(struct mm_struct *mm);
	15	extern struct page follow_trans_huge_pmd(struct mm_struct mm,
	16	unsigned long addr,
	17	pmd_t *pmd,
	18	unsigned int flags);
	19	extern int zap_huge_pmd(struct mmu_gather *tlb,
	20	struct vm_area_struct *vma,
	21	pmd_t *pmd);
0ca1634d JW	22	extern int mincore_huge_pmd(struct vm_area_struct vma, pmd_t pmd,
	23	unsigned long addr, unsigned long end,
	24	unsigned char *vec);
71e3aac0 AA	25
	26	enum transparent_hugepage_flag {
	27	TRANSPARENT_HUGEPAGE_FLAG,
	28	TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
	29	TRANSPARENT_HUGEPAGE_DEFRAG_FLAG,
	30	TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
ba76149f	31	TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG,
71e3aac0 AA	32	#ifdef CONFIG_DEBUG_VM
	33	TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG,
	34	#endif
	35	};
	36
	37	enum page_check_address_pmd_flag {
	38	PAGE_CHECK_ADDRESS_PMD_FLAG,
	39	PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG,
	40	PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG,
	41	};
	42	extern pmd_t page_check_address_pmd(struct page page,
	43	struct mm_struct *mm,
	44	unsigned long address,
	45	enum page_check_address_pmd_flag flag);
	46
	47	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	48	#define HPAGE_PMD_SHIFT HPAGE_SHIFT
	49	#define HPAGE_PMD_MASK HPAGE_MASK
	50	#define HPAGE_PMD_SIZE HPAGE_SIZE
	51
	52	#define transparent_hugepage_enabled(__vma) \
	53	(transparent_hugepage_flags & (1<<TRANSPARENT_HUGEPAGE_FLAG) \|\| \
	54	(transparent_hugepage_flags & \
	55	(1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \
	56	(__vma)->vm_flags & VM_HUGEPAGE))
	57	#define transparent_hugepage_defrag(__vma) \
	58	((transparent_hugepage_flags & \
	59	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) \|\| \
	60	(transparent_hugepage_flags & \
	61	(1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \
	62	(__vma)->vm_flags & VM_HUGEPAGE))
	63	#ifdef CONFIG_DEBUG_VM
	64	#define transparent_hugepage_debug_cow() \
	65	(transparent_hugepage_flags & \
	66	(1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG))
	67	#else /* CONFIG_DEBUG_VM */
	68	#define transparent_hugepage_debug_cow() 0
	69	#endif /* CONFIG_DEBUG_VM */
	70
	71	extern unsigned long transparent_hugepage_flags;
	72	extern int copy_pte_range(struct mm_struct dst_mm, struct mm_struct src_mm,
	73	pmd_t dst_pmd, pmd_t src_pmd,
	74	struct vm_area_struct *vma,
	75	unsigned long addr, unsigned long end);
	76	extern int handle_pte_fault(struct mm_struct *mm,
	77	struct vm_area_struct *vma, unsigned long address,
	78	pte_t pte, pmd_t pmd, unsigned int flags);
	79	extern int split_huge_page(struct page *page);
	80	extern void __split_huge_page_pmd(struct mm_struct mm, pmd_t pmd);
	81	#define split_huge_page_pmd(__mm, __pmd) \
	82	do { \
	83	pmd_t *____pmd = (__pmd); \
	84	if (unlikely(pmd_trans_huge(*____pmd))) \
	85	__split_huge_page_pmd(__mm, ____pmd); \
	86	} while (0)
	87	#define wait_split_huge_page(__anon_vma, __pmd) \
	88	do { \
	89	pmd_t *____pmd = (__pmd); \
	90	spin_unlock_wait(&(__anon_vma)->root->lock); \
	91	/* \
	92	* spin_unlock_wait() is just a loop in C and so the \
	93	* CPU can reorder anything around it. \
	94	*/ \
	95	smp_mb(); \
96	BUG_ON(pmd_trans_splitting(*____pmd) \|\| \
97	pmd_trans_huge(*____pmd)); \
98	} while (0)
99	#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)
100	#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)
101	#if HPAGE_PMD_ORDER > MAX_ORDER
102	#error "hugepages can't be allocated by the buddy allocator"
103	#endif
0af4e98b	104	extern int hugepage_madvise(unsigned long *vm_flags);
71e3aac0 AA	105	#else /* CONFIG_TRANSPARENT_HUGEPAGE */
	106	#define HPAGE_PMD_SHIFT ({ BUG(); 0; })
	107	#define HPAGE_PMD_MASK ({ BUG(); 0; })
	108	#define HPAGE_PMD_SIZE ({ BUG(); 0; })
	109
	110	#define transparent_hugepage_enabled(__vma) 0
	111
	112	#define transparent_hugepage_flags 0UL
	113	static inline int split_huge_page(struct page *page)
	114	{
	115	return 0;
	116	}
	117	#define split_huge_page_pmd(__mm, __pmd) \
	118	do { } while (0)
	119	#define wait_split_huge_page(__anon_vma, __pmd) \
	120	do { } while (0)
0af4e98b AA	121	static inline int hugepage_madvise(unsigned long *vm_flags)
	122	{
	123	BUG();
	124	return 0;
	125	}
71e3aac0 AA	126	#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
	127
	128	#endif /* _LINUX_HUGE_MM_H */