]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - include/linux/migrate.h
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_MIGRATE_H
3 #define _LINUX_MIGRATE_H
6 #include <linux/mempolicy.h>
7 #include <linux/migrate_mode.h>
8 #include <linux/hugetlb.h>
10 typedef struct page
*new_page_t(struct page
*page
, unsigned long private);
11 typedef void free_page_t(struct page
*page
, unsigned long private);
14 * Return values from addresss_space_operations.migratepage():
15 * - negative errno on page migration failure;
16 * - zero on page migration success;
18 #define MIGRATEPAGE_SUCCESS 0
24 MR_SYSCALL
, /* also applies to cpusets */
31 /* In mm/debug.c; also keep sync with include/trace/events/migrate.h */
32 extern const char *migrate_reason_names
[MR_TYPES
];
34 static inline struct page
*new_page_nodemask(struct page
*page
,
35 int preferred_nid
, nodemask_t
*nodemask
)
37 gfp_t gfp_mask
= GFP_USER
| __GFP_MOVABLE
| __GFP_RETRY_MAYFAIL
;
38 unsigned int order
= 0;
39 struct page
*new_page
= NULL
;
42 return alloc_huge_page_nodemask(page_hstate(compound_head(page
)),
43 preferred_nid
, nodemask
);
45 if (PageTransHuge(page
)) {
46 gfp_mask
|= GFP_TRANSHUGE
;
47 order
= HPAGE_PMD_ORDER
;
50 if (PageHighMem(page
) || (zone_idx(page_zone(page
)) == ZONE_MOVABLE
))
51 gfp_mask
|= __GFP_HIGHMEM
;
53 new_page
= __alloc_pages_nodemask(gfp_mask
, order
,
54 preferred_nid
, nodemask
);
56 if (new_page
&& PageTransHuge(new_page
))
57 prep_transhuge_page(new_page
);
62 #ifdef CONFIG_MIGRATION
64 extern void putback_movable_pages(struct list_head
*l
);
65 extern int migrate_page(struct address_space
*mapping
,
66 struct page
*newpage
, struct page
*page
,
67 enum migrate_mode mode
);
68 extern int migrate_pages(struct list_head
*l
, new_page_t
new, free_page_t free
,
69 unsigned long private, enum migrate_mode mode
, int reason
);
70 extern int isolate_movable_page(struct page
*page
, isolate_mode_t mode
);
71 extern void putback_movable_page(struct page
*page
);
73 extern int migrate_prep(void);
74 extern int migrate_prep_local(void);
75 extern void migrate_page_states(struct page
*newpage
, struct page
*page
);
76 extern void migrate_page_copy(struct page
*newpage
, struct page
*page
);
77 extern int migrate_huge_page_move_mapping(struct address_space
*mapping
,
78 struct page
*newpage
, struct page
*page
);
79 extern int migrate_page_move_mapping(struct address_space
*mapping
,
80 struct page
*newpage
, struct page
*page
, enum migrate_mode mode
,
84 static inline void putback_movable_pages(struct list_head
*l
) {}
85 static inline int migrate_pages(struct list_head
*l
, new_page_t
new,
86 free_page_t free
, unsigned long private, enum migrate_mode mode
,
89 static inline int isolate_movable_page(struct page
*page
, isolate_mode_t mode
)
92 static inline int migrate_prep(void) { return -ENOSYS
; }
93 static inline int migrate_prep_local(void) { return -ENOSYS
; }
95 static inline void migrate_page_states(struct page
*newpage
, struct page
*page
)
99 static inline void migrate_page_copy(struct page
*newpage
,
100 struct page
*page
) {}
102 static inline int migrate_huge_page_move_mapping(struct address_space
*mapping
,
103 struct page
*newpage
, struct page
*page
)
108 #endif /* CONFIG_MIGRATION */
110 #ifdef CONFIG_COMPACTION
111 extern int PageMovable(struct page
*page
);
112 extern void __SetPageMovable(struct page
*page
, struct address_space
*mapping
);
113 extern void __ClearPageMovable(struct page
*page
);
115 static inline int PageMovable(struct page
*page
) { return 0; };
116 static inline void __SetPageMovable(struct page
*page
,
117 struct address_space
*mapping
)
120 static inline void __ClearPageMovable(struct page
*page
)
125 #ifdef CONFIG_NUMA_BALANCING
126 extern bool pmd_trans_migrating(pmd_t pmd
);
127 extern int migrate_misplaced_page(struct page
*page
,
128 struct vm_area_struct
*vma
, int node
);
130 static inline bool pmd_trans_migrating(pmd_t pmd
)
134 static inline int migrate_misplaced_page(struct page
*page
,
135 struct vm_area_struct
*vma
, int node
)
137 return -EAGAIN
; /* can't migrate now */
139 #endif /* CONFIG_NUMA_BALANCING */
141 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
142 extern int migrate_misplaced_transhuge_page(struct mm_struct
*mm
,
143 struct vm_area_struct
*vma
,
144 pmd_t
*pmd
, pmd_t entry
,
145 unsigned long address
,
146 struct page
*page
, int node
);
148 static inline int migrate_misplaced_transhuge_page(struct mm_struct
*mm
,
149 struct vm_area_struct
*vma
,
150 pmd_t
*pmd
, pmd_t entry
,
151 unsigned long address
,
152 struct page
*page
, int node
)
156 #endif /* CONFIG_NUMA_BALANCING && CONFIG_TRANSPARENT_HUGEPAGE*/
159 #ifdef CONFIG_MIGRATION
162 * Watch out for PAE architecture, which has an unsigned long, and might not
163 * have enough bits to store all physical address and flags. So far we have
164 * enough room for all our flags.
166 #define MIGRATE_PFN_VALID (1UL << 0)
167 #define MIGRATE_PFN_MIGRATE (1UL << 1)
168 #define MIGRATE_PFN_LOCKED (1UL << 2)
169 #define MIGRATE_PFN_WRITE (1UL << 3)
170 #define MIGRATE_PFN_DEVICE (1UL << 4)
171 #define MIGRATE_PFN_ERROR (1UL << 5)
172 #define MIGRATE_PFN_SHIFT 6
174 static inline struct page
*migrate_pfn_to_page(unsigned long mpfn
)
176 if (!(mpfn
& MIGRATE_PFN_VALID
))
178 return pfn_to_page(mpfn
>> MIGRATE_PFN_SHIFT
);
181 static inline unsigned long migrate_pfn(unsigned long pfn
)
183 return (pfn
<< MIGRATE_PFN_SHIFT
) | MIGRATE_PFN_VALID
;
187 * struct migrate_vma_ops - migrate operation callback
189 * @alloc_and_copy: alloc destination memory and copy source memory to it
190 * @finalize_and_map: allow caller to map the successfully migrated pages
193 * The alloc_and_copy() callback happens once all source pages have been locked,
194 * unmapped and checked (checked whether pinned or not). All pages that can be
195 * migrated will have an entry in the src array set with the pfn value of the
196 * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set (other
197 * flags might be set but should be ignored by the callback).
199 * The alloc_and_copy() callback can then allocate destination memory and copy
200 * source memory to it for all those entries (ie with MIGRATE_PFN_VALID and
201 * MIGRATE_PFN_MIGRATE flag set). Once these are allocated and copied, the
202 * callback must update each corresponding entry in the dst array with the pfn
203 * value of the destination page and with the MIGRATE_PFN_VALID and
204 * MIGRATE_PFN_LOCKED flags set (destination pages must have their struct pages
205 * locked, via lock_page()).
207 * At this point the alloc_and_copy() callback is done and returns.
209 * Note that the callback does not have to migrate all the pages that are
210 * marked with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration
211 * from device memory to system memory (ie the MIGRATE_PFN_DEVICE flag is also
212 * set in the src array entry). If the device driver cannot migrate a device
213 * page back to system memory, then it must set the corresponding dst array
214 * entry to MIGRATE_PFN_ERROR. This will trigger a SIGBUS if CPU tries to
215 * access any of the virtual addresses originally backed by this page. Because
216 * a SIGBUS is such a severe result for the userspace process, the device
217 * driver should avoid setting MIGRATE_PFN_ERROR unless it is really in an
218 * unrecoverable state.
220 * For empty entry inside CPU page table (pte_none() or pmd_none() is true) we
221 * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
222 * allowing device driver to allocate device memory for those unback virtual
223 * address. For this the device driver simply have to allocate device memory
224 * and properly set the destination entry like for regular migration. Note that
225 * this can still fails and thus inside the device driver must check if the
226 * migration was successful for those entry inside the finalize_and_map()
227 * callback just like for regular migration.
229 * THE alloc_and_copy() CALLBACK MUST NOT CHANGE ANY OF THE SRC ARRAY ENTRIES
230 * OR BAD THINGS WILL HAPPEN !
233 * The finalize_and_map() callback happens after struct page migration from
234 * source to destination (destination struct pages are the struct pages for the
235 * memory allocated by the alloc_and_copy() callback). Migration can fail, and
236 * thus the finalize_and_map() allows the driver to inspect which pages were
237 * successfully migrated, and which were not. Successfully migrated pages will
238 * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
240 * It is safe to update device page table from within the finalize_and_map()
241 * callback because both destination and source page are still locked, and the
242 * mmap_sem is held in read mode (hence no one can unmap the range being
245 * Once callback is done cleaning up things and updating its page table (if it
246 * chose to do so, this is not an obligation) then it returns. At this point,
247 * the HMM core will finish up the final steps, and the migration is complete.
249 * THE finalize_and_map() CALLBACK MUST NOT CHANGE ANY OF THE SRC OR DST ARRAY
250 * ENTRIES OR BAD THINGS WILL HAPPEN !
252 struct migrate_vma_ops
{
253 void (*alloc_and_copy
)(struct vm_area_struct
*vma
,
254 const unsigned long *src
,
259 void (*finalize_and_map
)(struct vm_area_struct
*vma
,
260 const unsigned long *src
,
261 const unsigned long *dst
,
267 #if defined(CONFIG_MIGRATE_VMA_HELPER)
268 int migrate_vma(const struct migrate_vma_ops
*ops
,
269 struct vm_area_struct
*vma
,
276 static inline int migrate_vma(const struct migrate_vma_ops
*ops
,
277 struct vm_area_struct
*vma
,
286 #endif /* IS_ENABLED(CONFIG_MIGRATE_VMA_HELPER) */
288 #endif /* CONFIG_MIGRATION */
290 #endif /* _LINUX_MIGRATE_H */