include/linux/highmem-internal.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_HIGHMEM_INTERNAL_H
   3 #define _LINUX_HIGHMEM_INTERNAL_H
   4
   5 /*
   6  * Outside of CONFIG_HIGHMEM to support X86 32bit iomap_atomic() cruft.
   7  */
   8 #ifdef CONFIG_KMAP_LOCAL
   9 void *__kmap_local_pfn_prot(unsigned long pfn, pgprot_t prot);
  10 void *__kmap_local_page_prot(struct page *page, pgprot_t prot);
  11 void kunmap_local_indexed(void *vaddr);
  12 void kmap_local_fork(struct task_struct *tsk);
  13 void __kmap_local_sched_out(void);
  14 void __kmap_local_sched_in(void);
  15 static inline void kmap_assert_nomap(void)
  16 {
  17         DEBUG_LOCKS_WARN_ON(current->kmap_ctrl.idx);
  18 }
  19 #else
  20 static inline void kmap_local_fork(struct task_struct *tsk) { }
  21 static inline void kmap_assert_nomap(void) { }
  22 #endif
  23
  24 #ifdef CONFIG_HIGHMEM
  25 #include <asm/highmem.h>
  26
  27 #ifndef ARCH_HAS_KMAP_FLUSH_TLB
  28 static inline void kmap_flush_tlb(unsigned long addr) { }
  29 #endif
  30
  31 #ifndef kmap_prot
  32 #define kmap_prot PAGE_KERNEL
  33 #endif
  34
  35 void *kmap_high(struct page *page);
  36 void kunmap_high(struct page *page);
  37 void __kmap_flush_unused(void);
  38 struct page *__kmap_to_page(void *addr);
  39
  40 static inline void *kmap(struct page *page)
  41 {
  42         void *addr;
  43
  44         might_sleep();
  45         if (!PageHighMem(page))
  46                 addr = page_address(page);
  47         else
  48                 addr = kmap_high(page);
  49         kmap_flush_tlb((unsigned long)addr);
  50         return addr;
  51 }
  52
  53 static inline void kunmap(struct page *page)
  54 {
  55         might_sleep();
  56         if (!PageHighMem(page))
  57                 return;
  58         kunmap_high(page);
  59 }
  60
  61 static inline struct page *kmap_to_page(void *addr)
  62 {
  63         return __kmap_to_page(addr);
  64 }
  65
  66 static inline void kmap_flush_unused(void)
  67 {
  68         __kmap_flush_unused();
  69 }
  70
  71 static inline void *kmap_local_page(struct page *page)
  72 {
  73         return __kmap_local_page_prot(page, kmap_prot);
  74 }
  75
  76 static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot)
  77 {
  78         return __kmap_local_page_prot(page, prot);
  79 }
  80
  81 static inline void *kmap_local_pfn(unsigned long pfn)
  82 {
  83         return __kmap_local_pfn_prot(pfn, kmap_prot);
  84 }
  85
  86 static inline void __kunmap_local(void *vaddr)
  87 {
  88         kunmap_local_indexed(vaddr);
  89 }
  90
  91 static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
  92 {
  93         if (IS_ENABLED(CONFIG_PREEMPT_RT))
  94                 migrate_disable();
  95         else
  96                 preempt_disable();
  97
  98         pagefault_disable();
  99         return __kmap_local_page_prot(page, prot);
 100 }
 101
 102 static inline void *kmap_atomic(struct page *page)
 103 {
 104         return kmap_atomic_prot(page, kmap_prot);
 105 }
 106
 107 static inline void *kmap_atomic_pfn(unsigned long pfn)
 108 {
 109         if (IS_ENABLED(CONFIG_PREEMPT_RT))
 110                 migrate_disable();
 111         else
 112                 preempt_disable();
 113
 114         pagefault_disable();
 115         return __kmap_local_pfn_prot(pfn, kmap_prot);
 116 }
 117
 118 static inline void __kunmap_atomic(void *addr)
 119 {
 120         kunmap_local_indexed(addr);
 121         pagefault_enable();
 122         if (IS_ENABLED(CONFIG_PREEMPT_RT))
 123                 migrate_enable();
 124         else
 125                 preempt_enable();
 126 }
 127
 128 unsigned int __nr_free_highpages(void);
 129 extern atomic_long_t _totalhigh_pages;
 130
 131 static inline unsigned int nr_free_highpages(void)
 132 {
 133         return __nr_free_highpages();
 134 }
 135
 136 static inline unsigned long totalhigh_pages(void)
 137 {
 138         return (unsigned long)atomic_long_read(&_totalhigh_pages);
 139 }
 140
 141 static inline void totalhigh_pages_add(long count)
 142 {
 143         atomic_long_add(count, &_totalhigh_pages);
 144 }
 145
 146 #else /* CONFIG_HIGHMEM */
 147
 148 static inline struct page *kmap_to_page(void *addr)
 149 {
 150         return virt_to_page(addr);
 151 }
 152
 153 static inline void *kmap(struct page *page)
 154 {
 155         might_sleep();
 156         return page_address(page);
 157 }
 158
 159 static inline void kunmap_high(struct page *page) { }
 160 static inline void kmap_flush_unused(void) { }
 161
 162 static inline void kunmap(struct page *page)
 163 {
 164 #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
 165         kunmap_flush_on_unmap(page_address(page));
 166 #endif
 167 }
 168
 169 static inline void *kmap_local_page(struct page *page)
 170 {
 171         return page_address(page);
 172 }
 173
 174 static inline void *kmap_local_page_prot(struct page *page, pgprot_t prot)
 175 {
 176         return kmap_local_page(page);
 177 }
 178
 179 static inline void *kmap_local_pfn(unsigned long pfn)
 180 {
 181         return kmap_local_page(pfn_to_page(pfn));
 182 }
 183
 184 static inline void __kunmap_local(void *addr)
 185 {
 186 #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
 187         kunmap_flush_on_unmap(addr);
 188 #endif
 189 }
 190
 191 static inline void *kmap_atomic(struct page *page)
 192 {
 193         if (IS_ENABLED(CONFIG_PREEMPT_RT))
 194                 migrate_disable();
 195         else
 196                 preempt_disable();
 197         pagefault_disable();
 198         return page_address(page);
 199 }
 200
 201 static inline void *kmap_atomic_prot(struct page *page, pgprot_t prot)
 202 {
 203         return kmap_atomic(page);
 204 }
 205
 206 static inline void *kmap_atomic_pfn(unsigned long pfn)
 207 {
 208         return kmap_atomic(pfn_to_page(pfn));
 209 }
 210
 211 static inline void __kunmap_atomic(void *addr)
 212 {
 213 #ifdef ARCH_HAS_FLUSH_ON_KUNMAP
 214         kunmap_flush_on_unmap(addr);
 215 #endif
 216         pagefault_enable();
 217         if (IS_ENABLED(CONFIG_PREEMPT_RT))
 218                 migrate_enable();
 219         else
 220                 preempt_enable();
 221 }
 222
 223 static inline unsigned int nr_free_highpages(void) { return 0; }
 224 static inline unsigned long totalhigh_pages(void) { return 0UL; }
 225
 226 #endif /* CONFIG_HIGHMEM */
 227
 228 /*
 229  * Prevent people trying to call kunmap_atomic() as if it were kunmap()
 230  * kunmap_atomic() should get the return value of kmap_atomic, not the page.
 231  */
 232 #define kunmap_atomic(__addr)                                   \
 233 do {                                                            \
 234         BUILD_BUG_ON(__same_type((__addr), struct page *));     \
 235         __kunmap_atomic(__addr);                                \
 236 } while (0)
 237
 238 #define kunmap_local(__addr)                                    \
 239 do {                                                            \
 240         BUILD_BUG_ON(__same_type((__addr), struct page *));     \
 241         __kunmap_local(__addr);                                 \
 242 } while (0)
 243
 244 #endif