]>
Commit | Line | Data |
---|---|---|
40603526 RZ |
1 | /* |
2 | * Copyright(c) 2015 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of version 2 of the GNU General Public License as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | */ | |
13 | #ifndef __ASM_X86_PMEM_H__ | |
14 | #define __ASM_X86_PMEM_H__ | |
15 | ||
16 | #include <linux/uaccess.h> | |
17 | #include <asm/cacheflush.h> | |
18 | #include <asm/cpufeature.h> | |
19 | #include <asm/special_insns.h> | |
20 | ||
4a370df5 | 21 | #ifdef CONFIG_ARCH_HAS_PMEM_API |
40603526 RZ |
22 | /** |
23 | * arch_memcpy_to_pmem - copy data to persistent memory | |
24 | * @dst: destination buffer for the copy | |
25 | * @src: source buffer for the copy | |
26 | * @n: length of the copy in bytes | |
27 | * | |
28 | * Copy data to persistent memory media via non-temporal stores so that | |
7c8a6a71 | 29 | * a subsequent pmem driver flush operation will drain posted write queues. |
40603526 | 30 | */ |
7a9eb206 | 31 | static inline void arch_memcpy_to_pmem(void *dst, const void *src, size_t n) |
40603526 | 32 | { |
7a9eb206 | 33 | int rem; |
40603526 RZ |
34 | |
35 | /* | |
36 | * We are copying between two kernel buffers, if | |
37 | * __copy_from_user_inatomic_nocache() returns an error (page | |
38 | * fault) we would have already reported a general protection fault | |
39 | * before the WARN+BUG. | |
40 | */ | |
7a9eb206 DW |
41 | rem = __copy_from_user_inatomic_nocache(dst, (void __user *) src, n); |
42 | if (WARN(rem, "%s: fault copying %p <- %p unwritten: %d\n", | |
43 | __func__, dst, src, rem)) | |
40603526 RZ |
44 | BUG(); |
45 | } | |
46 | ||
7a9eb206 | 47 | static inline int arch_memcpy_from_pmem(void *dst, const void *src, size_t n) |
fc0c2028 DW |
48 | { |
49 | if (static_cpu_has(X86_FEATURE_MCE_RECOVERY)) | |
7a9eb206 DW |
50 | return memcpy_mcsafe(dst, src, n); |
51 | memcpy(dst, src, n); | |
fc0c2028 DW |
52 | return 0; |
53 | } | |
54 | ||
5de490da | 55 | /** |
3f4a2670 | 56 | * arch_wb_cache_pmem - write back a cache range with CLWB |
5de490da RZ |
57 | * @vaddr: virtual start address |
58 | * @size: number of bytes to write back | |
59 | * | |
60 | * Write back a cache range using the CLWB (cache line write back) | |
7c8a6a71 | 61 | * instruction. |
5de490da | 62 | */ |
7a9eb206 | 63 | static inline void arch_wb_cache_pmem(void *addr, size_t size) |
5de490da RZ |
64 | { |
65 | u16 x86_clflush_size = boot_cpu_data.x86_clflush_size; | |
66 | unsigned long clflush_mask = x86_clflush_size - 1; | |
7a9eb206 | 67 | void *vend = addr + size; |
5de490da RZ |
68 | void *p; |
69 | ||
7a9eb206 | 70 | for (p = (void *)((unsigned long)addr & ~clflush_mask); |
5de490da RZ |
71 | p < vend; p += x86_clflush_size) |
72 | clwb(p); | |
73 | } | |
74 | ||
75 | /* | |
76 | * copy_from_iter_nocache() on x86 only uses non-temporal stores for iovec | |
77 | * iterators, so for other types (bvec & kvec) we must do a cache write-back. | |
78 | */ | |
79 | static inline bool __iter_needs_pmem_wb(struct iov_iter *i) | |
80 | { | |
81 | return iter_is_iovec(i) == false; | |
82 | } | |
83 | ||
84 | /** | |
85 | * arch_copy_from_iter_pmem - copy data from an iterator to PMEM | |
86 | * @addr: PMEM destination address | |
87 | * @bytes: number of bytes to copy | |
88 | * @i: iterator with source data | |
89 | * | |
90 | * Copy data from the iterator 'i' to the PMEM buffer starting at 'addr'. | |
5de490da | 91 | */ |
7a9eb206 | 92 | static inline size_t arch_copy_from_iter_pmem(void *addr, size_t bytes, |
5de490da RZ |
93 | struct iov_iter *i) |
94 | { | |
5de490da RZ |
95 | size_t len; |
96 | ||
97 | /* TODO: skip the write-back by always using non-temporal stores */ | |
7a9eb206 | 98 | len = copy_from_iter_nocache(addr, bytes, i); |
5de490da RZ |
99 | |
100 | if (__iter_needs_pmem_wb(i)) | |
3f4a2670 | 101 | arch_wb_cache_pmem(addr, bytes); |
5de490da RZ |
102 | |
103 | return len; | |
104 | } | |
105 | ||
106 | /** | |
107 | * arch_clear_pmem - zero a PMEM memory range | |
108 | * @addr: virtual start address | |
109 | * @size: number of bytes to zero | |
110 | * | |
111 | * Write zeros into the memory range starting at 'addr' for 'size' bytes. | |
5de490da | 112 | */ |
7a9eb206 | 113 | static inline void arch_clear_pmem(void *addr, size_t size) |
5de490da | 114 | { |
7a9eb206 | 115 | memset(addr, 0, size); |
3f4a2670 | 116 | arch_wb_cache_pmem(addr, size); |
5de490da RZ |
117 | } |
118 | ||
7a9eb206 | 119 | static inline void arch_invalidate_pmem(void *addr, size_t size) |
59e64739 | 120 | { |
7a9eb206 | 121 | clflush_cache_range(addr, size); |
59e64739 | 122 | } |
4a370df5 | 123 | #endif /* CONFIG_ARCH_HAS_PMEM_API */ |
40603526 | 124 | #endif /* __ASM_X86_PMEM_H__ */ |