arch/x86/include/asm/pmem.h

   1 /*
   2  * Copyright(c) 2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #ifndef __ASM_X86_PMEM_H__
  14 #define __ASM_X86_PMEM_H__
  15
  16 #include <linux/uaccess.h>
  17 #include <asm/cacheflush.h>
  18 #include <asm/cpufeature.h>
  19 #include <asm/special_insns.h>
  20
  21 #ifdef ARCH_HAS_NOCACHE_UACCESS
  22
  23 /**
  24  * arch_memcpy_to_pmem - copy data to persistent memory
  25  * @dst: destination buffer for the copy
  26  * @src: source buffer for the copy
  27  * @n: length of the copy in bytes
  28  *
  29  * Copy data to persistent memory media via non-temporal stores so that
  30  * a subsequent arch_wmb_pmem() can flush cpu and memory controller
  31  * write buffers to guarantee durability.
  32  */
  33 static inline void arch_memcpy_to_pmem(void __pmem *dst, const void *src,
  34                 size_t n)
  35 {
  36         int unwritten;
  37
  38         /*
  39          * We are copying between two kernel buffers, if
  40          * __copy_from_user_inatomic_nocache() returns an error (page
  41          * fault) we would have already reported a general protection fault
  42          * before the WARN+BUG.
  43          */
  44         unwritten = __copy_from_user_inatomic_nocache((void __force *) dst,
  45                         (void __user *) src, n);
  46         if (WARN(unwritten, "%s: fault copying %p <- %p unwritten: %d\n",
  47                                 __func__, dst, src, unwritten))
  48                 BUG();
  49 }
  50
  51 /**
  52  * arch_wmb_pmem - synchronize writes to persistent memory
  53  *
  54  * After a series of arch_memcpy_to_pmem() operations this drains data
  55  * from cpu write buffers and any platform (memory controller) buffers
  56  * to ensure that written data is durable on persistent memory media.
  57  */
  58 static inline void arch_wmb_pmem(void)
  59 {
  60         /*
  61          * wmb() to 'sfence' all previous writes such that they are
  62          * architecturally visible to 'pcommit'.  Note, that we've
  63          * already arranged for pmem writes to avoid the cache via
  64          * arch_memcpy_to_pmem().
  65          */
  66         wmb();
  67         pcommit_sfence();
  68 }
  69
  70 static inline bool arch_has_wmb_pmem(void)
  71 {
  72 #ifdef CONFIG_X86_64
  73         /*
  74          * We require that wmb() be an 'sfence', that is only guaranteed on
  75          * 64-bit builds
  76          */
  77         return static_cpu_has(X86_FEATURE_PCOMMIT);
  78 #else
  79         return false;
  80 #endif
  81 }
  82 #else /* ARCH_HAS_NOCACHE_UACCESS i.e. ARCH=um */
  83 extern void arch_memcpy_to_pmem(void __pmem *dst, const void *src, size_t n);
  84 extern void arch_wmb_pmem(void);
  85
  86 static inline bool __arch_has_wmb_pmem(void)
  87 {
  88         return false;
  89 }
  90 #endif
  91
  92 #endif /* __ASM_X86_PMEM_H__ */