6 #include <linux/radix-tree.h>
7 #include <asm/pgtable.h>
12 * We use lowest available bit in exceptional entry for locking, one bit for
13 * the entry size (PMD) and two more to tell us if the entry is a huge zero
14 * page (HZP) or an empty entry that is just used for locking. In total four
17 * If the PMD bit isn't set the entry has size PAGE_SIZE, and if the HZP and
18 * EMPTY bits aren't set the entry is a normal DAX entry with a filesystem
21 #define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 4)
22 #define RADIX_DAX_ENTRY_LOCK (1 << RADIX_TREE_EXCEPTIONAL_SHIFT)
23 #define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
24 #define RADIX_DAX_HZP (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
25 #define RADIX_DAX_EMPTY (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 3))
27 static inline unsigned long dax_radix_sector(void *entry
)
29 return (unsigned long)entry
>> RADIX_DAX_SHIFT
;
32 static inline void *dax_radix_locked_entry(sector_t sector
, unsigned long flags
)
34 return (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY
| flags
|
35 ((unsigned long)sector
<< RADIX_DAX_SHIFT
) |
36 RADIX_DAX_ENTRY_LOCK
);
39 ssize_t
dax_iomap_rw(struct kiocb
*iocb
, struct iov_iter
*iter
,
40 struct iomap_ops
*ops
);
41 int dax_iomap_fault(struct vm_area_struct
*vma
, struct vm_fault
*vmf
,
42 struct iomap_ops
*ops
);
43 int dax_delete_mapping_entry(struct address_space
*mapping
, pgoff_t index
);
44 void dax_wake_mapping_entry_waiter(struct address_space
*mapping
,
45 pgoff_t index
, void *entry
, bool wake_all
);
48 struct page
*read_dax_sector(struct block_device
*bdev
, sector_t n
);
49 void dax_unlock_mapping_entry(struct address_space
*mapping
, pgoff_t index
);
50 int __dax_zero_page_range(struct block_device
*bdev
, sector_t sector
,
51 unsigned int offset
, unsigned int length
);
53 static inline struct page
*read_dax_sector(struct block_device
*bdev
,
56 return ERR_PTR(-ENXIO
);
58 /* Shouldn't ever be called when dax is disabled. */
59 static inline void dax_unlock_mapping_entry(struct address_space
*mapping
,
64 static inline int __dax_zero_page_range(struct block_device
*bdev
,
65 sector_t sector
, unsigned int offset
, unsigned int length
)
71 #ifdef CONFIG_FS_DAX_PMD
72 static inline unsigned int dax_radix_order(void *entry
)
74 if ((unsigned long)entry
& RADIX_DAX_PMD
)
75 return PMD_SHIFT
- PAGE_SHIFT
;
78 int dax_iomap_pmd_fault(struct vm_area_struct
*vma
, unsigned long address
,
79 pmd_t
*pmd
, unsigned int flags
, struct iomap_ops
*ops
);
81 static inline unsigned int dax_radix_order(void *entry
)
85 static inline int dax_iomap_pmd_fault(struct vm_area_struct
*vma
,
86 unsigned long address
, pmd_t
*pmd
, unsigned int flags
,
87 struct iomap_ops
*ops
)
89 return VM_FAULT_FALLBACK
;
92 int dax_pfn_mkwrite(struct vm_area_struct
*, struct vm_fault
*);
94 static inline bool vma_is_dax(struct vm_area_struct
*vma
)
96 return vma
->vm_file
&& IS_DAX(vma
->vm_file
->f_mapping
->host
);
99 static inline bool dax_mapping(struct address_space
*mapping
)
101 return mapping
->host
&& IS_DAX(mapping
->host
);
104 struct writeback_control
;
105 int dax_writeback_mapping_range(struct address_space
*mapping
,
106 struct block_device
*bdev
, struct writeback_control
*wbc
);