]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/mlx5/mem.c
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_umem_odp.h>
38 /* @umem: umem object to scan
39 * @addr: ib virtual address requested by the user
40 * @max_page_shift: high limit for page_shift - 0 means no limit
41 * @count: number of PAGE_SIZE pages covered by umem
42 * @shift: page shift for the compound pages found in the region
43 * @ncont: number of compund pages
44 * @order: log2 of the number of compound pages
46 void mlx5_ib_cont_pages(struct ib_umem
*umem
, u64 addr
,
47 unsigned long max_page_shift
,
48 int *count
, int *shift
,
49 int *ncont
, int *order
)
60 struct scatterlist
*sg
;
62 unsigned long page_shift
= ilog2(umem
->page_size
);
64 /* With ODP we must always match OS page size. */
66 *count
= ib_umem_page_count(umem
);
70 *order
= ilog2(roundup_pow_of_two(*count
));
75 addr
= addr
>> page_shift
;
76 tmp
= (unsigned long)addr
;
77 m
= find_first_bit(&tmp
, BITS_PER_LONG
);
79 m
= min_t(unsigned long, max_page_shift
- page_shift
, m
);
83 for_each_sg(umem
->sg_head
.sgl
, sg
, umem
->nmap
, entry
) {
84 len
= sg_dma_len(sg
) >> page_shift
;
85 pfn
= sg_dma_address(sg
) >> page_shift
;
86 for (k
= 0; k
< len
; k
++) {
88 tmp
= (unsigned long)pfn
;
89 m
= min_t(unsigned long, m
, find_first_bit(&tmp
, BITS_PER_LONG
));
95 if (base
+ p
!= pfn
) {
96 tmp
= (unsigned long)p
;
97 m
= find_first_bit(&tmp
, BITS_PER_LONG
);
110 m
= min_t(unsigned long, ilog2(roundup_pow_of_two(i
)), m
);
113 *order
= ilog2(roundup_pow_of_two(i
) >> m
);
115 *ncont
= DIV_ROUND_UP(i
, (1 << m
));
124 *shift
= page_shift
+ m
;
128 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
129 static u64
umem_dma_to_mtt(dma_addr_t umem_dma
)
131 u64 mtt_entry
= umem_dma
& ODP_DMA_ADDR_MASK
;
133 if (umem_dma
& ODP_READ_ALLOWED_BIT
)
134 mtt_entry
|= MLX5_IB_MTT_READ
;
135 if (umem_dma
& ODP_WRITE_ALLOWED_BIT
)
136 mtt_entry
|= MLX5_IB_MTT_WRITE
;
143 * Populate the given array with bus addresses from the umem.
145 * dev - mlx5_ib device
146 * umem - umem to use to fill the pages
147 * page_shift - determines the page size used in the resulting array
148 * offset - offset into the umem to start from,
149 * only implemented for ODP umems
150 * num_pages - total number of pages to fill
151 * pas - bus addresses array to fill
152 * access_flags - access flags to set on all present pages.
153 use enum mlx5_ib_mtt_access_flags for this.
155 void __mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
156 int page_shift
, size_t offset
, size_t num_pages
,
157 __be64
*pas
, int access_flags
)
159 unsigned long umem_page_shift
= ilog2(umem
->page_size
);
160 int shift
= page_shift
- umem_page_shift
;
161 int mask
= (1 << shift
) - 1;
166 struct scatterlist
*sg
;
168 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
169 const bool odp
= umem
->odp_data
!= NULL
;
173 WARN_ON(access_flags
!= (MLX5_IB_MTT_READ
| MLX5_IB_MTT_WRITE
));
175 for (i
= 0; i
< num_pages
; ++i
) {
176 dma_addr_t pa
= umem
->odp_data
->dma_list
[offset
+ i
];
178 pas
[i
] = cpu_to_be64(umem_dma_to_mtt(pa
));
185 for_each_sg(umem
->sg_head
.sgl
, sg
, umem
->nmap
, entry
) {
186 len
= sg_dma_len(sg
) >> umem_page_shift
;
187 base
= sg_dma_address(sg
);
189 /* Skip elements below offset */
190 if (i
+ len
< offset
<< shift
) {
195 /* Skip pages below offset */
196 if (i
< offset
<< shift
) {
197 k
= (offset
<< shift
) - i
;
203 for (; k
< len
; k
++) {
205 cur
= base
+ (k
<< umem_page_shift
);
207 idx
= (i
>> shift
) - offset
;
209 pas
[idx
] = cpu_to_be64(cur
);
210 mlx5_ib_dbg(dev
, "pas[%d] 0x%llx\n",
211 i
>> shift
, be64_to_cpu(pas
[idx
]));
215 /* Stop after num_pages reached */
216 if (i
>> shift
>= offset
+ num_pages
)
222 void mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
223 int page_shift
, __be64
*pas
, int access_flags
)
225 return __mlx5_ib_populate_pas(dev
, umem
, page_shift
, 0,
226 ib_umem_num_pages(umem
), pas
,
229 int mlx5_ib_get_buf_offset(u64 addr
, int page_shift
, u32
*offset
)
237 page_size
= (u64
)1 << page_shift
;
238 page_mask
= page_size
- 1;
239 buf_off
= addr
& page_mask
;
240 off_size
= page_size
>> 6;
241 off_mask
= off_size
- 1;
243 if (buf_off
& off_mask
)
246 *offset
= buf_off
>> ilog2(off_size
);