]>
git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/infiniband/hw/mlx5/mem.c
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/module.h>
34 #include <rdma/ib_umem.h>
35 #include <rdma/ib_umem_odp.h>
38 /* @umem: umem object to scan
39 * @addr: ib virtual address requested by the user
40 * @max_page_shift: high limit for page_shift - 0 means no limit
41 * @count: number of PAGE_SIZE pages covered by umem
42 * @shift: page shift for the compound pages found in the region
43 * @ncont: number of compund pages
44 * @order: log2 of the number of compound pages
46 void mlx5_ib_cont_pages(struct ib_umem
*umem
, u64 addr
,
47 unsigned long max_page_shift
,
48 int *count
, int *shift
,
49 int *ncont
, int *order
)
60 struct scatterlist
*sg
;
62 unsigned long page_shift
= umem
->page_shift
;
65 *ncont
= ib_umem_page_count(umem
);
66 *count
= *ncont
<< (page_shift
- PAGE_SHIFT
);
69 *order
= ilog2(roundup_pow_of_two(*ncont
));
74 addr
= addr
>> page_shift
;
75 tmp
= (unsigned long)addr
;
76 m
= find_first_bit(&tmp
, BITS_PER_LONG
);
78 m
= min_t(unsigned long, max_page_shift
- page_shift
, m
);
82 for_each_sg(umem
->sg_head
.sgl
, sg
, umem
->nmap
, entry
) {
83 len
= sg_dma_len(sg
) >> page_shift
;
84 pfn
= sg_dma_address(sg
) >> page_shift
;
85 for (k
= 0; k
< len
; k
++) {
87 tmp
= (unsigned long)pfn
;
88 m
= min_t(unsigned long, m
, find_first_bit(&tmp
, BITS_PER_LONG
));
94 if (base
+ p
!= pfn
) {
95 tmp
= (unsigned long)p
;
96 m
= find_first_bit(&tmp
, BITS_PER_LONG
);
109 m
= min_t(unsigned long, ilog2(roundup_pow_of_two(i
)), m
);
112 *order
= ilog2(roundup_pow_of_two(i
) >> m
);
114 *ncont
= DIV_ROUND_UP(i
, (1 << m
));
123 *shift
= page_shift
+ m
;
127 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
128 static u64
umem_dma_to_mtt(dma_addr_t umem_dma
)
130 u64 mtt_entry
= umem_dma
& ODP_DMA_ADDR_MASK
;
132 if (umem_dma
& ODP_READ_ALLOWED_BIT
)
133 mtt_entry
|= MLX5_IB_MTT_READ
;
134 if (umem_dma
& ODP_WRITE_ALLOWED_BIT
)
135 mtt_entry
|= MLX5_IB_MTT_WRITE
;
142 * Populate the given array with bus addresses from the umem.
144 * dev - mlx5_ib device
145 * umem - umem to use to fill the pages
146 * page_shift - determines the page size used in the resulting array
147 * offset - offset into the umem to start from,
148 * only implemented for ODP umems
149 * num_pages - total number of pages to fill
150 * pas - bus addresses array to fill
151 * access_flags - access flags to set on all present pages.
152 use enum mlx5_ib_mtt_access_flags for this.
154 void __mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
155 int page_shift
, size_t offset
, size_t num_pages
,
156 __be64
*pas
, int access_flags
)
158 unsigned long umem_page_shift
= umem
->page_shift
;
159 int shift
= page_shift
- umem_page_shift
;
160 int mask
= (1 << shift
) - 1;
165 struct scatterlist
*sg
;
167 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
168 const bool odp
= umem
->odp_data
!= NULL
;
172 WARN_ON(access_flags
!= (MLX5_IB_MTT_READ
| MLX5_IB_MTT_WRITE
));
174 for (i
= 0; i
< num_pages
; ++i
) {
175 dma_addr_t pa
= umem
->odp_data
->dma_list
[offset
+ i
];
177 pas
[i
] = cpu_to_be64(umem_dma_to_mtt(pa
));
184 for_each_sg(umem
->sg_head
.sgl
, sg
, umem
->nmap
, entry
) {
185 len
= sg_dma_len(sg
) >> umem_page_shift
;
186 base
= sg_dma_address(sg
);
188 /* Skip elements below offset */
189 if (i
+ len
< offset
<< shift
) {
194 /* Skip pages below offset */
195 if (i
< offset
<< shift
) {
196 k
= (offset
<< shift
) - i
;
202 for (; k
< len
; k
++) {
204 cur
= base
+ (k
<< umem_page_shift
);
206 idx
= (i
>> shift
) - offset
;
208 pas
[idx
] = cpu_to_be64(cur
);
209 mlx5_ib_dbg(dev
, "pas[%d] 0x%llx\n",
210 i
>> shift
, be64_to_cpu(pas
[idx
]));
214 /* Stop after num_pages reached */
215 if (i
>> shift
>= offset
+ num_pages
)
221 void mlx5_ib_populate_pas(struct mlx5_ib_dev
*dev
, struct ib_umem
*umem
,
222 int page_shift
, __be64
*pas
, int access_flags
)
224 return __mlx5_ib_populate_pas(dev
, umem
, page_shift
, 0,
225 ib_umem_num_pages(umem
), pas
,
228 int mlx5_ib_get_buf_offset(u64 addr
, int page_shift
, u32
*offset
)
236 page_size
= (u64
)1 << page_shift
;
237 page_mask
= page_size
- 1;
238 buf_off
= addr
& page_mask
;
239 off_size
= page_size
>> 6;
240 off_mask
= off_size
- 1;
242 if (buf_off
& off_mask
)
245 *offset
= buf_off
>> ilog2(off_size
);