]>
Commit | Line | Data |
---|---|---|
a1bd3bae AH |
1 | /* |
2 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
3 | * redistributing this file, you may do so under either license. | |
4 | * | |
5 | * GPL LICENSE SUMMARY | |
6 | * | |
7 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. | |
443b9a14 | 8 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
a1bd3bae AH |
9 | * |
10 | * This program is free software; you can redistribute it and/or modify | |
11 | * it under the terms of version 2 of the GNU General Public License as | |
12 | * published by the Free Software Foundation. | |
13 | * | |
14 | * This program is distributed in the hope that it will be useful, but | |
15 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * General Public License for more details. | |
18 | * | |
19 | * BSD LICENSE | |
20 | * | |
21 | * Copyright (C) 2015 EMC Corporation. All Rights Reserved. | |
443b9a14 | 22 | * Copyright (C) 2016 T-Platforms. All Rights Reserved. |
a1bd3bae AH |
23 | * |
24 | * Redistribution and use in source and binary forms, with or without | |
25 | * modification, are permitted provided that the following conditions | |
26 | * are met: | |
27 | * | |
28 | * * Redistributions of source code must retain the above copyright | |
29 | * notice, this list of conditions and the following disclaimer. | |
30 | * * Redistributions in binary form must reproduce the above copy | |
31 | * notice, this list of conditions and the following disclaimer in | |
32 | * the documentation and/or other materials provided with the | |
33 | * distribution. | |
34 | * * Neither the name of Intel Corporation nor the names of its | |
35 | * contributors may be used to endorse or promote products derived | |
36 | * from this software without specific prior written permission. | |
37 | * | |
38 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
39 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
40 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
41 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
42 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
43 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
44 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
45 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
46 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
47 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
48 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
49 | * | |
50 | * PCIe NTB Linux driver | |
51 | * | |
52 | * Contact Information: | |
53 | * Allen Hubbe <Allen.Hubbe@emc.com> | |
54 | */ | |
55 | ||
56 | #ifndef _NTB_H_ | |
57 | #define _NTB_H_ | |
58 | ||
59 | #include <linux/completion.h> | |
60 | #include <linux/device.h> | |
61 | ||
62 | struct ntb_client; | |
63 | struct ntb_dev; | |
64 | struct pci_dev; | |
65 | ||
66 | /** | |
67 | * enum ntb_topo - NTB connection topology | |
68 | * @NTB_TOPO_NONE: Topology is unknown or invalid. | |
69 | * @NTB_TOPO_PRI: On primary side of local ntb. | |
70 | * @NTB_TOPO_SEC: On secondary side of remote ntb. | |
71 | * @NTB_TOPO_B2B_USD: On primary side of local ntb upstream of remote ntb. | |
72 | * @NTB_TOPO_B2B_DSD: On primary side of local ntb downstream of remote ntb. | |
e099b45b | 73 | * @NTB_TOPO_SWITCH: Connected via a switch which supports ntb. |
a1bd3bae AH |
74 | */ |
75 | enum ntb_topo { | |
76 | NTB_TOPO_NONE = -1, | |
77 | NTB_TOPO_PRI, | |
78 | NTB_TOPO_SEC, | |
79 | NTB_TOPO_B2B_USD, | |
80 | NTB_TOPO_B2B_DSD, | |
e099b45b | 81 | NTB_TOPO_SWITCH, |
a1bd3bae AH |
82 | }; |
83 | ||
84 | static inline int ntb_topo_is_b2b(enum ntb_topo topo) | |
85 | { | |
86 | switch ((int)topo) { | |
87 | case NTB_TOPO_B2B_USD: | |
88 | case NTB_TOPO_B2B_DSD: | |
89 | return 1; | |
90 | } | |
91 | return 0; | |
92 | } | |
93 | ||
94 | static inline char *ntb_topo_string(enum ntb_topo topo) | |
95 | { | |
96 | switch (topo) { | |
97 | case NTB_TOPO_NONE: return "NTB_TOPO_NONE"; | |
98 | case NTB_TOPO_PRI: return "NTB_TOPO_PRI"; | |
99 | case NTB_TOPO_SEC: return "NTB_TOPO_SEC"; | |
100 | case NTB_TOPO_B2B_USD: return "NTB_TOPO_B2B_USD"; | |
101 | case NTB_TOPO_B2B_DSD: return "NTB_TOPO_B2B_DSD"; | |
e099b45b | 102 | case NTB_TOPO_SWITCH: return "NTB_TOPO_SWITCH"; |
a1bd3bae AH |
103 | } |
104 | return "NTB_TOPO_INVALID"; | |
105 | } | |
106 | ||
107 | /** | |
108 | * enum ntb_speed - NTB link training speed | |
109 | * @NTB_SPEED_AUTO: Request the max supported speed. | |
110 | * @NTB_SPEED_NONE: Link is not trained to any speed. | |
111 | * @NTB_SPEED_GEN1: Link is trained to gen1 speed. | |
112 | * @NTB_SPEED_GEN2: Link is trained to gen2 speed. | |
113 | * @NTB_SPEED_GEN3: Link is trained to gen3 speed. | |
85dce3aa | 114 | * @NTB_SPEED_GEN4: Link is trained to gen4 speed. |
a1bd3bae AH |
115 | */ |
116 | enum ntb_speed { | |
117 | NTB_SPEED_AUTO = -1, | |
118 | NTB_SPEED_NONE = 0, | |
119 | NTB_SPEED_GEN1 = 1, | |
120 | NTB_SPEED_GEN2 = 2, | |
121 | NTB_SPEED_GEN3 = 3, | |
85dce3aa | 122 | NTB_SPEED_GEN4 = 4 |
a1bd3bae AH |
123 | }; |
124 | ||
125 | /** | |
126 | * enum ntb_width - NTB link training width | |
127 | * @NTB_WIDTH_AUTO: Request the max supported width. | |
128 | * @NTB_WIDTH_NONE: Link is not trained to any width. | |
129 | * @NTB_WIDTH_1: Link is trained to 1 lane width. | |
130 | * @NTB_WIDTH_2: Link is trained to 2 lane width. | |
131 | * @NTB_WIDTH_4: Link is trained to 4 lane width. | |
132 | * @NTB_WIDTH_8: Link is trained to 8 lane width. | |
133 | * @NTB_WIDTH_12: Link is trained to 12 lane width. | |
134 | * @NTB_WIDTH_16: Link is trained to 16 lane width. | |
135 | * @NTB_WIDTH_32: Link is trained to 32 lane width. | |
136 | */ | |
137 | enum ntb_width { | |
138 | NTB_WIDTH_AUTO = -1, | |
139 | NTB_WIDTH_NONE = 0, | |
140 | NTB_WIDTH_1 = 1, | |
141 | NTB_WIDTH_2 = 2, | |
142 | NTB_WIDTH_4 = 4, | |
143 | NTB_WIDTH_8 = 8, | |
144 | NTB_WIDTH_12 = 12, | |
145 | NTB_WIDTH_16 = 16, | |
146 | NTB_WIDTH_32 = 32, | |
147 | }; | |
148 | ||
1e530119 SS |
149 | /** |
150 | * enum ntb_default_port - NTB default port number | |
151 | * @NTB_PORT_PRI_USD: Default port of the NTB_TOPO_PRI/NTB_TOPO_B2B_USD | |
152 | * topologies | |
153 | * @NTB_PORT_SEC_DSD: Default port of the NTB_TOPO_SEC/NTB_TOPO_B2B_DSD | |
154 | * topologies | |
155 | */ | |
156 | enum ntb_default_port { | |
157 | NTB_PORT_PRI_USD, | |
158 | NTB_PORT_SEC_DSD | |
159 | }; | |
160 | #define NTB_DEF_PEER_CNT (1) | |
161 | #define NTB_DEF_PEER_IDX (0) | |
162 | ||
a1bd3bae AH |
163 | /** |
164 | * struct ntb_client_ops - ntb client operations | |
165 | * @probe: Notify client of a new device. | |
166 | * @remove: Notify client to remove a device. | |
167 | */ | |
168 | struct ntb_client_ops { | |
169 | int (*probe)(struct ntb_client *client, struct ntb_dev *ntb); | |
170 | void (*remove)(struct ntb_client *client, struct ntb_dev *ntb); | |
171 | }; | |
172 | ||
173 | static inline int ntb_client_ops_is_valid(const struct ntb_client_ops *ops) | |
174 | { | |
175 | /* commented callbacks are not required: */ | |
176 | return | |
177 | ops->probe && | |
178 | ops->remove && | |
179 | 1; | |
180 | } | |
181 | ||
182 | /** | |
183 | * struct ntb_ctx_ops - ntb driver context operations | |
184 | * @link_event: See ntb_link_event(). | |
185 | * @db_event: See ntb_db_event(). | |
bc3e49ad | 186 | * @msg_event: See ntb_msg_event(). |
a1bd3bae AH |
187 | */ |
188 | struct ntb_ctx_ops { | |
189 | void (*link_event)(void *ctx); | |
190 | void (*db_event)(void *ctx, int db_vector); | |
bc3e49ad | 191 | void (*msg_event)(void *ctx); |
a1bd3bae AH |
192 | }; |
193 | ||
194 | static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops) | |
195 | { | |
196 | /* commented callbacks are not required: */ | |
197 | return | |
198 | /* ops->link_event && */ | |
199 | /* ops->db_event && */ | |
bc3e49ad | 200 | /* ops->msg_event && */ |
a1bd3bae AH |
201 | 1; |
202 | } | |
203 | ||
204 | /** | |
205 | * struct ntb_ctx_ops - ntb device operations | |
1e530119 SS |
206 | * @port_number: See ntb_port_number(). |
207 | * @peer_port_count: See ntb_peer_port_count(). | |
208 | * @peer_port_number: See ntb_peer_port_number(). | |
209 | * @peer_port_idx: See ntb_peer_port_idx(). | |
60934b20 SS |
210 | * @link_is_up: See ntb_link_is_up(). |
211 | * @link_enable: See ntb_link_enable(). | |
212 | * @link_disable: See ntb_link_disable(). | |
a1bd3bae | 213 | * @mw_count: See ntb_mw_count(). |
443b9a14 | 214 | * @mw_get_align: See ntb_mw_get_align(). |
a1bd3bae AH |
215 | * @mw_set_trans: See ntb_mw_set_trans(). |
216 | * @mw_clear_trans: See ntb_mw_clear_trans(). | |
443b9a14 SS |
217 | * @peer_mw_count: See ntb_peer_mw_count(). |
218 | * @peer_mw_get_addr: See ntb_peer_mw_get_addr(). | |
219 | * @peer_mw_set_trans: See ntb_peer_mw_set_trans(). | |
220 | * @peer_mw_clear_trans:See ntb_peer_mw_clear_trans(). | |
a1bd3bae AH |
221 | * @db_is_unsafe: See ntb_db_is_unsafe(). |
222 | * @db_valid_mask: See ntb_db_valid_mask(). | |
223 | * @db_vector_count: See ntb_db_vector_count(). | |
224 | * @db_vector_mask: See ntb_db_vector_mask(). | |
225 | * @db_read: See ntb_db_read(). | |
226 | * @db_set: See ntb_db_set(). | |
227 | * @db_clear: See ntb_db_clear(). | |
228 | * @db_read_mask: See ntb_db_read_mask(). | |
229 | * @db_set_mask: See ntb_db_set_mask(). | |
230 | * @db_clear_mask: See ntb_db_clear_mask(). | |
231 | * @peer_db_addr: See ntb_peer_db_addr(). | |
232 | * @peer_db_read: See ntb_peer_db_read(). | |
233 | * @peer_db_set: See ntb_peer_db_set(). | |
234 | * @peer_db_clear: See ntb_peer_db_clear(). | |
235 | * @peer_db_read_mask: See ntb_peer_db_read_mask(). | |
236 | * @peer_db_set_mask: See ntb_peer_db_set_mask(). | |
237 | * @peer_db_clear_mask: See ntb_peer_db_clear_mask(). | |
238 | * @spad_is_unsafe: See ntb_spad_is_unsafe(). | |
239 | * @spad_count: See ntb_spad_count(). | |
240 | * @spad_read: See ntb_spad_read(). | |
241 | * @spad_write: See ntb_spad_write(). | |
242 | * @peer_spad_addr: See ntb_peer_spad_addr(). | |
243 | * @peer_spad_read: See ntb_peer_spad_read(). | |
244 | * @peer_spad_write: See ntb_peer_spad_write(). | |
bc3e49ad SS |
245 | * @msg_count: See ntb_msg_count(). |
246 | * @msg_inbits: See ntb_msg_inbits(). | |
247 | * @msg_outbits: See ntb_msg_outbits(). | |
248 | * @msg_read_sts: See ntb_msg_read_sts(). | |
249 | * @msg_clear_sts: See ntb_msg_clear_sts(). | |
250 | * @msg_set_mask: See ntb_msg_set_mask(). | |
251 | * @msg_clear_mask: See ntb_msg_clear_mask(). | |
252 | * @msg_read: See ntb_msg_read(). | |
253 | * @msg_write: See ntb_msg_write(). | |
a1bd3bae AH |
254 | */ |
255 | struct ntb_dev_ops { | |
1e530119 SS |
256 | int (*port_number)(struct ntb_dev *ntb); |
257 | int (*peer_port_count)(struct ntb_dev *ntb); | |
258 | int (*peer_port_number)(struct ntb_dev *ntb, int pidx); | |
259 | int (*peer_port_idx)(struct ntb_dev *ntb, int port); | |
260 | ||
4e8c11b7 | 261 | u64 (*link_is_up)(struct ntb_dev *ntb, |
60934b20 SS |
262 | enum ntb_speed *speed, enum ntb_width *width); |
263 | int (*link_enable)(struct ntb_dev *ntb, | |
264 | enum ntb_speed max_speed, enum ntb_width max_width); | |
265 | int (*link_disable)(struct ntb_dev *ntb); | |
266 | ||
443b9a14 SS |
267 | int (*mw_count)(struct ntb_dev *ntb, int pidx); |
268 | int (*mw_get_align)(struct ntb_dev *ntb, int pidx, int widx, | |
269 | resource_size_t *addr_align, | |
270 | resource_size_t *size_align, | |
271 | resource_size_t *size_max); | |
272 | int (*mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx, | |
a1bd3bae | 273 | dma_addr_t addr, resource_size_t size); |
443b9a14 SS |
274 | int (*mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx); |
275 | int (*peer_mw_count)(struct ntb_dev *ntb); | |
276 | int (*peer_mw_get_addr)(struct ntb_dev *ntb, int widx, | |
277 | phys_addr_t *base, resource_size_t *size); | |
278 | int (*peer_mw_set_trans)(struct ntb_dev *ntb, int pidx, int widx, | |
279 | u64 addr, resource_size_t size); | |
280 | int (*peer_mw_clear_trans)(struct ntb_dev *ntb, int pidx, int widx); | |
a1bd3bae | 281 | |
a1bd3bae AH |
282 | int (*db_is_unsafe)(struct ntb_dev *ntb); |
283 | u64 (*db_valid_mask)(struct ntb_dev *ntb); | |
284 | int (*db_vector_count)(struct ntb_dev *ntb); | |
285 | u64 (*db_vector_mask)(struct ntb_dev *ntb, int db_vector); | |
286 | ||
287 | u64 (*db_read)(struct ntb_dev *ntb); | |
288 | int (*db_set)(struct ntb_dev *ntb, u64 db_bits); | |
289 | int (*db_clear)(struct ntb_dev *ntb, u64 db_bits); | |
290 | ||
291 | u64 (*db_read_mask)(struct ntb_dev *ntb); | |
292 | int (*db_set_mask)(struct ntb_dev *ntb, u64 db_bits); | |
293 | int (*db_clear_mask)(struct ntb_dev *ntb, u64 db_bits); | |
294 | ||
295 | int (*peer_db_addr)(struct ntb_dev *ntb, | |
296 | phys_addr_t *db_addr, resource_size_t *db_size); | |
297 | u64 (*peer_db_read)(struct ntb_dev *ntb); | |
298 | int (*peer_db_set)(struct ntb_dev *ntb, u64 db_bits); | |
299 | int (*peer_db_clear)(struct ntb_dev *ntb, u64 db_bits); | |
300 | ||
301 | u64 (*peer_db_read_mask)(struct ntb_dev *ntb); | |
302 | int (*peer_db_set_mask)(struct ntb_dev *ntb, u64 db_bits); | |
303 | int (*peer_db_clear_mask)(struct ntb_dev *ntb, u64 db_bits); | |
304 | ||
305 | int (*spad_is_unsafe)(struct ntb_dev *ntb); | |
306 | int (*spad_count)(struct ntb_dev *ntb); | |
307 | ||
d67288a3 SS |
308 | u32 (*spad_read)(struct ntb_dev *ntb, int sidx); |
309 | int (*spad_write)(struct ntb_dev *ntb, int sidx, u32 val); | |
a1bd3bae | 310 | |
d67288a3 | 311 | int (*peer_spad_addr)(struct ntb_dev *ntb, int pidx, int sidx, |
a1bd3bae | 312 | phys_addr_t *spad_addr); |
d67288a3 SS |
313 | u32 (*peer_spad_read)(struct ntb_dev *ntb, int pidx, int sidx); |
314 | int (*peer_spad_write)(struct ntb_dev *ntb, int pidx, int sidx, | |
315 | u32 val); | |
bc3e49ad SS |
316 | |
317 | int (*msg_count)(struct ntb_dev *ntb); | |
318 | u64 (*msg_inbits)(struct ntb_dev *ntb); | |
319 | u64 (*msg_outbits)(struct ntb_dev *ntb); | |
320 | u64 (*msg_read_sts)(struct ntb_dev *ntb); | |
321 | int (*msg_clear_sts)(struct ntb_dev *ntb, u64 sts_bits); | |
322 | int (*msg_set_mask)(struct ntb_dev *ntb, u64 mask_bits); | |
323 | int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits); | |
324 | int (*msg_read)(struct ntb_dev *ntb, int midx, int *pidx, u32 *msg); | |
325 | int (*msg_write)(struct ntb_dev *ntb, int midx, int pidx, u32 msg); | |
a1bd3bae AH |
326 | }; |
327 | ||
328 | static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) | |
329 | { | |
330 | /* commented callbacks are not required: */ | |
331 | return | |
3c69f5d6 | 332 | /* Port operations are required for multiport devices */ |
1e530119 SS |
333 | !ops->peer_port_count == !ops->port_number && |
334 | !ops->peer_port_number == !ops->port_number && | |
335 | !ops->peer_port_idx == !ops->port_number && | |
3c69f5d6 SS |
336 | |
337 | /* Link operations are required */ | |
60934b20 SS |
338 | ops->link_is_up && |
339 | ops->link_enable && | |
340 | ops->link_disable && | |
3c69f5d6 SS |
341 | |
342 | /* One or both MW interfaces should be developed */ | |
a1bd3bae | 343 | ops->mw_count && |
443b9a14 SS |
344 | ops->mw_get_align && |
345 | (ops->mw_set_trans || | |
346 | ops->peer_mw_set_trans) && | |
a1bd3bae | 347 | /* ops->mw_clear_trans && */ |
443b9a14 SS |
348 | ops->peer_mw_count && |
349 | ops->peer_mw_get_addr && | |
350 | /* ops->peer_mw_clear_trans && */ | |
60934b20 | 351 | |
3c69f5d6 | 352 | /* Doorbell operations are mostly required */ |
a1bd3bae AH |
353 | /* ops->db_is_unsafe && */ |
354 | ops->db_valid_mask && | |
a1bd3bae | 355 | /* both set, or both unset */ |
3c69f5d6 | 356 | (!ops->db_vector_count == !ops->db_vector_mask) && |
a1bd3bae AH |
357 | ops->db_read && |
358 | /* ops->db_set && */ | |
359 | ops->db_clear && | |
360 | /* ops->db_read_mask && */ | |
361 | ops->db_set_mask && | |
362 | ops->db_clear_mask && | |
afc54992 | 363 | /* ops->peer_db_addr && */ |
a1bd3bae AH |
364 | /* ops->peer_db_read && */ |
365 | ops->peer_db_set && | |
366 | /* ops->peer_db_clear && */ | |
367 | /* ops->peer_db_read_mask && */ | |
368 | /* ops->peer_db_set_mask && */ | |
369 | /* ops->peer_db_clear_mask && */ | |
3c69f5d6 SS |
370 | |
371 | /* Scrachpads interface is optional */ | |
d67288a3 SS |
372 | /* !ops->spad_is_unsafe == !ops->spad_count && */ |
373 | !ops->spad_read == !ops->spad_count && | |
374 | !ops->spad_write == !ops->spad_count && | |
375 | /* !ops->peer_spad_addr == !ops->spad_count && */ | |
376 | /* !ops->peer_spad_read == !ops->spad_count && */ | |
377 | !ops->peer_spad_write == !ops->spad_count && | |
bc3e49ad | 378 | |
3c69f5d6 | 379 | /* Messaging interface is optional */ |
bc3e49ad SS |
380 | !ops->msg_inbits == !ops->msg_count && |
381 | !ops->msg_outbits == !ops->msg_count && | |
382 | !ops->msg_read_sts == !ops->msg_count && | |
383 | !ops->msg_clear_sts == !ops->msg_count && | |
384 | /* !ops->msg_set_mask == !ops->msg_count && */ | |
385 | /* !ops->msg_clear_mask == !ops->msg_count && */ | |
386 | !ops->msg_read == !ops->msg_count && | |
387 | !ops->msg_write == !ops->msg_count && | |
a1bd3bae AH |
388 | 1; |
389 | } | |
390 | ||
391 | /** | |
392 | * struct ntb_client - client interested in ntb devices | |
393 | * @drv: Linux driver object. | |
394 | * @ops: See &ntb_client_ops. | |
395 | */ | |
396 | struct ntb_client { | |
397 | struct device_driver drv; | |
398 | const struct ntb_client_ops ops; | |
399 | }; | |
a1bd3bae AH |
400 | #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv) |
401 | ||
402 | /** | |
403 | * struct ntb_device - ntb device | |
404 | * @dev: Linux device object. | |
3c69f5d6 | 405 | * @pdev: PCI device entry of the ntb. |
a1bd3bae AH |
406 | * @topo: Detected topology of the ntb. |
407 | * @ops: See &ntb_dev_ops. | |
408 | * @ctx: See &ntb_ctx_ops. | |
409 | * @ctx_ops: See &ntb_ctx_ops. | |
410 | */ | |
411 | struct ntb_dev { | |
412 | struct device dev; | |
413 | struct pci_dev *pdev; | |
414 | enum ntb_topo topo; | |
415 | const struct ntb_dev_ops *ops; | |
416 | void *ctx; | |
417 | const struct ntb_ctx_ops *ctx_ops; | |
418 | ||
419 | /* private: */ | |
420 | ||
421 | /* synchronize setting, clearing, and calling ctx_ops */ | |
422 | spinlock_t ctx_lock; | |
423 | /* block unregister until device is fully released */ | |
424 | struct completion released; | |
425 | }; | |
a1bd3bae AH |
426 | #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev) |
427 | ||
428 | /** | |
429 | * ntb_register_client() - register a client for interest in ntb devices | |
430 | * @client: Client context. | |
431 | * | |
432 | * The client will be added to the list of clients interested in ntb devices. | |
433 | * The client will be notified of any ntb devices that are not already | |
434 | * associated with a client, or if ntb devices are registered later. | |
435 | * | |
436 | * Return: Zero if the client is registered, otherwise an error number. | |
437 | */ | |
438 | #define ntb_register_client(client) \ | |
439 | __ntb_register_client((client), THIS_MODULE, KBUILD_MODNAME) | |
440 | ||
441 | int __ntb_register_client(struct ntb_client *client, struct module *mod, | |
442 | const char *mod_name); | |
443 | ||
444 | /** | |
445 | * ntb_unregister_client() - unregister a client for interest in ntb devices | |
446 | * @client: Client context. | |
447 | * | |
448 | * The client will be removed from the list of clients interested in ntb | |
449 | * devices. If any ntb devices are associated with the client, the client will | |
450 | * be notified to remove those devices. | |
451 | */ | |
452 | void ntb_unregister_client(struct ntb_client *client); | |
453 | ||
454 | #define module_ntb_client(__ntb_client) \ | |
455 | module_driver(__ntb_client, ntb_register_client, \ | |
456 | ntb_unregister_client) | |
457 | ||
458 | /** | |
459 | * ntb_register_device() - register a ntb device | |
460 | * @ntb: NTB device context. | |
461 | * | |
462 | * The device will be added to the list of ntb devices. If any clients are | |
463 | * interested in ntb devices, each client will be notified of the ntb device, | |
464 | * until at most one client accepts the device. | |
465 | * | |
466 | * Return: Zero if the device is registered, otherwise an error number. | |
467 | */ | |
468 | int ntb_register_device(struct ntb_dev *ntb); | |
469 | ||
470 | /** | |
471 | * ntb_register_device() - unregister a ntb device | |
472 | * @ntb: NTB device context. | |
473 | * | |
474 | * The device will be removed from the list of ntb devices. If the ntb device | |
475 | * is associated with a client, the client will be notified to remove the | |
476 | * device. | |
477 | */ | |
478 | void ntb_unregister_device(struct ntb_dev *ntb); | |
479 | ||
480 | /** | |
481 | * ntb_set_ctx() - associate a driver context with an ntb device | |
482 | * @ntb: NTB device context. | |
483 | * @ctx: Driver context. | |
484 | * @ctx_ops: Driver context operations. | |
485 | * | |
486 | * Associate a driver context and operations with a ntb device. The context is | |
487 | * provided by the client driver, and the driver may associate a different | |
488 | * context with each ntb device. | |
489 | * | |
490 | * Return: Zero if the context is associated, otherwise an error number. | |
491 | */ | |
492 | int ntb_set_ctx(struct ntb_dev *ntb, void *ctx, | |
493 | const struct ntb_ctx_ops *ctx_ops); | |
494 | ||
495 | /** | |
496 | * ntb_clear_ctx() - disassociate any driver context from an ntb device | |
497 | * @ntb: NTB device context. | |
498 | * | |
499 | * Clear any association that may exist between a driver context and the ntb | |
500 | * device. | |
501 | */ | |
502 | void ntb_clear_ctx(struct ntb_dev *ntb); | |
503 | ||
504 | /** | |
505 | * ntb_link_event() - notify driver context of a change in link status | |
506 | * @ntb: NTB device context. | |
507 | * | |
508 | * Notify the driver context that the link status may have changed. The driver | |
509 | * should call ntb_link_is_up() to get the current status. | |
510 | */ | |
511 | void ntb_link_event(struct ntb_dev *ntb); | |
512 | ||
513 | /** | |
514 | * ntb_db_event() - notify driver context of a doorbell event | |
515 | * @ntb: NTB device context. | |
516 | * @vector: Interrupt vector number. | |
517 | * | |
518 | * Notify the driver context of a doorbell event. If hardware supports | |
519 | * multiple interrupt vectors for doorbells, the vector number indicates which | |
520 | * vector received the interrupt. The vector number is relative to the first | |
521 | * vector used for doorbells, starting at zero, and must be less than | |
3c69f5d6 | 522 | * ntb_db_vector_count(). The driver may call ntb_db_read() to check which |
a1bd3bae AH |
523 | * doorbell bits need service, and ntb_db_vector_mask() to determine which of |
524 | * those bits are associated with the vector number. | |
525 | */ | |
526 | void ntb_db_event(struct ntb_dev *ntb, int vector); | |
527 | ||
bc3e49ad SS |
528 | /** |
529 | * ntb_msg_event() - notify driver context of a message event | |
530 | * @ntb: NTB device context. | |
531 | * | |
532 | * Notify the driver context of a message event. If hardware supports | |
533 | * message registers, this event indicates, that a new message arrived in | |
534 | * some incoming message register or last sent message couldn't be delivered. | |
535 | * The events can be masked/unmasked by the methods ntb_msg_set_mask() and | |
536 | * ntb_msg_clear_mask(). | |
537 | */ | |
538 | void ntb_msg_event(struct ntb_dev *ntb); | |
539 | ||
1e530119 SS |
540 | /** |
541 | * ntb_default_port_number() - get the default local port number | |
542 | * @ntb: NTB device context. | |
543 | * | |
544 | * If hardware driver doesn't specify port_number() callback method, the NTB | |
545 | * is considered with just two ports. So this method returns default local | |
546 | * port number in compliance with topology. | |
547 | * | |
548 | * NOTE Don't call this method directly. The ntb_port_number() function should | |
549 | * be used instead. | |
550 | * | |
551 | * Return: the default local port number | |
552 | */ | |
553 | int ntb_default_port_number(struct ntb_dev *ntb); | |
554 | ||
555 | /** | |
556 | * ntb_default_port_count() - get the default number of peer device ports | |
557 | * @ntb: NTB device context. | |
558 | * | |
559 | * By default hardware driver supports just one peer device. | |
560 | * | |
561 | * NOTE Don't call this method directly. The ntb_peer_port_count() function | |
562 | * should be used instead. | |
563 | * | |
564 | * Return: the default number of peer ports | |
565 | */ | |
566 | int ntb_default_peer_port_count(struct ntb_dev *ntb); | |
567 | ||
568 | /** | |
569 | * ntb_default_peer_port_number() - get the default peer port by given index | |
570 | * @ntb: NTB device context. | |
571 | * @idx: Peer port index (should not differ from zero). | |
572 | * | |
573 | * By default hardware driver supports just one peer device, so this method | |
574 | * shall return the corresponding value from enum ntb_default_port. | |
575 | * | |
576 | * NOTE Don't call this method directly. The ntb_peer_port_number() function | |
577 | * should be used instead. | |
578 | * | |
579 | * Return: the peer device port or negative value indicating an error | |
580 | */ | |
581 | int ntb_default_peer_port_number(struct ntb_dev *ntb, int pidx); | |
582 | ||
583 | /** | |
584 | * ntb_default_peer_port_idx() - get the default peer device port index by | |
585 | * given port number | |
586 | * @ntb: NTB device context. | |
587 | * @port: Peer port number (should be one of enum ntb_default_port). | |
588 | * | |
589 | * By default hardware driver supports just one peer device, so while | |
590 | * specified port-argument indicates peer port from enum ntb_default_port, | |
591 | * the return value shall be zero. | |
592 | * | |
593 | * NOTE Don't call this method directly. The ntb_peer_port_idx() function | |
594 | * should be used instead. | |
595 | * | |
596 | * Return: the peer port index or negative value indicating an error | |
597 | */ | |
598 | int ntb_default_peer_port_idx(struct ntb_dev *ntb, int port); | |
599 | ||
600 | /** | |
601 | * ntb_port_number() - get the local port number | |
602 | * @ntb: NTB device context. | |
603 | * | |
604 | * Hardware must support at least simple two-ports ntb connection | |
605 | * | |
606 | * Return: the local port number | |
607 | */ | |
608 | static inline int ntb_port_number(struct ntb_dev *ntb) | |
609 | { | |
610 | if (!ntb->ops->port_number) | |
611 | return ntb_default_port_number(ntb); | |
612 | ||
613 | return ntb->ops->port_number(ntb); | |
614 | } | |
615 | ||
616 | /** | |
617 | * ntb_peer_port_count() - get the number of peer device ports | |
618 | * @ntb: NTB device context. | |
619 | * | |
620 | * Hardware may support an access to memory of several remote domains | |
621 | * over multi-port NTB devices. This method returns the number of peers, | |
622 | * local device can have shared memory with. | |
623 | * | |
624 | * Return: the number of peer ports | |
625 | */ | |
626 | static inline int ntb_peer_port_count(struct ntb_dev *ntb) | |
627 | { | |
628 | if (!ntb->ops->peer_port_count) | |
629 | return ntb_default_peer_port_count(ntb); | |
630 | ||
631 | return ntb->ops->peer_port_count(ntb); | |
632 | } | |
633 | ||
634 | /** | |
635 | * ntb_peer_port_number() - get the peer port by given index | |
636 | * @ntb: NTB device context. | |
637 | * @pidx: Peer port index. | |
638 | * | |
639 | * Peer ports are continuously enumerated by NTB API logic, so this method | |
640 | * lets to retrieve port real number by its index. | |
641 | * | |
642 | * Return: the peer device port or negative value indicating an error | |
643 | */ | |
644 | static inline int ntb_peer_port_number(struct ntb_dev *ntb, int pidx) | |
645 | { | |
646 | if (!ntb->ops->peer_port_number) | |
647 | return ntb_default_peer_port_number(ntb, pidx); | |
648 | ||
649 | return ntb->ops->peer_port_number(ntb, pidx); | |
650 | } | |
651 | ||
652 | /** | |
653 | * ntb_peer_port_idx() - get the peer device port index by given port number | |
654 | * @ntb: NTB device context. | |
655 | * @port: Peer port number. | |
656 | * | |
657 | * Inverse operation of ntb_peer_port_number(), so one can get port index | |
658 | * by specified port number. | |
659 | * | |
660 | * Return: the peer port index or negative value indicating an error | |
661 | */ | |
662 | static inline int ntb_peer_port_idx(struct ntb_dev *ntb, int port) | |
663 | { | |
664 | if (!ntb->ops->peer_port_idx) | |
665 | return ntb_default_peer_port_idx(ntb, port); | |
666 | ||
667 | return ntb->ops->peer_port_idx(ntb, port); | |
668 | } | |
669 | ||
60934b20 SS |
670 | /** |
671 | * ntb_link_is_up() - get the current ntb link state | |
672 | * @ntb: NTB device context. | |
673 | * @speed: OUT - The link speed expressed as PCIe generation number. | |
674 | * @width: OUT - The link width expressed as the number of PCIe lanes. | |
675 | * | |
676 | * Get the current state of the ntb link. It is recommended to query the link | |
677 | * state once after every link event. It is safe to query the link state in | |
678 | * the context of the link event callback. | |
679 | * | |
4e8c11b7 SS |
680 | * Return: bitfield of indexed ports link state: bit is set/cleared if the |
681 | * link is up/down respectively. | |
60934b20 | 682 | */ |
4e8c11b7 | 683 | static inline u64 ntb_link_is_up(struct ntb_dev *ntb, |
60934b20 SS |
684 | enum ntb_speed *speed, enum ntb_width *width) |
685 | { | |
686 | return ntb->ops->link_is_up(ntb, speed, width); | |
687 | } | |
688 | ||
689 | /** | |
4e8c11b7 | 690 | * ntb_link_enable() - enable the local port ntb connection |
60934b20 SS |
691 | * @ntb: NTB device context. |
692 | * @max_speed: The maximum link speed expressed as PCIe generation number. | |
693 | * @max_width: The maximum link width expressed as the number of PCIe lanes. | |
694 | * | |
4e8c11b7 SS |
695 | * Enable the NTB/PCIe link on the local or remote (for bridge-to-bridge |
696 | * topology) side of the bridge. If it's supported the ntb device should train | |
697 | * the link to its maximum speed and width, or the requested speed and width, | |
698 | * whichever is smaller. Some hardware doesn't support PCIe link training, so | |
699 | * the last two arguments will be ignored then. | |
60934b20 SS |
700 | * |
701 | * Return: Zero on success, otherwise an error number. | |
702 | */ | |
703 | static inline int ntb_link_enable(struct ntb_dev *ntb, | |
704 | enum ntb_speed max_speed, | |
705 | enum ntb_width max_width) | |
706 | { | |
707 | return ntb->ops->link_enable(ntb, max_speed, max_width); | |
708 | } | |
709 | ||
710 | /** | |
4e8c11b7 | 711 | * ntb_link_disable() - disable the local port ntb connection |
60934b20 SS |
712 | * @ntb: NTB device context. |
713 | * | |
4e8c11b7 SS |
714 | * Disable the link on the local or remote (for b2b topology) of the ntb. |
715 | * The ntb device should disable the link. Returning from this call must | |
716 | * indicate that a barrier has passed, though with no more writes may pass in | |
717 | * either direction across the link, except if this call returns an error | |
718 | * number. | |
60934b20 SS |
719 | * |
720 | * Return: Zero on success, otherwise an error number. | |
721 | */ | |
722 | static inline int ntb_link_disable(struct ntb_dev *ntb) | |
723 | { | |
724 | return ntb->ops->link_disable(ntb); | |
725 | } | |
726 | ||
a1bd3bae | 727 | /** |
443b9a14 SS |
728 | * ntb_mw_count() - get the number of inbound memory windows, which could |
729 | * be created for a specified peer device | |
a1bd3bae | 730 | * @ntb: NTB device context. |
443b9a14 | 731 | * @pidx: Port index of peer device. |
a1bd3bae AH |
732 | * |
733 | * Hardware and topology may support a different number of memory windows. | |
443b9a14 SS |
734 | * Moreover different peer devices can support different number of memory |
735 | * windows. Simply speaking this method returns the number of possible inbound | |
fa5ab66e LG |
736 | * memory windows to share with specified peer device. Note: this may return |
737 | * zero if the link is not up yet. | |
a1bd3bae AH |
738 | * |
739 | * Return: the number of memory windows. | |
740 | */ | |
443b9a14 | 741 | static inline int ntb_mw_count(struct ntb_dev *ntb, int pidx) |
a1bd3bae | 742 | { |
443b9a14 | 743 | return ntb->ops->mw_count(ntb, pidx); |
a1bd3bae AH |
744 | } |
745 | ||
746 | /** | |
443b9a14 | 747 | * ntb_mw_get_align() - get the restriction parameters of inbound memory window |
a1bd3bae | 748 | * @ntb: NTB device context. |
443b9a14 SS |
749 | * @pidx: Port index of peer device. |
750 | * @widx: Memory window index. | |
751 | * @addr_align: OUT - the base alignment for translating the memory window | |
752 | * @size_align: OUT - the size alignment for translating the memory window | |
753 | * @size_max: OUT - the maximum size of the memory window | |
754 | * | |
755 | * Get the alignments of an inbound memory window with specified index. | |
756 | * NULL may be given for any output parameter if the value is not needed. | |
757 | * The alignment and size parameters may be used for allocation of proper | |
fa5ab66e | 758 | * shared memory. Note: this must only be called when the link is up. |
443b9a14 SS |
759 | * |
760 | * Return: Zero on success, otherwise a negative error number. | |
761 | */ | |
762 | static inline int ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx, | |
763 | resource_size_t *addr_align, | |
764 | resource_size_t *size_align, | |
765 | resource_size_t *size_max) | |
a1bd3bae | 766 | { |
fa5ab66e LG |
767 | if (!(ntb_link_is_up(ntb, NULL, NULL) & (1 << pidx))) |
768 | return -ENOTCONN; | |
769 | ||
443b9a14 SS |
770 | return ntb->ops->mw_get_align(ntb, pidx, widx, addr_align, size_align, |
771 | size_max); | |
a1bd3bae AH |
772 | } |
773 | ||
774 | /** | |
443b9a14 | 775 | * ntb_mw_set_trans() - set the translation of an inbound memory window |
a1bd3bae | 776 | * @ntb: NTB device context. |
443b9a14 SS |
777 | * @pidx: Port index of peer device. |
778 | * @widx: Memory window index. | |
779 | * @addr: The dma address of local memory to expose to the peer. | |
a1bd3bae AH |
780 | * @size: The size of the local memory to expose to the peer. |
781 | * | |
782 | * Set the translation of a memory window. The peer may access local memory | |
783 | * through the window starting at the address, up to the size. The address | |
443b9a14 SS |
784 | * and size must be aligned in compliance with restrictions of |
785 | * ntb_mw_get_align(). The region size should not exceed the size_max parameter | |
786 | * of that method. | |
787 | * | |
788 | * This method may not be implemented due to the hardware specific memory | |
789 | * windows interface. | |
a1bd3bae AH |
790 | * |
791 | * Return: Zero on success, otherwise an error number. | |
792 | */ | |
443b9a14 | 793 | static inline int ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, |
a1bd3bae AH |
794 | dma_addr_t addr, resource_size_t size) |
795 | { | |
443b9a14 SS |
796 | if (!ntb->ops->mw_set_trans) |
797 | return 0; | |
798 | ||
799 | return ntb->ops->mw_set_trans(ntb, pidx, widx, addr, size); | |
a1bd3bae AH |
800 | } |
801 | ||
802 | /** | |
443b9a14 SS |
803 | * ntb_mw_clear_trans() - clear the translation address of an inbound memory |
804 | * window | |
a1bd3bae | 805 | * @ntb: NTB device context. |
443b9a14 SS |
806 | * @pidx: Port index of peer device. |
807 | * @widx: Memory window index. | |
a1bd3bae | 808 | * |
443b9a14 SS |
809 | * Clear the translation of an inbound memory window. The peer may no longer |
810 | * access local memory through the window. | |
a1bd3bae AH |
811 | * |
812 | * Return: Zero on success, otherwise an error number. | |
813 | */ | |
443b9a14 | 814 | static inline int ntb_mw_clear_trans(struct ntb_dev *ntb, int pidx, int widx) |
a1bd3bae AH |
815 | { |
816 | if (!ntb->ops->mw_clear_trans) | |
443b9a14 SS |
817 | return ntb_mw_set_trans(ntb, pidx, widx, 0, 0); |
818 | ||
819 | return ntb->ops->mw_clear_trans(ntb, pidx, widx); | |
820 | } | |
821 | ||
822 | /** | |
823 | * ntb_peer_mw_count() - get the number of outbound memory windows, which could | |
824 | * be mapped to access a shared memory | |
825 | * @ntb: NTB device context. | |
826 | * | |
827 | * Hardware and topology may support a different number of memory windows. | |
828 | * This method returns the number of outbound memory windows supported by | |
829 | * local device. | |
830 | * | |
831 | * Return: the number of memory windows. | |
832 | */ | |
833 | static inline int ntb_peer_mw_count(struct ntb_dev *ntb) | |
834 | { | |
835 | return ntb->ops->peer_mw_count(ntb); | |
836 | } | |
837 | ||
838 | /** | |
839 | * ntb_peer_mw_get_addr() - get map address of an outbound memory window | |
840 | * @ntb: NTB device context. | |
841 | * @widx: Memory window index (within ntb_peer_mw_count() return value). | |
842 | * @base: OUT - the base address of mapping region. | |
843 | * @size: OUT - the size of mapping region. | |
844 | * | |
845 | * Get base and size of memory region to map. NULL may be given for any output | |
846 | * parameter if the value is not needed. The base and size may be used for | |
847 | * mapping the memory window, to access the peer memory. | |
848 | * | |
849 | * Return: Zero on success, otherwise a negative error number. | |
850 | */ | |
851 | static inline int ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx, | |
852 | phys_addr_t *base, resource_size_t *size) | |
853 | { | |
854 | return ntb->ops->peer_mw_get_addr(ntb, widx, base, size); | |
855 | } | |
856 | ||
857 | /** | |
858 | * ntb_peer_mw_set_trans() - set a translation address of a memory window | |
859 | * retrieved from a peer device | |
860 | * @ntb: NTB device context. | |
861 | * @pidx: Port index of peer device the translation address received from. | |
862 | * @widx: Memory window index. | |
863 | * @addr: The dma address of the shared memory to access. | |
864 | * @size: The size of the shared memory to access. | |
865 | * | |
866 | * Set the translation of an outbound memory window. The local device may | |
867 | * access shared memory allocated by a peer device sent the address. | |
868 | * | |
869 | * This method may not be implemented due to the hardware specific memory | |
870 | * windows interface, so a translation address can be only set on the side, | |
871 | * where shared memory (inbound memory windows) is allocated. | |
872 | * | |
873 | * Return: Zero on success, otherwise an error number. | |
874 | */ | |
875 | static inline int ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, | |
876 | u64 addr, resource_size_t size) | |
877 | { | |
878 | if (!ntb->ops->peer_mw_set_trans) | |
879 | return 0; | |
880 | ||
881 | return ntb->ops->peer_mw_set_trans(ntb, pidx, widx, addr, size); | |
882 | } | |
883 | ||
884 | /** | |
885 | * ntb_peer_mw_clear_trans() - clear the translation address of an outbound | |
886 | * memory window | |
887 | * @ntb: NTB device context. | |
888 | * @pidx: Port index of peer device. | |
889 | * @widx: Memory window index. | |
890 | * | |
891 | * Clear the translation of a outbound memory window. The local device may no | |
892 | * longer access a shared memory through the window. | |
893 | * | |
894 | * This method may not be implemented due to the hardware specific memory | |
895 | * windows interface. | |
896 | * | |
897 | * Return: Zero on success, otherwise an error number. | |
898 | */ | |
899 | static inline int ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx, | |
900 | int widx) | |
901 | { | |
902 | if (!ntb->ops->peer_mw_clear_trans) | |
903 | return ntb_peer_mw_set_trans(ntb, pidx, widx, 0, 0); | |
a1bd3bae | 904 | |
443b9a14 | 905 | return ntb->ops->peer_mw_clear_trans(ntb, pidx, widx); |
a1bd3bae AH |
906 | } |
907 | ||
a1bd3bae AH |
908 | /** |
909 | * ntb_db_is_unsafe() - check if it is safe to use hardware doorbell | |
910 | * @ntb: NTB device context. | |
911 | * | |
912 | * It is possible for some ntb hardware to be affected by errata. Hardware | |
913 | * drivers can advise clients to avoid using doorbells. Clients may ignore | |
914 | * this advice, though caution is recommended. | |
915 | * | |
916 | * Return: Zero if it is safe to use doorbells, or One if it is not safe. | |
917 | */ | |
918 | static inline int ntb_db_is_unsafe(struct ntb_dev *ntb) | |
919 | { | |
920 | if (!ntb->ops->db_is_unsafe) | |
921 | return 0; | |
922 | ||
923 | return ntb->ops->db_is_unsafe(ntb); | |
924 | } | |
925 | ||
926 | /** | |
927 | * ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb | |
928 | * @ntb: NTB device context. | |
929 | * | |
930 | * Hardware may support different number or arrangement of doorbell bits. | |
931 | * | |
932 | * Return: A mask of doorbell bits supported by the ntb. | |
933 | */ | |
934 | static inline u64 ntb_db_valid_mask(struct ntb_dev *ntb) | |
935 | { | |
936 | return ntb->ops->db_valid_mask(ntb); | |
937 | } | |
938 | ||
939 | /** | |
940 | * ntb_db_vector_count() - get the number of doorbell interrupt vectors | |
941 | * @ntb: NTB device context. | |
942 | * | |
943 | * Hardware may support different number of interrupt vectors. | |
944 | * | |
945 | * Return: The number of doorbell interrupt vectors. | |
946 | */ | |
947 | static inline int ntb_db_vector_count(struct ntb_dev *ntb) | |
948 | { | |
949 | if (!ntb->ops->db_vector_count) | |
950 | return 1; | |
951 | ||
952 | return ntb->ops->db_vector_count(ntb); | |
953 | } | |
954 | ||
955 | /** | |
956 | * ntb_db_vector_mask() - get a mask of doorbell bits serviced by a vector | |
957 | * @ntb: NTB device context. | |
958 | * @vector: Doorbell vector number. | |
959 | * | |
960 | * Each interrupt vector may have a different number or arrangement of bits. | |
961 | * | |
962 | * Return: A mask of doorbell bits serviced by a vector. | |
963 | */ | |
964 | static inline u64 ntb_db_vector_mask(struct ntb_dev *ntb, int vector) | |
965 | { | |
966 | if (!ntb->ops->db_vector_mask) | |
967 | return ntb_db_valid_mask(ntb); | |
968 | ||
969 | return ntb->ops->db_vector_mask(ntb, vector); | |
970 | } | |
971 | ||
972 | /** | |
973 | * ntb_db_read() - read the local doorbell register | |
974 | * @ntb: NTB device context. | |
975 | * | |
976 | * Read the local doorbell register, and return the bits that are set. | |
977 | * | |
978 | * Return: The bits currently set in the local doorbell register. | |
979 | */ | |
980 | static inline u64 ntb_db_read(struct ntb_dev *ntb) | |
981 | { | |
982 | return ntb->ops->db_read(ntb); | |
983 | } | |
984 | ||
985 | /** | |
986 | * ntb_db_set() - set bits in the local doorbell register | |
987 | * @ntb: NTB device context. | |
988 | * @db_bits: Doorbell bits to set. | |
989 | * | |
990 | * Set bits in the local doorbell register, which may generate a local doorbell | |
991 | * interrupt. Bits that were already set must remain set. | |
992 | * | |
993 | * This is unusual, and hardware may not support it. | |
994 | * | |
995 | * Return: Zero on success, otherwise an error number. | |
996 | */ | |
997 | static inline int ntb_db_set(struct ntb_dev *ntb, u64 db_bits) | |
998 | { | |
999 | if (!ntb->ops->db_set) | |
1000 | return -EINVAL; | |
1001 | ||
1002 | return ntb->ops->db_set(ntb, db_bits); | |
1003 | } | |
1004 | ||
1005 | /** | |
1006 | * ntb_db_clear() - clear bits in the local doorbell register | |
1007 | * @ntb: NTB device context. | |
1008 | * @db_bits: Doorbell bits to clear. | |
1009 | * | |
1010 | * Clear bits in the local doorbell register, arming the bits for the next | |
1011 | * doorbell. | |
1012 | * | |
1013 | * Return: Zero on success, otherwise an error number. | |
1014 | */ | |
1015 | static inline int ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) | |
1016 | { | |
1017 | return ntb->ops->db_clear(ntb, db_bits); | |
1018 | } | |
1019 | ||
1020 | /** | |
1021 | * ntb_db_read_mask() - read the local doorbell mask | |
1022 | * @ntb: NTB device context. | |
1023 | * | |
1024 | * Read the local doorbell mask register, and return the bits that are set. | |
1025 | * | |
1026 | * This is unusual, though hardware is likely to support it. | |
1027 | * | |
1028 | * Return: The bits currently set in the local doorbell mask register. | |
1029 | */ | |
1030 | static inline u64 ntb_db_read_mask(struct ntb_dev *ntb) | |
1031 | { | |
1032 | if (!ntb->ops->db_read_mask) | |
1033 | return 0; | |
1034 | ||
1035 | return ntb->ops->db_read_mask(ntb); | |
1036 | } | |
1037 | ||
1038 | /** | |
1039 | * ntb_db_set_mask() - set bits in the local doorbell mask | |
1040 | * @ntb: NTB device context. | |
1041 | * @db_bits: Doorbell mask bits to set. | |
1042 | * | |
1043 | * Set bits in the local doorbell mask register, preventing doorbell interrupts | |
1044 | * from being generated for those doorbell bits. Bits that were already set | |
1045 | * must remain set. | |
1046 | * | |
1047 | * Return: Zero on success, otherwise an error number. | |
1048 | */ | |
1049 | static inline int ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) | |
1050 | { | |
1051 | return ntb->ops->db_set_mask(ntb, db_bits); | |
1052 | } | |
1053 | ||
1054 | /** | |
1055 | * ntb_db_clear_mask() - clear bits in the local doorbell mask | |
1056 | * @ntb: NTB device context. | |
1057 | * @db_bits: Doorbell bits to clear. | |
1058 | * | |
1059 | * Clear bits in the local doorbell mask register, allowing doorbell interrupts | |
1060 | * from being generated for those doorbell bits. If a doorbell bit is already | |
1061 | * set at the time the mask is cleared, and the corresponding mask bit is | |
1062 | * changed from set to clear, then the ntb driver must ensure that | |
1063 | * ntb_db_event() is called. If the hardware does not generate the interrupt | |
1064 | * on clearing the mask bit, then the driver must call ntb_db_event() anyway. | |
1065 | * | |
1066 | * Return: Zero on success, otherwise an error number. | |
1067 | */ | |
1068 | static inline int ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) | |
1069 | { | |
1070 | return ntb->ops->db_clear_mask(ntb, db_bits); | |
1071 | } | |
1072 | ||
1073 | /** | |
1074 | * ntb_peer_db_addr() - address and size of the peer doorbell register | |
1075 | * @ntb: NTB device context. | |
1076 | * @db_addr: OUT - The address of the peer doorbell register. | |
1077 | * @db_size: OUT - The number of bytes to write the peer doorbell register. | |
1078 | * | |
1079 | * Return the address of the peer doorbell register. This may be used, for | |
1080 | * example, by drivers that offload memory copy operations to a dma engine. | |
1081 | * The drivers may wish to ring the peer doorbell at the completion of memory | |
1082 | * copy operations. For efficiency, and to simplify ordering of operations | |
1083 | * between the dma memory copies and the ringing doorbell, the driver may | |
1084 | * append one additional dma memory copy with the doorbell register as the | |
1085 | * destination, after the memory copy operations. | |
1086 | * | |
1087 | * Return: Zero on success, otherwise an error number. | |
1088 | */ | |
1089 | static inline int ntb_peer_db_addr(struct ntb_dev *ntb, | |
1090 | phys_addr_t *db_addr, | |
1091 | resource_size_t *db_size) | |
1092 | { | |
afc54992 AH |
1093 | if (!ntb->ops->peer_db_addr) |
1094 | return -EINVAL; | |
1095 | ||
a1bd3bae AH |
1096 | return ntb->ops->peer_db_addr(ntb, db_addr, db_size); |
1097 | } | |
1098 | ||
1099 | /** | |
1100 | * ntb_peer_db_read() - read the peer doorbell register | |
1101 | * @ntb: NTB device context. | |
1102 | * | |
1103 | * Read the peer doorbell register, and return the bits that are set. | |
1104 | * | |
1105 | * This is unusual, and hardware may not support it. | |
1106 | * | |
1107 | * Return: The bits currently set in the peer doorbell register. | |
1108 | */ | |
1109 | static inline u64 ntb_peer_db_read(struct ntb_dev *ntb) | |
1110 | { | |
1111 | if (!ntb->ops->peer_db_read) | |
1112 | return 0; | |
1113 | ||
1114 | return ntb->ops->peer_db_read(ntb); | |
1115 | } | |
1116 | ||
1117 | /** | |
1118 | * ntb_peer_db_set() - set bits in the peer doorbell register | |
1119 | * @ntb: NTB device context. | |
1120 | * @db_bits: Doorbell bits to set. | |
1121 | * | |
1122 | * Set bits in the peer doorbell register, which may generate a peer doorbell | |
1123 | * interrupt. Bits that were already set must remain set. | |
1124 | * | |
1125 | * Return: Zero on success, otherwise an error number. | |
1126 | */ | |
1127 | static inline int ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) | |
1128 | { | |
1129 | return ntb->ops->peer_db_set(ntb, db_bits); | |
1130 | } | |
1131 | ||
1132 | /** | |
86663c91 | 1133 | * ntb_peer_db_clear() - clear bits in the peer doorbell register |
a1bd3bae AH |
1134 | * @ntb: NTB device context. |
1135 | * @db_bits: Doorbell bits to clear. | |
1136 | * | |
1137 | * Clear bits in the peer doorbell register, arming the bits for the next | |
1138 | * doorbell. | |
1139 | * | |
1140 | * This is unusual, and hardware may not support it. | |
1141 | * | |
1142 | * Return: Zero on success, otherwise an error number. | |
1143 | */ | |
1144 | static inline int ntb_peer_db_clear(struct ntb_dev *ntb, u64 db_bits) | |
1145 | { | |
1146 | if (!ntb->ops->db_clear) | |
1147 | return -EINVAL; | |
1148 | ||
1149 | return ntb->ops->peer_db_clear(ntb, db_bits); | |
1150 | } | |
1151 | ||
1152 | /** | |
1153 | * ntb_peer_db_read_mask() - read the peer doorbell mask | |
1154 | * @ntb: NTB device context. | |
1155 | * | |
1156 | * Read the peer doorbell mask register, and return the bits that are set. | |
1157 | * | |
1158 | * This is unusual, and hardware may not support it. | |
1159 | * | |
1160 | * Return: The bits currently set in the peer doorbell mask register. | |
1161 | */ | |
1162 | static inline u64 ntb_peer_db_read_mask(struct ntb_dev *ntb) | |
1163 | { | |
1164 | if (!ntb->ops->db_read_mask) | |
1165 | return 0; | |
1166 | ||
1167 | return ntb->ops->peer_db_read_mask(ntb); | |
1168 | } | |
1169 | ||
1170 | /** | |
1171 | * ntb_peer_db_set_mask() - set bits in the peer doorbell mask | |
1172 | * @ntb: NTB device context. | |
1173 | * @db_bits: Doorbell mask bits to set. | |
1174 | * | |
1175 | * Set bits in the peer doorbell mask register, preventing doorbell interrupts | |
1176 | * from being generated for those doorbell bits. Bits that were already set | |
1177 | * must remain set. | |
1178 | * | |
1179 | * This is unusual, and hardware may not support it. | |
1180 | * | |
1181 | * Return: Zero on success, otherwise an error number. | |
1182 | */ | |
1183 | static inline int ntb_peer_db_set_mask(struct ntb_dev *ntb, u64 db_bits) | |
1184 | { | |
1185 | if (!ntb->ops->db_set_mask) | |
1186 | return -EINVAL; | |
1187 | ||
1188 | return ntb->ops->peer_db_set_mask(ntb, db_bits); | |
1189 | } | |
1190 | ||
1191 | /** | |
1192 | * ntb_peer_db_clear_mask() - clear bits in the peer doorbell mask | |
1193 | * @ntb: NTB device context. | |
1194 | * @db_bits: Doorbell bits to clear. | |
1195 | * | |
1196 | * Clear bits in the peer doorbell mask register, allowing doorbell interrupts | |
1197 | * from being generated for those doorbell bits. If the hardware does not | |
1198 | * generate the interrupt on clearing the mask bit, then the driver should not | |
1199 | * implement this function! | |
1200 | * | |
1201 | * This is unusual, and hardware may not support it. | |
1202 | * | |
1203 | * Return: Zero on success, otherwise an error number. | |
1204 | */ | |
1205 | static inline int ntb_peer_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) | |
1206 | { | |
1207 | if (!ntb->ops->db_clear_mask) | |
1208 | return -EINVAL; | |
1209 | ||
1210 | return ntb->ops->peer_db_clear_mask(ntb, db_bits); | |
1211 | } | |
1212 | ||
1213 | /** | |
1214 | * ntb_spad_is_unsafe() - check if it is safe to use the hardware scratchpads | |
1215 | * @ntb: NTB device context. | |
1216 | * | |
1217 | * It is possible for some ntb hardware to be affected by errata. Hardware | |
1218 | * drivers can advise clients to avoid using scratchpads. Clients may ignore | |
1219 | * this advice, though caution is recommended. | |
1220 | * | |
1221 | * Return: Zero if it is safe to use scratchpads, or One if it is not safe. | |
1222 | */ | |
1223 | static inline int ntb_spad_is_unsafe(struct ntb_dev *ntb) | |
1224 | { | |
1225 | if (!ntb->ops->spad_is_unsafe) | |
1226 | return 0; | |
1227 | ||
1228 | return ntb->ops->spad_is_unsafe(ntb); | |
1229 | } | |
1230 | ||
1231 | /** | |
74dcba35 | 1232 | * ntb_spad_count() - get the number of scratchpads |
a1bd3bae AH |
1233 | * @ntb: NTB device context. |
1234 | * | |
1235 | * Hardware and topology may support a different number of scratchpads. | |
d67288a3 | 1236 | * Although it must be the same for all ports per NTB device. |
a1bd3bae AH |
1237 | * |
1238 | * Return: the number of scratchpads. | |
1239 | */ | |
1240 | static inline int ntb_spad_count(struct ntb_dev *ntb) | |
1241 | { | |
d67288a3 SS |
1242 | if (!ntb->ops->spad_count) |
1243 | return 0; | |
1244 | ||
a1bd3bae AH |
1245 | return ntb->ops->spad_count(ntb); |
1246 | } | |
1247 | ||
1248 | /** | |
1249 | * ntb_spad_read() - read the local scratchpad register | |
1250 | * @ntb: NTB device context. | |
d67288a3 | 1251 | * @sidx: Scratchpad index. |
a1bd3bae AH |
1252 | * |
1253 | * Read the local scratchpad register, and return the value. | |
1254 | * | |
1255 | * Return: The value of the local scratchpad register. | |
1256 | */ | |
d67288a3 | 1257 | static inline u32 ntb_spad_read(struct ntb_dev *ntb, int sidx) |
a1bd3bae | 1258 | { |
d67288a3 SS |
1259 | if (!ntb->ops->spad_read) |
1260 | return ~(u32)0; | |
1261 | ||
1262 | return ntb->ops->spad_read(ntb, sidx); | |
a1bd3bae AH |
1263 | } |
1264 | ||
1265 | /** | |
1266 | * ntb_spad_write() - write the local scratchpad register | |
1267 | * @ntb: NTB device context. | |
d67288a3 | 1268 | * @sidx: Scratchpad index. |
a1bd3bae AH |
1269 | * @val: Scratchpad value. |
1270 | * | |
1271 | * Write the value to the local scratchpad register. | |
1272 | * | |
1273 | * Return: Zero on success, otherwise an error number. | |
1274 | */ | |
d67288a3 | 1275 | static inline int ntb_spad_write(struct ntb_dev *ntb, int sidx, u32 val) |
a1bd3bae | 1276 | { |
d67288a3 SS |
1277 | if (!ntb->ops->spad_write) |
1278 | return -EINVAL; | |
1279 | ||
1280 | return ntb->ops->spad_write(ntb, sidx, val); | |
a1bd3bae AH |
1281 | } |
1282 | ||
1283 | /** | |
1284 | * ntb_peer_spad_addr() - address of the peer scratchpad register | |
1285 | * @ntb: NTB device context. | |
d67288a3 SS |
1286 | * @pidx: Port index of peer device. |
1287 | * @sidx: Scratchpad index. | |
a1bd3bae AH |
1288 | * @spad_addr: OUT - The address of the peer scratchpad register. |
1289 | * | |
1290 | * Return the address of the peer doorbell register. This may be used, for | |
1291 | * example, by drivers that offload memory copy operations to a dma engine. | |
1292 | * | |
1293 | * Return: Zero on success, otherwise an error number. | |
1294 | */ | |
d67288a3 | 1295 | static inline int ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, |
a1bd3bae AH |
1296 | phys_addr_t *spad_addr) |
1297 | { | |
afc54992 AH |
1298 | if (!ntb->ops->peer_spad_addr) |
1299 | return -EINVAL; | |
1300 | ||
d67288a3 | 1301 | return ntb->ops->peer_spad_addr(ntb, pidx, sidx, spad_addr); |
a1bd3bae AH |
1302 | } |
1303 | ||
1304 | /** | |
1305 | * ntb_peer_spad_read() - read the peer scratchpad register | |
1306 | * @ntb: NTB device context. | |
d67288a3 SS |
1307 | * @pidx: Port index of peer device. |
1308 | * @sidx: Scratchpad index. | |
a1bd3bae AH |
1309 | * |
1310 | * Read the peer scratchpad register, and return the value. | |
1311 | * | |
1312 | * Return: The value of the local scratchpad register. | |
1313 | */ | |
d67288a3 | 1314 | static inline u32 ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) |
a1bd3bae | 1315 | { |
5c43c52d | 1316 | if (!ntb->ops->peer_spad_read) |
d67288a3 | 1317 | return ~(u32)0; |
5c43c52d | 1318 | |
d67288a3 | 1319 | return ntb->ops->peer_spad_read(ntb, pidx, sidx); |
a1bd3bae AH |
1320 | } |
1321 | ||
1322 | /** | |
1323 | * ntb_peer_spad_write() - write the peer scratchpad register | |
1324 | * @ntb: NTB device context. | |
d67288a3 SS |
1325 | * @pidx: Port index of peer device. |
1326 | * @sidx: Scratchpad index. | |
a1bd3bae AH |
1327 | * @val: Scratchpad value. |
1328 | * | |
1329 | * Write the value to the peer scratchpad register. | |
1330 | * | |
1331 | * Return: Zero on success, otherwise an error number. | |
1332 | */ | |
d67288a3 SS |
1333 | static inline int ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, |
1334 | u32 val) | |
a1bd3bae | 1335 | { |
d67288a3 SS |
1336 | if (!ntb->ops->peer_spad_write) |
1337 | return -EINVAL; | |
1338 | ||
1339 | return ntb->ops->peer_spad_write(ntb, pidx, sidx, val); | |
a1bd3bae AH |
1340 | } |
1341 | ||
bc3e49ad SS |
1342 | /** |
1343 | * ntb_msg_count() - get the number of message registers | |
1344 | * @ntb: NTB device context. | |
1345 | * | |
1346 | * Hardware may support a different number of message registers. | |
1347 | * | |
1348 | * Return: the number of message registers. | |
1349 | */ | |
1350 | static inline int ntb_msg_count(struct ntb_dev *ntb) | |
1351 | { | |
1352 | if (!ntb->ops->msg_count) | |
1353 | return 0; | |
1354 | ||
1355 | return ntb->ops->msg_count(ntb); | |
1356 | } | |
1357 | ||
1358 | /** | |
1359 | * ntb_msg_inbits() - get a bitfield of inbound message registers status | |
1360 | * @ntb: NTB device context. | |
1361 | * | |
1362 | * The method returns the bitfield of status and mask registers, which related | |
1363 | * to inbound message registers. | |
1364 | * | |
1365 | * Return: bitfield of inbound message registers. | |
1366 | */ | |
1367 | static inline u64 ntb_msg_inbits(struct ntb_dev *ntb) | |
1368 | { | |
1369 | if (!ntb->ops->msg_inbits) | |
1370 | return 0; | |
1371 | ||
1372 | return ntb->ops->msg_inbits(ntb); | |
1373 | } | |
1374 | ||
1375 | /** | |
1376 | * ntb_msg_outbits() - get a bitfield of outbound message registers status | |
1377 | * @ntb: NTB device context. | |
1378 | * | |
1379 | * The method returns the bitfield of status and mask registers, which related | |
1380 | * to outbound message registers. | |
1381 | * | |
1382 | * Return: bitfield of outbound message registers. | |
1383 | */ | |
1384 | static inline u64 ntb_msg_outbits(struct ntb_dev *ntb) | |
1385 | { | |
1386 | if (!ntb->ops->msg_outbits) | |
1387 | return 0; | |
1388 | ||
1389 | return ntb->ops->msg_outbits(ntb); | |
1390 | } | |
1391 | ||
1392 | /** | |
1393 | * ntb_msg_read_sts() - read the message registers status | |
1394 | * @ntb: NTB device context. | |
1395 | * | |
1396 | * Read the status of message register. Inbound and outbound message registers | |
1397 | * related bits can be filtered by masks retrieved from ntb_msg_inbits() and | |
1398 | * ntb_msg_outbits(). | |
1399 | * | |
1400 | * Return: status bits of message registers | |
1401 | */ | |
1402 | static inline u64 ntb_msg_read_sts(struct ntb_dev *ntb) | |
1403 | { | |
1404 | if (!ntb->ops->msg_read_sts) | |
1405 | return 0; | |
1406 | ||
1407 | return ntb->ops->msg_read_sts(ntb); | |
1408 | } | |
1409 | ||
1410 | /** | |
1411 | * ntb_msg_clear_sts() - clear status bits of message registers | |
1412 | * @ntb: NTB device context. | |
1413 | * @sts_bits: Status bits to clear. | |
1414 | * | |
1415 | * Clear bits in the status register. | |
1416 | * | |
1417 | * Return: Zero on success, otherwise a negative error number. | |
1418 | */ | |
1419 | static inline int ntb_msg_clear_sts(struct ntb_dev *ntb, u64 sts_bits) | |
1420 | { | |
1421 | if (!ntb->ops->msg_clear_sts) | |
1422 | return -EINVAL; | |
1423 | ||
1424 | return ntb->ops->msg_clear_sts(ntb, sts_bits); | |
1425 | } | |
1426 | ||
1427 | /** | |
1428 | * ntb_msg_set_mask() - set mask of message register status bits | |
1429 | * @ntb: NTB device context. | |
1430 | * @mask_bits: Mask bits. | |
1431 | * | |
1432 | * Mask the message registers status bits from raising the message event. | |
1433 | * | |
1434 | * Return: Zero on success, otherwise a negative error number. | |
1435 | */ | |
1436 | static inline int ntb_msg_set_mask(struct ntb_dev *ntb, u64 mask_bits) | |
1437 | { | |
1438 | if (!ntb->ops->msg_set_mask) | |
1439 | return -EINVAL; | |
1440 | ||
1441 | return ntb->ops->msg_set_mask(ntb, mask_bits); | |
1442 | } | |
1443 | ||
1444 | /** | |
1445 | * ntb_msg_clear_mask() - clear message registers mask | |
1446 | * @ntb: NTB device context. | |
1447 | * @mask_bits: Mask bits to clear. | |
1448 | * | |
1449 | * Clear bits in the message events mask register. | |
1450 | * | |
1451 | * Return: Zero on success, otherwise a negative error number. | |
1452 | */ | |
1453 | static inline int ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits) | |
1454 | { | |
1455 | if (!ntb->ops->msg_clear_mask) | |
1456 | return -EINVAL; | |
1457 | ||
1458 | return ntb->ops->msg_clear_mask(ntb, mask_bits); | |
1459 | } | |
1460 | ||
1461 | /** | |
1462 | * ntb_msg_read() - read message register with specified index | |
1463 | * @ntb: NTB device context. | |
1464 | * @midx: Message register index | |
1465 | * @pidx: OUT - Port index of peer device a message retrieved from | |
1466 | * @msg: OUT - Data | |
1467 | * | |
1468 | * Read data from the specified message register. Source port index of a | |
1469 | * message is retrieved as well. | |
1470 | * | |
1471 | * Return: Zero on success, otherwise a negative error number. | |
1472 | */ | |
1473 | static inline int ntb_msg_read(struct ntb_dev *ntb, int midx, int *pidx, | |
1474 | u32 *msg) | |
1475 | { | |
1476 | if (!ntb->ops->msg_read) | |
1477 | return -EINVAL; | |
1478 | ||
1479 | return ntb->ops->msg_read(ntb, midx, pidx, msg); | |
1480 | } | |
1481 | ||
1482 | /** | |
1483 | * ntb_msg_write() - write data to the specified message register | |
1484 | * @ntb: NTB device context. | |
1485 | * @midx: Message register index | |
1486 | * @pidx: Port index of peer device a message being sent to | |
1487 | * @msg: Data to send | |
1488 | * | |
1489 | * Send data to a specified peer device using the defined message register. | |
1490 | * Message event can be raised if the midx registers isn't empty while | |
1491 | * calling this method and the corresponding interrupt isn't masked. | |
1492 | * | |
1493 | * Return: Zero on success, otherwise a negative error number. | |
1494 | */ | |
1495 | static inline int ntb_msg_write(struct ntb_dev *ntb, int midx, int pidx, | |
1496 | u32 msg) | |
1497 | { | |
1498 | if (!ntb->ops->msg_write) | |
1499 | return -EINVAL; | |
1500 | ||
1501 | return ntb->ops->msg_write(ntb, midx, pidx, msg); | |
1502 | } | |
1503 | ||
a1bd3bae | 1504 | #endif |