]>
Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | |
7 | */ | |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
e17d416b | 16 | #include <linux/delay.h> |
94bd2708 | 17 | #include <asm/uncached.h> |
261f3b49 | 18 | #include <asm/sn/mspec.h> |
94bd2708 DN |
19 | #include <asm/sn/sn_sal.h> |
20 | #include "xpc.h" | |
21 | ||
ee6665e3 DN |
22 | /* |
23 | * Define the number of u64s required to represent all the C-brick nasids | |
24 | * as a bitmap. The cross-partition kernel modules deal only with | |
25 | * C-brick nasids, thus the need for bitmaps which don't account for | |
26 | * odd-numbered (non C-brick) nasids. | |
27 | */ | |
28 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
29 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
30 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
31 | ||
32 | /* | |
33 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
34 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
35 | * for cached references and an alternate TLB handler to never provide a | |
36 | * cacheable mapping for the entire region. This will prevent speculative | |
37 | * reading of cached copies of our lines from being issued which will cause | |
38 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
39 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
40 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
41 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
42 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
43 | * local XPC and 1 amo variable to request partition deactivation. | |
44 | */ | |
45 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
46 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
47 | XP_MAX_NPARTITIONS_SN2) | |
48 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
49 | XP_NASID_MASK_WORDS_SN2) | |
50 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
51 | ||
52 | /* | |
53 | * Buffer used to store a local copy of portions of a remote partition's | |
54 | * reserved page (either its header and part_nasids mask, or its vars). | |
55 | */ | |
ee6665e3 | 56 | static void *xpc_remote_copy_buffer_base_sn2; |
5b8669df | 57 | static char *xpc_remote_copy_buffer_sn2; |
ee6665e3 | 58 | |
8e85c23e DN |
59 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
60 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 61 | |
5b8669df DN |
62 | static int |
63 | xpc_setup_partitions_sn_sn2(void) | |
64 | { | |
65 | /* nothing needs to be done */ | |
66 | return 0; | |
67 | } | |
68 | ||
6e41017a | 69 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
70 | static u64 xpc_sh1_IPI_access_sn2; |
71 | static u64 xpc_sh2_IPI_access0_sn2; | |
72 | static u64 xpc_sh2_IPI_access1_sn2; | |
73 | static u64 xpc_sh2_IPI_access2_sn2; | |
74 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
75 | |
76 | /* | |
77 | * Change protections to allow IPI operations. | |
78 | */ | |
79 | static void | |
80 | xpc_allow_IPI_ops_sn2(void) | |
81 | { | |
82 | int node; | |
83 | int nasid; | |
84 | ||
ea57f80c | 85 | /* !!! The following should get moved into SAL. */ |
6e41017a | 86 | if (is_shub2()) { |
8e85c23e | 87 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 88 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 89 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 90 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 91 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 92 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 93 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
94 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
95 | ||
96 | for_each_online_node(node) { | |
97 | nasid = cnodeid_to_nasid(node); | |
98 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
99 | -1UL); | |
100 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
101 | -1UL); | |
102 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
103 | -1UL); | |
104 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
105 | -1UL); | |
106 | } | |
107 | } else { | |
8e85c23e | 108 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
109 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
110 | ||
111 | for_each_online_node(node) { | |
112 | nasid = cnodeid_to_nasid(node); | |
113 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
114 | -1UL); | |
115 | } | |
116 | } | |
117 | } | |
118 | ||
119 | /* | |
120 | * Restrict protections to disallow IPI operations. | |
121 | */ | |
122 | static void | |
123 | xpc_disallow_IPI_ops_sn2(void) | |
124 | { | |
125 | int node; | |
126 | int nasid; | |
127 | ||
ea57f80c | 128 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
129 | if (is_shub2()) { |
130 | for_each_online_node(node) { | |
131 | nasid = cnodeid_to_nasid(node); | |
132 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 133 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 134 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 135 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 136 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 137 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 138 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 139 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
140 | } |
141 | } else { | |
142 | for_each_online_node(node) { | |
143 | nasid = cnodeid_to_nasid(node); | |
144 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 145 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
146 | } |
147 | } | |
148 | } | |
149 | ||
33ba3c77 | 150 | /* |
7fb5e59d DN |
151 | * The following set of functions are used for the sending and receiving of |
152 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
153 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
154 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
155 | */ |
156 | ||
157 | static u64 | |
c39838ce | 158 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
159 | { |
160 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
161 | } | |
162 | ||
163 | static enum xp_retval | |
c39838ce DN |
164 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
165 | int vector) | |
33ba3c77 DN |
166 | { |
167 | int ret = 0; | |
168 | unsigned long irq_flags; | |
169 | ||
170 | local_irq_save(irq_flags); | |
171 | ||
172 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
173 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
174 | ||
175 | /* | |
176 | * We must always use the nofault function regardless of whether we | |
177 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
178 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 179 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
180 | */ |
181 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
182 | xp_nofault_PIOR_target)); | |
183 | ||
184 | local_irq_restore(irq_flags); | |
185 | ||
261f3b49 | 186 | return (ret == 0) ? xpSuccess : xpPioReadError; |
33ba3c77 DN |
187 | } |
188 | ||
c39838ce | 189 | static struct amo * |
7fb5e59d | 190 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 191 | { |
8e85c23e | 192 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 193 | |
c39838ce | 194 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
195 | return amo; |
196 | } | |
197 | ||
198 | /* | |
7fb5e59d | 199 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
200 | */ |
201 | ||
6e41017a DN |
202 | /* |
203 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
204 | */ | |
205 | static irqreturn_t | |
206 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
207 | { | |
5b8669df DN |
208 | unsigned long irq_flags; |
209 | ||
210 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
211 | xpc_activate_IRQ_rcvd++; | |
212 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
213 | ||
6e41017a DN |
214 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
215 | return IRQ_HANDLED; | |
216 | } | |
217 | ||
33ba3c77 | 218 | /* |
c39838ce | 219 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
220 | */ |
221 | static void | |
a812dcc3 DN |
222 | xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, |
223 | int to_nasid, int to_phys_cpuid) | |
33ba3c77 | 224 | { |
c39838ce | 225 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 226 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 227 | sizeof(struct amo))); |
33ba3c77 | 228 | |
04de7418 DN |
229 | (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], |
230 | BIT_MASK(from_nasid / 2), to_nasid, | |
33ba3c77 DN |
231 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
232 | } | |
233 | ||
234 | static void | |
7fb5e59d | 235 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 | 236 | { |
5b8669df | 237 | unsigned long irq_flags; |
8e85c23e | 238 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 239 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 240 | sizeof(struct amo))); |
33ba3c77 DN |
241 | |
242 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
04de7418 DN |
243 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), |
244 | FETCHOP_OR, BIT_MASK(from_nasid / 2)); | |
245 | ||
5b8669df DN |
246 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); |
247 | xpc_activate_IRQ_rcvd++; | |
248 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
249 | ||
6e41017a | 250 | wake_up_interruptible(&xpc_activate_IRQ_wq); |
33ba3c77 DN |
251 | } |
252 | ||
a47d5dac | 253 | /* |
7fb5e59d | 254 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 255 | */ |
33ba3c77 | 256 | |
a47d5dac | 257 | /* |
7fb5e59d | 258 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 259 | */ |
33ba3c77 | 260 | static void |
7fb5e59d | 261 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 262 | { |
7fb5e59d | 263 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 264 | unsigned long irq_flags; |
33ba3c77 | 265 | |
7fb5e59d DN |
266 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
267 | local_chctl_amo_va); | |
268 | if (chctl.all_flags == 0) | |
a47d5dac DN |
269 | return; |
270 | ||
7fb5e59d DN |
271 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
272 | part->chctl.all_flags |= chctl.all_flags; | |
273 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 274 | |
7fb5e59d DN |
275 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
276 | "0x%lx\n", XPC_PARTID(part), chctl.all_flags); | |
a47d5dac DN |
277 | |
278 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
279 | } |
280 | ||
a47d5dac DN |
281 | /* |
282 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
283 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 284 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 285 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
286 | * associated kthread to handle it. |
287 | * | |
7fb5e59d | 288 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
289 | * running on other partitions. |
290 | * | |
291 | * Noteworthy Arguments: | |
292 | * | |
293 | * irq - Interrupt ReQuest number. NOT USED. | |
294 | * | |
7fb5e59d | 295 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
296 | */ |
297 | static irqreturn_t | |
298 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 299 | { |
a47d5dac DN |
300 | short partid = (short)(u64)dev_id; |
301 | struct xpc_partition *part = &xpc_partitions[partid]; | |
302 | ||
261f3b49 | 303 | DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); |
a47d5dac DN |
304 | |
305 | if (xpc_part_ref(part)) { | |
7fb5e59d | 306 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
307 | |
308 | xpc_part_deref(part); | |
309 | } | |
310 | return IRQ_HANDLED; | |
33ba3c77 DN |
311 | } |
312 | ||
313 | /* | |
7fb5e59d DN |
314 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
315 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 316 | * was received. |
33ba3c77 | 317 | */ |
a47d5dac | 318 | static void |
7fb5e59d | 319 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
320 | { |
321 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
322 | ||
323 | if (xpc_part_ref(part)) { | |
7fb5e59d | 324 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
325 | |
326 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 327 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
328 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
329 | xpc_part_deref(part); | |
330 | } | |
331 | } | |
33ba3c77 DN |
332 | |
333 | /* | |
7fb5e59d | 334 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
335 | * specified channel. |
336 | */ | |
337 | static void | |
7fb5e59d DN |
338 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
339 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
340 | { |
341 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 342 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 343 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
344 | enum xp_retval ret; |
345 | ||
83469b55 | 346 | if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) { |
7fb5e59d DN |
347 | chctl.flags[ch->number] = chctl_flag; |
348 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
349 | chctl.all_flags, | |
350 | part_sn2->notify_IRQ_nasid, | |
351 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
352 | SGI_XPC_NOTIFY); |
353 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 354 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
355 | if (unlikely(ret != xpSuccess)) { |
356 | if (irq_flags != NULL) | |
357 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
358 | XPC_DEACTIVATE_PARTITION(part, ret); | |
359 | if (irq_flags != NULL) | |
360 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
361 | } | |
362 | } | |
363 | } | |
364 | ||
7fb5e59d DN |
365 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
366 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
367 | |
368 | /* | |
369 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
370 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
371 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
372 | */ |
373 | static void | |
7fb5e59d DN |
374 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
375 | char *chctl_flag_string) | |
33ba3c77 DN |
376 | { |
377 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 378 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 379 | |
7fb5e59d DN |
380 | chctl.flags[ch->number] = chctl_flag; |
381 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
382 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 383 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 384 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
385 | } |
386 | ||
7fb5e59d DN |
387 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
388 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
389 | |
390 | static void | |
7fb5e59d DN |
391 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
392 | unsigned long *irq_flags) | |
33ba3c77 | 393 | { |
5b8669df | 394 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
395 | |
396 | args->reason = ch->reason; | |
7fb5e59d | 397 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
398 | } |
399 | ||
400 | static void | |
7fb5e59d | 401 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 402 | { |
7fb5e59d | 403 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
404 | } |
405 | ||
406 | static void | |
7fb5e59d | 407 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 408 | { |
5b8669df | 409 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 | 410 | |
bd3e64c1 | 411 | args->entry_size = ch->entry_size; |
33ba3c77 | 412 | args->local_nentries = ch->local_nentries; |
7fb5e59d | 413 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
414 | } |
415 | ||
416 | static void | |
7fb5e59d | 417 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 418 | { |
5b8669df | 419 | struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args; |
33ba3c77 DN |
420 | |
421 | args->remote_nentries = ch->remote_nentries; | |
422 | args->local_nentries = ch->local_nentries; | |
5b8669df | 423 | args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue); |
7fb5e59d | 424 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
425 | } |
426 | ||
427 | static void | |
7fb5e59d | 428 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 429 | { |
7fb5e59d | 430 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
431 | } |
432 | ||
433 | static void | |
7fb5e59d | 434 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 435 | { |
7fb5e59d | 436 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
437 | } |
438 | ||
5b8669df DN |
439 | static void |
440 | xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, | |
441 | unsigned long msgqueue_pa) | |
442 | { | |
443 | ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; | |
444 | } | |
445 | ||
33ba3c77 DN |
446 | /* |
447 | * This next set of functions are used to keep track of when a partition is | |
448 | * potentially engaged in accessing memory belonging to another partition. | |
449 | */ | |
450 | ||
451 | static void | |
a47d5dac | 452 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
453 | { |
454 | unsigned long irq_flags; | |
c39838ce | 455 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 456 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 457 | sizeof(struct amo))); |
33ba3c77 DN |
458 | |
459 | local_irq_save(irq_flags); | |
460 | ||
c39838ce | 461 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 462 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
463 | BIT(sn_partition_id)); |
464 | ||
33ba3c77 DN |
465 | /* |
466 | * We must always use the nofault function regardless of whether we | |
467 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
468 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 469 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
470 | */ |
471 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
472 | variable), | |
473 | xp_nofault_PIOR_target)); | |
474 | ||
475 | local_irq_restore(irq_flags); | |
476 | } | |
477 | ||
478 | static void | |
a47d5dac | 479 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 480 | { |
a47d5dac | 481 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 482 | unsigned long irq_flags; |
c39838ce | 483 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 484 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 485 | sizeof(struct amo))); |
33ba3c77 DN |
486 | |
487 | local_irq_save(irq_flags); | |
488 | ||
c39838ce | 489 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 490 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
491 | ~BIT(sn_partition_id)); |
492 | ||
33ba3c77 DN |
493 | /* |
494 | * We must always use the nofault function regardless of whether we | |
495 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
496 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 497 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
498 | */ |
499 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
500 | variable), | |
501 | xp_nofault_PIOR_target)); | |
502 | ||
503 | local_irq_restore(irq_flags); | |
33ba3c77 | 504 | |
33ba3c77 | 505 | /* |
a47d5dac | 506 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 507 | * bit in their engaged partitions amo. |
33ba3c77 | 508 | */ |
7fb5e59d | 509 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
510 | cnodeid_to_nasid(0), |
511 | part_sn2->activate_IRQ_nasid, | |
512 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
513 | } |
514 | ||
5b8669df DN |
515 | static void |
516 | xpc_assume_partition_disengaged_sn2(short partid) | |
517 | { | |
518 | struct amo *amo = xpc_vars_sn2->amos_page + | |
519 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
520 | ||
521 | /* clear bit(s) based on partid mask in our partition's amo */ | |
522 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, | |
523 | ~BIT(partid)); | |
524 | } | |
525 | ||
a47d5dac DN |
526 | static int |
527 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 528 | { |
8e85c23e DN |
529 | struct amo *amo = xpc_vars_sn2->amos_page + |
530 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 531 | |
c39838ce | 532 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 533 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 534 | BIT(partid)) != 0; |
33ba3c77 DN |
535 | } |
536 | ||
a47d5dac DN |
537 | static int |
538 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 539 | { |
8e85c23e DN |
540 | struct amo *amo = xpc_vars_sn2->amos_page + |
541 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 542 | |
c39838ce | 543 | /* our partition's amo variable */ |
a47d5dac | 544 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
545 | } |
546 | ||
6e41017a DN |
547 | /* original protection values for each node */ |
548 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
549 | ||
550 | /* | |
c39838ce | 551 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
552 | */ |
553 | static enum xp_retval | |
c39838ce | 554 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a | 555 | { |
6c1c325d | 556 | enum xp_retval ret = xpSuccess; |
6e41017a DN |
557 | |
558 | /* | |
559 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
560 | * collides with memory operations. On those systems we call | |
c39838ce | 561 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a | 562 | */ |
6c1c325d DN |
563 | if (!enable_shub_wars_1_1()) |
564 | ret = xp_expand_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE); | |
565 | ||
566 | return ret; | |
6e41017a DN |
567 | } |
568 | ||
569 | /* | |
c39838ce | 570 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
571 | */ |
572 | static void | |
c39838ce | 573 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
574 | { |
575 | int node; | |
576 | int nasid; | |
577 | ||
578 | if (!enable_shub_wars_1_1()) | |
579 | return; | |
580 | ||
581 | for_each_online_node(node) { | |
582 | nasid = cnodeid_to_nasid(node); | |
583 | /* save current protection values */ | |
584 | xpc_prot_vec_sn2[node] = | |
585 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
586 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
587 | /* open up everything */ | |
588 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
589 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
590 | -1UL); | |
591 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
592 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
593 | -1UL); | |
594 | } | |
595 | } | |
596 | ||
261f3b49 | 597 | static enum xp_retval |
a812dcc3 | 598 | xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, |
261f3b49 DN |
599 | size_t *len) |
600 | { | |
601 | s64 status; | |
602 | enum xp_retval ret; | |
603 | ||
a812dcc3 | 604 | status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len); |
261f3b49 DN |
605 | if (status == SALRET_OK) |
606 | ret = xpSuccess; | |
607 | else if (status == SALRET_MORE_PASSES) | |
608 | ret = xpNeedMoreInfo; | |
609 | else | |
610 | ret = xpSalError; | |
611 | ||
612 | return ret; | |
613 | } | |
614 | ||
615 | ||
5b8669df DN |
616 | static int |
617 | xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page *rp) | |
94bd2708 | 618 | { |
c39838ce | 619 | struct amo *amos_page; |
94bd2708 DN |
620 | int i; |
621 | int ret; | |
622 | ||
8e85c23e | 623 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 624 | |
a812dcc3 | 625 | rp->sn.vars_pa = xp_pa(xpc_vars_sn2); |
94bd2708 | 626 | |
e17d416b | 627 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
628 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
629 | XPC_RP_VARS_SIZE); | |
e17d416b | 630 | |
94bd2708 | 631 | /* |
8e85c23e DN |
632 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
633 | * previously allocated. If not we'll need to allocate one and set | |
634 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 635 | * |
c39838ce | 636 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 637 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 638 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 639 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 640 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
641 | * memory protections are never restricted. |
642 | */ | |
8e85c23e | 643 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 644 | if (amos_page == NULL) { |
c39838ce | 645 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 646 | if (amos_page == NULL) { |
c39838ce | 647 | dev_err(xpc_part, "can't allocate page of amos\n"); |
5b8669df | 648 | return -ENOMEM; |
94bd2708 DN |
649 | } |
650 | ||
651 | /* | |
c39838ce DN |
652 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
653 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 654 | */ |
c39838ce | 655 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 656 | if (ret != xpSuccess) { |
c39838ce | 657 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
658 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
659 | TO_PHYS((u64)amos_page), 1); | |
5b8669df | 660 | return -EPERM; |
94bd2708 DN |
661 | } |
662 | } | |
663 | ||
8e85c23e DN |
664 | /* clear xpc_vars_sn2 */ |
665 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 666 | |
8e85c23e DN |
667 | xpc_vars_sn2->version = XPC_V_VERSION; |
668 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
669 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
a812dcc3 | 670 | xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); |
8e85c23e DN |
671 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); |
672 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 673 | |
8e85c23e DN |
674 | /* clear xpc_vars_part_sn2 */ |
675 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
261f3b49 | 676 | XP_MAX_NPARTITIONS_SN2); |
94bd2708 | 677 | |
c39838ce | 678 | /* initialize the activate IRQ related amo variables */ |
04de7418 | 679 | for (i = 0; i < xpc_nasid_mask_nlongs; i++) |
ee6665e3 | 680 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); |
94bd2708 | 681 | |
c39838ce | 682 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
683 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
684 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 | 685 | |
5b8669df | 686 | return 0; |
33ba3c77 DN |
687 | } |
688 | ||
689 | static void | |
690 | xpc_increment_heartbeat_sn2(void) | |
691 | { | |
8e85c23e | 692 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
693 | } |
694 | ||
695 | static void | |
696 | xpc_offline_heartbeat_sn2(void) | |
697 | { | |
698 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 699 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
700 | } |
701 | ||
702 | static void | |
703 | xpc_online_heartbeat_sn2(void) | |
704 | { | |
705 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 706 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
707 | } |
708 | ||
709 | static void | |
710 | xpc_heartbeat_init_sn2(void) | |
711 | { | |
8e85c23e | 712 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 713 | |
8e85c23e DN |
714 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
715 | xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0]; | |
33ba3c77 DN |
716 | xpc_online_heartbeat_sn2(); |
717 | } | |
718 | ||
719 | static void | |
720 | xpc_heartbeat_exit_sn2(void) | |
721 | { | |
722 | xpc_offline_heartbeat_sn2(); | |
723 | } | |
724 | ||
61deb86e DN |
725 | static enum xp_retval |
726 | xpc_get_remote_heartbeat_sn2(struct xpc_partition *part) | |
33ba3c77 DN |
727 | { |
728 | struct xpc_vars_sn2 *remote_vars; | |
33ba3c77 DN |
729 | enum xp_retval ret; |
730 | ||
ee6665e3 | 731 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 | 732 | |
61deb86e DN |
733 | /* pull the remote vars structure that contains the heartbeat */ |
734 | ret = xp_remote_memcpy(xp_pa(remote_vars), | |
735 | part->sn.sn2.remote_vars_pa, | |
736 | XPC_RP_VARS_SIZE); | |
737 | if (ret != xpSuccess) | |
738 | return ret; | |
33ba3c77 | 739 | |
61deb86e DN |
740 | dev_dbg(xpc_part, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, " |
741 | "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part), | |
742 | remote_vars->heartbeat, part->last_heartbeat, | |
743 | remote_vars->heartbeat_offline, | |
744 | remote_vars->heartbeating_to_mask[0]); | |
745 | ||
746 | if ((remote_vars->heartbeat == part->last_heartbeat && | |
747 | remote_vars->heartbeat_offline == 0) || | |
748 | !xpc_hb_allowed(sn_partition_id, | |
749 | &remote_vars->heartbeating_to_mask)) { | |
750 | ret = xpNoHeartbeat; | |
751 | } else { | |
33ba3c77 DN |
752 | part->last_heartbeat = remote_vars->heartbeat; |
753 | } | |
61deb86e DN |
754 | |
755 | return ret; | |
33ba3c77 DN |
756 | } |
757 | ||
758 | /* | |
759 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
760 | * | |
761 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
762 | * assumed to be of size XPC_RP_VARS_SIZE. | |
763 | */ | |
764 | static enum xp_retval | |
a812dcc3 DN |
765 | xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, |
766 | struct xpc_vars_sn2 *remote_vars) | |
33ba3c77 DN |
767 | { |
768 | enum xp_retval ret; | |
769 | ||
770 | if (remote_vars_pa == 0) | |
771 | return xpVarsNotSet; | |
772 | ||
773 | /* pull over the cross partition variables */ | |
a812dcc3 | 774 | ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, |
33ba3c77 DN |
775 | XPC_RP_VARS_SIZE); |
776 | if (ret != xpSuccess) | |
777 | return ret; | |
778 | ||
779 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
780 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
781 | return xpBadVersion; | |
782 | } | |
94bd2708 DN |
783 | |
784 | return xpSuccess; | |
785 | } | |
786 | ||
33ba3c77 | 787 | static void |
a47d5dac | 788 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
a812dcc3 | 789 | unsigned long remote_rp_pa, int nasid) |
33ba3c77 | 790 | { |
7fb5e59d | 791 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
792 | } |
793 | ||
794 | static void | |
795 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
796 | { | |
7fb5e59d | 797 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
798 | } |
799 | ||
800 | static void | |
801 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
802 | { | |
803 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
804 | unsigned long irq_flags; | |
c39838ce | 805 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 806 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 807 | sizeof(struct amo))); |
a47d5dac DN |
808 | |
809 | local_irq_save(irq_flags); | |
810 | ||
c39838ce | 811 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 812 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
813 | BIT(sn_partition_id)); |
814 | ||
a47d5dac DN |
815 | /* |
816 | * We must always use the nofault function regardless of whether we | |
817 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
818 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 819 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
820 | */ |
821 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
822 | variable), | |
823 | xp_nofault_PIOR_target)); | |
824 | ||
825 | local_irq_restore(irq_flags); | |
826 | ||
827 | /* | |
828 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 829 | * bit in their deactivate request amo. |
a47d5dac | 830 | */ |
7fb5e59d | 831 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
832 | cnodeid_to_nasid(0), |
833 | part_sn2->activate_IRQ_nasid, | |
834 | part_sn2->activate_IRQ_phys_cpuid); | |
835 | } | |
836 | ||
837 | static void | |
838 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
839 | { | |
840 | unsigned long irq_flags; | |
c39838ce | 841 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 842 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 843 | sizeof(struct amo))); |
a47d5dac DN |
844 | |
845 | local_irq_save(irq_flags); | |
846 | ||
c39838ce | 847 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 848 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
849 | ~BIT(sn_partition_id)); |
850 | ||
a47d5dac DN |
851 | /* |
852 | * We must always use the nofault function regardless of whether we | |
853 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
854 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 855 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
856 | */ |
857 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
858 | variable), | |
859 | xp_nofault_PIOR_target)); | |
860 | ||
861 | local_irq_restore(irq_flags); | |
862 | } | |
863 | ||
864 | static int | |
865 | xpc_partition_deactivation_requested_sn2(short partid) | |
866 | { | |
8e85c23e DN |
867 | struct amo *amo = xpc_vars_sn2->amos_page + |
868 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 869 | |
c39838ce | 870 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac | 871 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 872 | BIT(partid)) != 0; |
33ba3c77 DN |
873 | } |
874 | ||
875 | /* | |
876 | * Update the remote partition's info. | |
877 | */ | |
878 | static void | |
879 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
81fe7883 | 880 | unsigned long *remote_rp_ts_jiffies, |
a812dcc3 DN |
881 | unsigned long remote_rp_pa, |
882 | unsigned long remote_vars_pa, | |
33ba3c77 DN |
883 | struct xpc_vars_sn2 *remote_vars) |
884 | { | |
a47d5dac DN |
885 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
886 | ||
33ba3c77 DN |
887 | part->remote_rp_version = remote_rp_version; |
888 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
889 | part->remote_rp_version); | |
890 | ||
81fe7883 DN |
891 | part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; |
892 | dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", | |
893 | part->remote_rp_ts_jiffies); | |
33ba3c77 DN |
894 | |
895 | part->remote_rp_pa = remote_rp_pa; | |
896 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
897 | ||
a47d5dac | 898 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 899 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 900 | part_sn2->remote_vars_pa); |
33ba3c77 DN |
901 | |
902 | part->last_heartbeat = remote_vars->heartbeat; | |
903 | dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n", | |
904 | part->last_heartbeat); | |
905 | ||
a47d5dac | 906 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 907 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 908 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 909 | |
a47d5dac DN |
910 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
911 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
912 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 913 | |
a47d5dac DN |
914 | part_sn2->activate_IRQ_phys_cpuid = |
915 | remote_vars->activate_IRQ_phys_cpuid; | |
916 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
917 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 918 | |
a47d5dac | 919 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 920 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 921 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 922 | |
a47d5dac | 923 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 924 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 925 | part_sn2->remote_vars_version); |
33ba3c77 DN |
926 | } |
927 | ||
928 | /* | |
7fb5e59d DN |
929 | * Prior code has determined the nasid which generated a activate IRQ. |
930 | * Inspect that nasid to determine if its partition needs to be activated | |
931 | * or deactivated. | |
33ba3c77 | 932 | * |
7fb5e59d | 933 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
934 | * flags indicate it is not active and it has a heartbeat. A |
935 | * partition is considered "awaiting deactivation" if our partition | |
936 | * flags indicate it is active but it has no heartbeat or it is not | |
937 | * sending its heartbeat to us. | |
938 | * | |
939 | * To determine the heartbeat, the remote nasid must have a properly | |
940 | * initialized reserved page. | |
941 | */ | |
942 | static void | |
6e41017a | 943 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
944 | { |
945 | struct xpc_rsvd_page *remote_rp; | |
946 | struct xpc_vars_sn2 *remote_vars; | |
a812dcc3 DN |
947 | unsigned long remote_rp_pa; |
948 | unsigned long remote_vars_pa; | |
33ba3c77 DN |
949 | int remote_rp_version; |
950 | int reactivate = 0; | |
81fe7883 | 951 | unsigned long remote_rp_ts_jiffies = 0; |
33ba3c77 DN |
952 | short partid; |
953 | struct xpc_partition *part; | |
a47d5dac | 954 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
955 | enum xp_retval ret; |
956 | ||
957 | /* pull over the reserved page structure */ | |
958 | ||
ee6665e3 | 959 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
960 | |
961 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
962 | if (ret != xpSuccess) { | |
963 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
964 | "which sent interrupt, reason=%d\n", nasid, ret); | |
965 | return; | |
966 | } | |
967 | ||
968 | remote_vars_pa = remote_rp->sn.vars_pa; | |
969 | remote_rp_version = remote_rp->version; | |
81fe7883 | 970 | remote_rp_ts_jiffies = remote_rp->ts_jiffies; |
33ba3c77 DN |
971 | |
972 | partid = remote_rp->SAL_partid; | |
973 | part = &xpc_partitions[partid]; | |
a47d5dac | 974 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
975 | |
976 | /* pull over the cross partition variables */ | |
977 | ||
ee6665e3 | 978 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
979 | |
980 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
981 | if (ret != xpSuccess) { | |
33ba3c77 DN |
982 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
983 | "which sent interrupt, reason=%d\n", nasid, ret); | |
984 | ||
985 | XPC_DEACTIVATE_PARTITION(part, ret); | |
986 | return; | |
987 | } | |
988 | ||
6e41017a | 989 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
990 | |
991 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
6e41017a | 992 | "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd, |
33ba3c77 DN |
993 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
994 | ||
995 | if (xpc_partition_disengaged(part) && | |
83469b55 | 996 | part->act_state == XPC_P_AS_INACTIVE) { |
33ba3c77 DN |
997 | |
998 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
999 | &remote_rp_ts_jiffies, |
1000 | remote_rp_pa, remote_vars_pa, | |
1001 | remote_vars); | |
33ba3c77 | 1002 | |
a47d5dac DN |
1003 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
1004 | /* | |
1005 | * Other side is waiting on us to deactivate even though | |
1006 | * we already have. | |
1007 | */ | |
1008 | return; | |
33ba3c77 DN |
1009 | } |
1010 | ||
1011 | xpc_activate_partition(part); | |
1012 | return; | |
1013 | } | |
1014 | ||
1015 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 1016 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 1017 | |
81fe7883 | 1018 | if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { |
33ba3c77 | 1019 | |
a47d5dac | 1020 | /* the other side rebooted */ |
33ba3c77 | 1021 | |
a47d5dac DN |
1022 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1023 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1024 | |
1025 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1026 | &remote_rp_ts_jiffies, |
1027 | remote_rp_pa, remote_vars_pa, | |
1028 | remote_vars); | |
33ba3c77 | 1029 | reactivate = 1; |
33ba3c77 DN |
1030 | } |
1031 | ||
a47d5dac | 1032 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1033 | /* still waiting on other side to disengage from us */ |
1034 | return; | |
1035 | } | |
1036 | ||
a47d5dac | 1037 | if (reactivate) |
33ba3c77 | 1038 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1039 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1040 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1041 | } |
1042 | ||
1043 | /* | |
c39838ce | 1044 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1045 | * which are set. Each bit indicates a nasid sending a partition |
1046 | * activation or deactivation request. | |
1047 | * | |
1048 | * Return #of IRQs detected. | |
1049 | */ | |
1050 | int | |
6e41017a | 1051 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 | 1052 | { |
04de7418 DN |
1053 | int l; |
1054 | int b; | |
1055 | unsigned long nasid_mask_long; | |
33ba3c77 DN |
1056 | u64 nasid; /* remote nasid */ |
1057 | int n_IRQs_detected = 0; | |
c39838ce | 1058 | struct amo *act_amos; |
33ba3c77 | 1059 | |
8e85c23e | 1060 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1061 | |
04de7418 DN |
1062 | /* scan through activate amo variables looking for non-zero entries */ |
1063 | for (l = 0; l < xpc_nasid_mask_nlongs; l++) { | |
33ba3c77 DN |
1064 | |
1065 | if (xpc_exiting) | |
1066 | break; | |
1067 | ||
04de7418 DN |
1068 | nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); |
1069 | ||
1070 | b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); | |
1071 | if (b >= BITS_PER_LONG) { | |
1072 | /* no IRQs from nasids in this amo variable */ | |
33ba3c77 DN |
1073 | continue; |
1074 | } | |
1075 | ||
04de7418 DN |
1076 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, |
1077 | nasid_mask_long); | |
33ba3c77 DN |
1078 | |
1079 | /* | |
1080 | * If this nasid has been added to the machine since | |
1081 | * our partition was reset, this will retain the | |
1082 | * remote nasid in our reserved pages machine mask. | |
1083 | * This is used in the event of module reload. | |
1084 | */ | |
04de7418 | 1085 | xpc_mach_nasids[l] |= nasid_mask_long; |
33ba3c77 DN |
1086 | |
1087 | /* locate the nasid(s) which sent interrupts */ | |
1088 | ||
04de7418 DN |
1089 | do { |
1090 | n_IRQs_detected++; | |
1091 | nasid = (l * BITS_PER_LONG + b) * 2; | |
1092 | dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid); | |
1093 | xpc_identify_activate_IRQ_req_sn2(nasid); | |
1094 | ||
1095 | b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, | |
1096 | b + 1); | |
1097 | } while (b < BITS_PER_LONG); | |
33ba3c77 DN |
1098 | } |
1099 | return n_IRQs_detected; | |
1100 | } | |
1101 | ||
1102 | static void | |
5b8669df | 1103 | xpc_process_activate_IRQ_rcvd_sn2(void) |
33ba3c77 | 1104 | { |
5b8669df DN |
1105 | unsigned long irq_flags; |
1106 | int n_IRQs_expected; | |
33ba3c77 DN |
1107 | int n_IRQs_detected; |
1108 | ||
5b8669df DN |
1109 | DBUG_ON(xpc_activate_IRQ_rcvd == 0); |
1110 | ||
1111 | spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
1112 | n_IRQs_expected = xpc_activate_IRQ_rcvd; | |
1113 | xpc_activate_IRQ_rcvd = 0; | |
1114 | spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); | |
1115 | ||
6e41017a | 1116 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1117 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1118 | /* retry once to help avoid missing amo */ |
6e41017a | 1119 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1120 | } |
1121 | } | |
1122 | ||
185c3a1b | 1123 | /* |
5b8669df | 1124 | * Setup the channel structures that are sn2 specific. |
e17d416b DN |
1125 | */ |
1126 | static enum xp_retval | |
5b8669df | 1127 | xpc_setup_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1128 | { |
a47d5dac | 1129 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
5b8669df | 1130 | struct xpc_channel_sn2 *ch_sn2; |
e17d416b DN |
1131 | enum xp_retval retval; |
1132 | int ret; | |
1133 | int cpuid; | |
1134 | int ch_number; | |
e17d416b DN |
1135 | struct timer_list *timer; |
1136 | short partid = XPC_PARTID(part); | |
1137 | ||
e17d416b DN |
1138 | /* allocate all the required GET/PUT values */ |
1139 | ||
185c3a1b | 1140 | part_sn2->local_GPs = |
5b8669df DN |
1141 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1142 | &part_sn2->local_GPs_base); | |
a47d5dac | 1143 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1144 | dev_err(xpc_chan, "can't get memory for local get/put " |
1145 | "values\n"); | |
5b8669df | 1146 | return xpNoMemory; |
e17d416b DN |
1147 | } |
1148 | ||
185c3a1b | 1149 | part_sn2->remote_GPs = |
5b8669df DN |
1150 | xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL, |
1151 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1152 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1153 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1154 | "values\n"); | |
1155 | retval = xpNoMemory; | |
5b8669df | 1156 | goto out_1; |
e17d416b DN |
1157 | } |
1158 | ||
a47d5dac | 1159 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1160 | |
1161 | /* allocate all the required open and close args */ | |
1162 | ||
5b8669df DN |
1163 | part_sn2->local_openclose_args = |
1164 | xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE, | |
1165 | GFP_KERNEL, &part_sn2-> | |
1166 | local_openclose_args_base); | |
1167 | if (part_sn2->local_openclose_args == NULL) { | |
e17d416b DN |
1168 | dev_err(xpc_chan, "can't get memory for local connect args\n"); |
1169 | retval = xpNoMemory; | |
5b8669df | 1170 | goto out_2; |
e17d416b DN |
1171 | } |
1172 | ||
a47d5dac | 1173 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1174 | |
7fb5e59d | 1175 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
e17d416b | 1176 | |
7fb5e59d DN |
1177 | part_sn2->notify_IRQ_nasid = 0; |
1178 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1179 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b | 1180 | |
7fb5e59d | 1181 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1182 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1183 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1184 | (void *)(u64)partid); |
e17d416b DN |
1185 | if (ret != 0) { |
1186 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1187 | "errno=%d\n", -ret); | |
1188 | retval = xpLackOfResources; | |
5b8669df | 1189 | goto out_3; |
e17d416b DN |
1190 | } |
1191 | ||
7fb5e59d | 1192 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1193 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1194 | init_timer(timer); |
a47d5dac | 1195 | timer->function = |
7fb5e59d | 1196 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1197 | timer->data = (unsigned long)part; |
7fb5e59d | 1198 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1199 | add_timer(timer); |
1200 | ||
e17d416b | 1201 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { |
5b8669df | 1202 | ch_sn2 = &part->channels[ch_number].sn.sn2; |
e17d416b | 1203 | |
5b8669df DN |
1204 | ch_sn2->local_GP = &part_sn2->local_GPs[ch_number]; |
1205 | ch_sn2->local_openclose_args = | |
1206 | &part_sn2->local_openclose_args[ch_number]; | |
e17d416b | 1207 | |
5b8669df | 1208 | mutex_init(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1209 | } |
1210 | ||
e17d416b DN |
1211 | /* |
1212 | * Setup the per partition specific variables required by the | |
1213 | * remote partition to establish channel connections with us. | |
1214 | * | |
1215 | * The setting of the magic # indicates that these per partition | |
1216 | * specific variables are ready to be used. | |
1217 | */ | |
a812dcc3 | 1218 | xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); |
8e85c23e | 1219 | xpc_vars_part_sn2[partid].openclose_args_pa = |
5b8669df | 1220 | xp_pa(part_sn2->local_openclose_args); |
8e85c23e | 1221 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
a812dcc3 | 1222 | xp_pa(part_sn2->local_chctl_amo_va); |
e17d416b | 1223 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1224 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1225 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1226 | cpu_physical_id(cpuid); | |
1227 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
5b8669df | 1228 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2; |
e17d416b DN |
1229 | |
1230 | return xpSuccess; | |
1231 | ||
5b8669df | 1232 | /* setup of ch structures failed */ |
e17d416b | 1233 | out_3: |
5b8669df DN |
1234 | kfree(part_sn2->local_openclose_args_base); |
1235 | part_sn2->local_openclose_args = NULL; | |
1236 | out_2: | |
a47d5dac DN |
1237 | kfree(part_sn2->remote_GPs_base); |
1238 | part_sn2->remote_GPs = NULL; | |
5b8669df | 1239 | out_1: |
a47d5dac DN |
1240 | kfree(part_sn2->local_GPs_base); |
1241 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1242 | return retval; |
1243 | } | |
1244 | ||
1245 | /* | |
5b8669df | 1246 | * Teardown the channel structures that are sn2 specific. |
e17d416b DN |
1247 | */ |
1248 | static void | |
5b8669df | 1249 | xpc_teardown_ch_structures_sn_sn2(struct xpc_partition *part) |
e17d416b | 1250 | { |
a47d5dac | 1251 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1252 | short partid = XPC_PARTID(part); |
1253 | ||
1254 | /* | |
5b8669df DN |
1255 | * Indicate that the variables specific to the remote partition are no |
1256 | * longer available for its use. | |
e17d416b | 1257 | */ |
8e85c23e | 1258 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b | 1259 | |
e17d416b | 1260 | /* in case we've still got outstanding timers registered... */ |
a47d5dac | 1261 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
5b8669df | 1262 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); |
e17d416b | 1263 | |
5b8669df DN |
1264 | kfree(part_sn2->local_openclose_args_base); |
1265 | part_sn2->local_openclose_args = NULL; | |
a47d5dac DN |
1266 | kfree(part_sn2->remote_GPs_base); |
1267 | part_sn2->remote_GPs = NULL; | |
1268 | kfree(part_sn2->local_GPs_base); | |
1269 | part_sn2->local_GPs = NULL; | |
7fb5e59d | 1270 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1271 | } |
1272 | ||
1273 | /* | |
1274 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1275 | * (or multiple cachelines) from a remote partition. | |
1276 | * | |
a812dcc3 | 1277 | * src_pa must be a cacheline aligned physical address on the remote partition. |
e17d416b DN |
1278 | * dst must be a cacheline aligned virtual address on this partition. |
1279 | * cnt must be cacheline sized | |
1280 | */ | |
ea57f80c | 1281 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1282 | static enum xp_retval |
1283 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
a812dcc3 | 1284 | const unsigned long src_pa, size_t cnt) |
e17d416b DN |
1285 | { |
1286 | enum xp_retval ret; | |
1287 | ||
a812dcc3 DN |
1288 | DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); |
1289 | DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); | |
e17d416b DN |
1290 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); |
1291 | ||
83469b55 | 1292 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1293 | return part->reason; |
1294 | ||
a812dcc3 | 1295 | ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); |
e17d416b DN |
1296 | if (ret != xpSuccess) { |
1297 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1298 | " ret=%d\n", XPC_PARTID(part), ret); | |
1299 | } | |
1300 | return ret; | |
1301 | } | |
1302 | ||
1303 | /* | |
1304 | * Pull the remote per partition specific variables from the specified | |
1305 | * partition. | |
1306 | */ | |
1307 | static enum xp_retval | |
1308 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1309 | { | |
a47d5dac | 1310 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1311 | u8 buffer[L1_CACHE_BYTES * 2]; |
1312 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1313 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1314 | struct xpc_vars_part_sn2 *pulled_entry; | |
a812dcc3 DN |
1315 | unsigned long remote_entry_cacheline_pa; |
1316 | unsigned long remote_entry_pa; | |
e17d416b DN |
1317 | short partid = XPC_PARTID(part); |
1318 | enum xp_retval ret; | |
1319 | ||
1320 | /* pull the cacheline that contains the variables we're interested in */ | |
1321 | ||
a47d5dac DN |
1322 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1323 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1324 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1325 | ||
a47d5dac | 1326 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1327 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1328 | ||
1329 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1330 | ||
1331 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1332 | + (remote_entry_pa & | |
1333 | (L1_CACHE_BYTES - 1))); | |
1334 | ||
1335 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
a812dcc3 | 1336 | remote_entry_cacheline_pa, |
e17d416b DN |
1337 | L1_CACHE_BYTES); |
1338 | if (ret != xpSuccess) { | |
1339 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1340 | "partition %d, ret=%d\n", partid, ret); | |
1341 | return ret; | |
1342 | } | |
1343 | ||
1344 | /* see if they've been set up yet */ | |
1345 | ||
5b8669df DN |
1346 | if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 && |
1347 | pulled_entry->magic != XPC_VP_MAGIC2_SN2) { | |
e17d416b DN |
1348 | |
1349 | if (pulled_entry->magic != 0) { | |
1350 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
1351 | "partition %d has bad magic value (=0x%lx)\n", | |
1352 | partid, sn_partition_id, pulled_entry->magic); | |
1353 | return xpBadMagic; | |
1354 | } | |
1355 | ||
1356 | /* they've not been initialized yet */ | |
1357 | return xpRetry; | |
1358 | } | |
1359 | ||
5b8669df | 1360 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) { |
e17d416b DN |
1361 | |
1362 | /* validate the variables */ | |
1363 | ||
1364 | if (pulled_entry->GPs_pa == 0 || | |
1365 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1366 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1367 | |
1368 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1369 | "partition %d are not valid\n", partid, | |
1370 | sn_partition_id); | |
1371 | return xpInvalidAddress; | |
1372 | } | |
1373 | ||
1374 | /* the variables we imported look to be valid */ | |
1375 | ||
a47d5dac DN |
1376 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1377 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1378 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1379 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1380 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1381 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1382 | part_sn2->notify_IRQ_phys_cpuid = | |
1383 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1384 | |
1385 | if (part->nchannels > pulled_entry->nchannels) | |
1386 | part->nchannels = pulled_entry->nchannels; | |
1387 | ||
1388 | /* let the other side know that we've pulled their variables */ | |
1389 | ||
5b8669df | 1390 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2; |
e17d416b DN |
1391 | } |
1392 | ||
5b8669df | 1393 | if (pulled_entry->magic == XPC_VP_MAGIC1_SN2) |
e17d416b DN |
1394 | return xpRetry; |
1395 | ||
1396 | return xpSuccess; | |
1397 | } | |
1398 | ||
1399 | /* | |
1400 | * Establish first contact with the remote partititon. This involves pulling | |
1401 | * the XPC per partition variables from the remote partition and waiting for | |
1402 | * the remote partition to pull ours. | |
1403 | */ | |
1404 | static enum xp_retval | |
1405 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1406 | { | |
a47d5dac | 1407 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1408 | enum xp_retval ret; |
1409 | ||
33ba3c77 | 1410 | /* |
c39838ce | 1411 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1412 | * and cleanup errors within that address range should the remote |
1413 | * partition go down. We don't unregister this range because it is | |
1414 | * difficult to tell when outstanding writes to the remote partition | |
1415 | * are finished and thus when it is safe to unregister. This should | |
1416 | * not result in wasted space in the SAL xp_addr_region table because | |
1417 | * we should get the same page for remote_amos_page_pa after module | |
1418 | * reloads and system reboots. | |
1419 | */ | |
a47d5dac | 1420 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1421 | PAGE_SIZE, 1) < 0) { |
1422 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1423 | "xp_addr region\n", XPC_PARTID(part)); | |
1424 | ||
1425 | ret = xpPhysAddrRegFailed; | |
1426 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1427 | return ret; | |
1428 | } | |
1429 | ||
a47d5dac DN |
1430 | /* |
1431 | * Send activate IRQ to get other side to activate if they've not | |
1432 | * already begun to do so. | |
1433 | */ | |
7fb5e59d | 1434 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1435 | cnodeid_to_nasid(0), |
1436 | part_sn2->activate_IRQ_nasid, | |
1437 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1438 | |
e17d416b DN |
1439 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1440 | if (ret != xpRetry) { | |
1441 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1442 | return ret; | |
1443 | } | |
1444 | ||
1445 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1446 | "partition %d\n", XPC_PARTID(part)); | |
1447 | ||
1448 | /* wait a 1/4 of a second or so */ | |
1449 | (void)msleep_interruptible(250); | |
1450 | ||
83469b55 | 1451 | if (part->act_state == XPC_P_AS_DEACTIVATING) |
e17d416b DN |
1452 | return part->reason; |
1453 | } | |
1454 | ||
1455 | return xpSuccess; | |
1456 | } | |
1457 | ||
1458 | /* | |
7fb5e59d | 1459 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1460 | */ |
1461 | static u64 | |
7fb5e59d | 1462 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1463 | { |
a47d5dac | 1464 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1465 | unsigned long irq_flags; |
7fb5e59d | 1466 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1467 | enum xp_retval ret; |
1468 | ||
1469 | /* | |
7fb5e59d | 1470 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1471 | */ |
1472 | ||
7fb5e59d DN |
1473 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1474 | chctl = part->chctl; | |
1475 | if (chctl.all_flags != 0) | |
1476 | part->chctl.all_flags = 0; | |
e17d416b | 1477 | |
7fb5e59d | 1478 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1479 | |
7fb5e59d | 1480 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1481 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1482 | remote_openclose_args, | |
a812dcc3 | 1483 | part_sn2-> |
e17d416b DN |
1484 | remote_openclose_args_pa, |
1485 | XPC_OPENCLOSE_ARGS_SIZE); | |
1486 | if (ret != xpSuccess) { | |
1487 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1488 | ||
1489 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1490 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1491 | ret); | |
1492 | ||
7fb5e59d DN |
1493 | /* don't bother processing chctl flags anymore */ |
1494 | chctl.all_flags = 0; | |
e17d416b DN |
1495 | } |
1496 | } | |
1497 | ||
7fb5e59d | 1498 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac | 1499 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
a812dcc3 | 1500 | part_sn2->remote_GPs_pa, |
e17d416b DN |
1501 | XPC_GP_SIZE); |
1502 | if (ret != xpSuccess) { | |
1503 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1504 | ||
1505 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1506 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1507 | ||
7fb5e59d DN |
1508 | /* don't bother processing chctl flags anymore */ |
1509 | chctl.all_flags = 0; | |
e17d416b DN |
1510 | } |
1511 | } | |
1512 | ||
7fb5e59d | 1513 | return chctl.all_flags; |
e17d416b DN |
1514 | } |
1515 | ||
185c3a1b DN |
1516 | /* |
1517 | * Allocate the local message queue and the notify queue. | |
1518 | */ | |
1519 | static enum xp_retval | |
1520 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1521 | { | |
5b8669df | 1522 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1523 | unsigned long irq_flags; |
1524 | int nentries; | |
1525 | size_t nbytes; | |
1526 | ||
1527 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1528 | ||
bd3e64c1 | 1529 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1530 | ch_sn2->local_msgqueue = |
1531 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, | |
1532 | &ch_sn2->local_msgqueue_base); | |
1533 | if (ch_sn2->local_msgqueue == NULL) | |
185c3a1b DN |
1534 | continue; |
1535 | ||
bd3e64c1 | 1536 | nbytes = nentries * sizeof(struct xpc_notify_sn2); |
5b8669df DN |
1537 | ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL); |
1538 | if (ch_sn2->notify_queue == NULL) { | |
1539 | kfree(ch_sn2->local_msgqueue_base); | |
1540 | ch_sn2->local_msgqueue = NULL; | |
185c3a1b DN |
1541 | continue; |
1542 | } | |
1543 | ||
1544 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1545 | if (nentries < ch->local_nentries) { | |
1546 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1547 | "partid=%d, channel=%d\n", nentries, | |
1548 | ch->local_nentries, ch->partid, ch->number); | |
1549 | ||
1550 | ch->local_nentries = nentries; | |
1551 | } | |
1552 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1553 | return xpSuccess; | |
1554 | } | |
1555 | ||
1556 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1557 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1558 | return xpNoMemory; | |
1559 | } | |
1560 | ||
1561 | /* | |
1562 | * Allocate the cached remote message queue. | |
1563 | */ | |
1564 | static enum xp_retval | |
1565 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1566 | { | |
5b8669df | 1567 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1568 | unsigned long irq_flags; |
1569 | int nentries; | |
1570 | size_t nbytes; | |
1571 | ||
1572 | DBUG_ON(ch->remote_nentries <= 0); | |
1573 | ||
1574 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1575 | ||
bd3e64c1 | 1576 | nbytes = nentries * ch->entry_size; |
5b8669df DN |
1577 | ch_sn2->remote_msgqueue = |
1578 | xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2-> | |
1579 | remote_msgqueue_base); | |
1580 | if (ch_sn2->remote_msgqueue == NULL) | |
185c3a1b DN |
1581 | continue; |
1582 | ||
1583 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1584 | if (nentries < ch->remote_nentries) { | |
1585 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1586 | "partid=%d, channel=%d\n", nentries, | |
1587 | ch->remote_nentries, ch->partid, ch->number); | |
1588 | ||
1589 | ch->remote_nentries = nentries; | |
1590 | } | |
1591 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1592 | return xpSuccess; | |
1593 | } | |
1594 | ||
1595 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1596 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1597 | return xpNoMemory; | |
1598 | } | |
1599 | ||
1600 | /* | |
1601 | * Allocate message queues and other stuff associated with a channel. | |
1602 | * | |
1603 | * Note: Assumes all of the channel sizes are filled in. | |
1604 | */ | |
1605 | static enum xp_retval | |
5b8669df | 1606 | xpc_setup_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b | 1607 | { |
5b8669df | 1608 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
185c3a1b DN |
1609 | enum xp_retval ret; |
1610 | ||
1611 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1612 | ||
1613 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1614 | if (ret == xpSuccess) { | |
1615 | ||
1616 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1617 | if (ret != xpSuccess) { | |
5b8669df DN |
1618 | kfree(ch_sn2->local_msgqueue_base); |
1619 | ch_sn2->local_msgqueue = NULL; | |
1620 | kfree(ch_sn2->notify_queue); | |
1621 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1622 | } |
1623 | } | |
1624 | return ret; | |
1625 | } | |
1626 | ||
1627 | /* | |
1628 | * Free up message queues and other stuff that were allocated for the specified | |
1629 | * channel. | |
185c3a1b DN |
1630 | */ |
1631 | static void | |
5b8669df | 1632 | xpc_teardown_msg_structures_sn2(struct xpc_channel *ch) |
185c3a1b DN |
1633 | { |
1634 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1635 | ||
1636 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
185c3a1b | 1637 | |
5b8669df | 1638 | ch_sn2->remote_msgqueue_pa = 0; |
185c3a1b DN |
1639 | |
1640 | ch_sn2->local_GP->get = 0; | |
1641 | ch_sn2->local_GP->put = 0; | |
1642 | ch_sn2->remote_GP.get = 0; | |
1643 | ch_sn2->remote_GP.put = 0; | |
1644 | ch_sn2->w_local_GP.get = 0; | |
1645 | ch_sn2->w_local_GP.put = 0; | |
1646 | ch_sn2->w_remote_GP.get = 0; | |
1647 | ch_sn2->w_remote_GP.put = 0; | |
1648 | ch_sn2->next_msg_to_pull = 0; | |
1649 | ||
1650 | if (ch->flags & XPC_C_SETUP) { | |
1651 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1652 | ch->flags, ch->partid, ch->number); | |
1653 | ||
5b8669df DN |
1654 | kfree(ch_sn2->local_msgqueue_base); |
1655 | ch_sn2->local_msgqueue = NULL; | |
1656 | kfree(ch_sn2->remote_msgqueue_base); | |
1657 | ch_sn2->remote_msgqueue = NULL; | |
1658 | kfree(ch_sn2->notify_queue); | |
1659 | ch_sn2->notify_queue = NULL; | |
185c3a1b DN |
1660 | } |
1661 | } | |
1662 | ||
a47d5dac DN |
1663 | /* |
1664 | * Notify those who wanted to be notified upon delivery of their message. | |
1665 | */ | |
1666 | static void | |
1667 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1668 | { | |
bd3e64c1 | 1669 | struct xpc_notify_sn2 *notify; |
a47d5dac DN |
1670 | u8 notify_type; |
1671 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1672 | ||
1673 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1674 | ||
5b8669df | 1675 | notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries]; |
a47d5dac DN |
1676 | |
1677 | /* | |
1678 | * See if the notify entry indicates it was associated with | |
1679 | * a message who's sender wants to be notified. It is possible | |
1680 | * that it is, but someone else is doing or has done the | |
1681 | * notification. | |
1682 | */ | |
1683 | notify_type = notify->type; | |
1684 | if (notify_type == 0 || | |
1685 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1686 | continue; | |
1687 | } | |
1688 | ||
1689 | DBUG_ON(notify_type != XPC_N_CALL); | |
1690 | ||
1691 | atomic_dec(&ch->n_to_notify); | |
1692 | ||
1693 | if (notify->func != NULL) { | |
bd3e64c1 DN |
1694 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p " |
1695 | "msg_number=%ld partid=%d channel=%d\n", | |
a47d5dac DN |
1696 | (void *)notify, get, ch->partid, ch->number); |
1697 | ||
1698 | notify->func(reason, ch->partid, ch->number, | |
1699 | notify->key); | |
1700 | ||
bd3e64c1 DN |
1701 | dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p" |
1702 | " msg_number=%ld partid=%d channel=%d\n", | |
1703 | (void *)notify, get, ch->partid, ch->number); | |
a47d5dac DN |
1704 | } |
1705 | } | |
1706 | } | |
1707 | ||
1708 | static void | |
1709 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1710 | { | |
1711 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1712 | } | |
1713 | ||
1714 | /* | |
1715 | * Clear some of the msg flags in the local message queue. | |
1716 | */ | |
1717 | static inline void | |
1718 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1719 | { | |
1720 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1721 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1722 | s64 get; |
1723 | ||
1724 | get = ch_sn2->w_remote_GP.get; | |
1725 | do { | |
bd3e64c1 DN |
1726 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
1727 | (get % ch->local_nentries) * | |
1728 | ch->entry_size); | |
a47d5dac DN |
1729 | msg->flags = 0; |
1730 | } while (++get < ch_sn2->remote_GP.get); | |
1731 | } | |
1732 | ||
1733 | /* | |
1734 | * Clear some of the msg flags in the remote message queue. | |
1735 | */ | |
1736 | static inline void | |
1737 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1738 | { | |
1739 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1740 | struct xpc_msg_sn2 *msg; |
a47d5dac DN |
1741 | s64 put; |
1742 | ||
1743 | put = ch_sn2->w_remote_GP.put; | |
1744 | do { | |
bd3e64c1 DN |
1745 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + |
1746 | (put % ch->remote_nentries) * | |
1747 | ch->entry_size); | |
a47d5dac DN |
1748 | msg->flags = 0; |
1749 | } while (++put < ch_sn2->remote_GP.put); | |
1750 | } | |
1751 | ||
bd3e64c1 DN |
1752 | static int |
1753 | xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch) | |
1754 | { | |
1755 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
1756 | } | |
1757 | ||
a47d5dac | 1758 | static void |
7fb5e59d | 1759 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1760 | { |
1761 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1762 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
bd3e64c1 | 1763 | int npayloads_sent; |
a47d5dac DN |
1764 | |
1765 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1766 | ||
1767 | /* See what, if anything, has changed for each connected channel */ | |
1768 | ||
1769 | xpc_msgqueue_ref(ch); | |
1770 | ||
1771 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1772 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1773 | /* nothing changed since GPs were last pulled */ | |
1774 | xpc_msgqueue_deref(ch); | |
1775 | return; | |
1776 | } | |
1777 | ||
1778 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1779 | xpc_msgqueue_deref(ch); | |
1780 | return; | |
1781 | } | |
1782 | ||
1783 | /* | |
1784 | * First check to see if messages recently sent by us have been | |
1785 | * received by the other side. (The remote GET value will have | |
1786 | * changed since we last looked at it.) | |
1787 | */ | |
1788 | ||
1789 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1790 | ||
1791 | /* | |
1792 | * We need to notify any senders that want to be notified | |
1793 | * that their sent messages have been received by their | |
1794 | * intended recipients. We need to do this before updating | |
1795 | * w_remote_GP.get so that we don't allocate the same message | |
1796 | * queue entries prematurely (see xpc_allocate_msg()). | |
1797 | */ | |
1798 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1799 | /* | |
1800 | * Notify senders that messages sent have been | |
1801 | * received and delivered by the other side. | |
1802 | */ | |
1803 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1804 | ch_sn2->remote_GP.get); | |
1805 | } | |
1806 | ||
1807 | /* | |
1808 | * Clear msg->flags in previously sent messages, so that | |
1809 | * they're ready for xpc_allocate_msg(). | |
1810 | */ | |
1811 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1812 | ||
1813 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1814 | ||
1815 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, " | |
1816 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, | |
1817 | ch->number); | |
1818 | ||
1819 | /* | |
1820 | * If anyone was waiting for message queue entries to become | |
1821 | * available, wake them up. | |
1822 | */ | |
1823 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1824 | wake_up(&ch->msg_allocate_wq); | |
1825 | } | |
1826 | ||
1827 | /* | |
1828 | * Now check for newly sent messages by the other side. (The remote | |
1829 | * PUT value will have changed since we last looked at it.) | |
1830 | */ | |
1831 | ||
1832 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1833 | /* | |
1834 | * Clear msg->flags in previously received messages, so that | |
bd3e64c1 | 1835 | * they're ready for xpc_get_deliverable_payload_sn2(). |
a47d5dac DN |
1836 | */ |
1837 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1838 | ||
1839 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; | |
1840 | ||
1841 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, " | |
1842 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, | |
1843 | ch->number); | |
1844 | ||
bd3e64c1 DN |
1845 | npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch); |
1846 | if (npayloads_sent > 0) { | |
a47d5dac DN |
1847 | dev_dbg(xpc_chan, "msgs waiting to be copied and " |
1848 | "delivered=%d, partid=%d, channel=%d\n", | |
bd3e64c1 | 1849 | npayloads_sent, ch->partid, ch->number); |
a47d5dac DN |
1850 | |
1851 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
bd3e64c1 | 1852 | xpc_activate_kthreads(ch, npayloads_sent); |
a47d5dac DN |
1853 | } |
1854 | } | |
1855 | ||
1856 | xpc_msgqueue_deref(ch); | |
1857 | } | |
1858 | ||
bd3e64c1 | 1859 | static struct xpc_msg_sn2 * |
e17d416b DN |
1860 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) |
1861 | { | |
1862 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1863 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
a812dcc3 | 1864 | unsigned long remote_msg_pa; |
bd3e64c1 | 1865 | struct xpc_msg_sn2 *msg; |
a812dcc3 DN |
1866 | u32 msg_index; |
1867 | u32 nmsgs; | |
e17d416b DN |
1868 | u64 msg_offset; |
1869 | enum xp_retval ret; | |
1870 | ||
a47d5dac | 1871 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1872 | /* we were interrupted by a signal */ |
1873 | return NULL; | |
1874 | } | |
1875 | ||
a47d5dac | 1876 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1877 | |
1878 | /* pull as many messages as are ready and able to be pulled */ | |
1879 | ||
a47d5dac | 1880 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1881 | |
a47d5dac DN |
1882 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1883 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1884 | if (msg_index + nmsgs > ch->remote_nentries) { |
1885 | /* ignore the ones that wrap the msg queue for now */ | |
1886 | nmsgs = ch->remote_nentries - msg_index; | |
1887 | } | |
1888 | ||
bd3e64c1 DN |
1889 | msg_offset = msg_index * ch->entry_size; |
1890 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + | |
5b8669df DN |
1891 | msg_offset); |
1892 | remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset; | |
e17d416b | 1893 | |
a812dcc3 | 1894 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, |
bd3e64c1 | 1895 | nmsgs * ch->entry_size); |
e17d416b DN |
1896 | if (ret != xpSuccess) { |
1897 | ||
1898 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
1899 | " msg %ld from partition %d, channel=%d, " | |
a47d5dac | 1900 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
1901 | ch->partid, ch->number, ret); |
1902 | ||
1903 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1904 | ||
a47d5dac | 1905 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1906 | return NULL; |
1907 | } | |
1908 | ||
a47d5dac | 1909 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
1910 | } |
1911 | ||
a47d5dac | 1912 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1913 | |
1914 | /* return the message we were looking for */ | |
bd3e64c1 DN |
1915 | msg_offset = (get % ch->remote_nentries) * ch->entry_size; |
1916 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset); | |
e17d416b DN |
1917 | |
1918 | return msg; | |
1919 | } | |
1920 | ||
1921 | /* | |
bd3e64c1 | 1922 | * Get the next deliverable message's payload. |
e17d416b | 1923 | */ |
bd3e64c1 DN |
1924 | static void * |
1925 | xpc_get_deliverable_payload_sn2(struct xpc_channel *ch) | |
e17d416b | 1926 | { |
a47d5dac | 1927 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
1928 | struct xpc_msg_sn2 *msg; |
1929 | void *payload = NULL; | |
e17d416b DN |
1930 | s64 get; |
1931 | ||
1932 | do { | |
1933 | if (ch->flags & XPC_C_DISCONNECTING) | |
1934 | break; | |
1935 | ||
a47d5dac | 1936 | get = ch_sn2->w_local_GP.get; |
e17d416b | 1937 | rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 1938 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
1939 | break; |
1940 | ||
1941 | /* There are messages waiting to be pulled and delivered. | |
1942 | * We need to try to secure one for ourselves. We'll do this | |
1943 | * by trying to increment w_local_GP.get and hope that no one | |
1944 | * else beats us to it. If they do, we'll we'll simply have | |
1945 | * to try again for the next one. | |
1946 | */ | |
1947 | ||
a47d5dac | 1948 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
1949 | /* we got the entry referenced by get */ |
1950 | ||
1951 | dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, " | |
1952 | "partid=%d, channel=%d\n", get + 1, | |
1953 | ch->partid, ch->number); | |
1954 | ||
1955 | /* pull the message from the remote partition */ | |
1956 | ||
1957 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
1958 | ||
1959 | DBUG_ON(msg != NULL && msg->number != get); | |
bd3e64c1 DN |
1960 | DBUG_ON(msg != NULL && (msg->flags & XPC_M_SN2_DONE)); |
1961 | DBUG_ON(msg != NULL && !(msg->flags & XPC_M_SN2_READY)); | |
e17d416b | 1962 | |
bd3e64c1 | 1963 | payload = &msg->payload; |
e17d416b DN |
1964 | break; |
1965 | } | |
1966 | ||
1967 | } while (1); | |
1968 | ||
bd3e64c1 | 1969 | return payload; |
e17d416b DN |
1970 | } |
1971 | ||
33ba3c77 DN |
1972 | /* |
1973 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
1974 | * the local message queue's Put value and then send a chctl msgrequest to the |
1975 | * recipient partition. | |
33ba3c77 DN |
1976 | */ |
1977 | static void | |
1978 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
1979 | { | |
a47d5dac | 1980 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 1981 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 1982 | s64 put = initial_put + 1; |
7fb5e59d | 1983 | int send_msgrequest = 0; |
33ba3c77 DN |
1984 | |
1985 | while (1) { | |
1986 | ||
1987 | while (1) { | |
a47d5dac | 1988 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
1989 | break; |
1990 | ||
bd3e64c1 DN |
1991 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
1992 | local_msgqueue + (put % | |
1993 | ch->local_nentries) * | |
1994 | ch->entry_size); | |
33ba3c77 | 1995 | |
bd3e64c1 | 1996 | if (!(msg->flags & XPC_M_SN2_READY)) |
33ba3c77 DN |
1997 | break; |
1998 | ||
1999 | put++; | |
2000 | } | |
2001 | ||
2002 | if (put == initial_put) { | |
2003 | /* nothing's changed */ | |
2004 | break; | |
2005 | } | |
2006 | ||
a47d5dac | 2007 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2008 | initial_put) { |
2009 | /* someone else beat us to it */ | |
a47d5dac | 2010 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2011 | break; |
2012 | } | |
2013 | ||
2014 | /* we just set the new value of local_GP->put */ | |
2015 | ||
2016 | dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, " | |
2017 | "channel=%d\n", put, ch->partid, ch->number); | |
2018 | ||
7fb5e59d | 2019 | send_msgrequest = 1; |
33ba3c77 DN |
2020 | |
2021 | /* | |
2022 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2023 | * local_GP->put is not XPC_M_SN2_READY or that local_GP->put |
33ba3c77 DN |
2024 | * equals w_local_GP.put, so we'll go have a look. |
2025 | */ | |
2026 | initial_put = put; | |
2027 | } | |
2028 | ||
7fb5e59d DN |
2029 | if (send_msgrequest) |
2030 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2031 | } |
2032 | ||
2033 | /* | |
2034 | * Allocate an entry for a message from the message queue associated with the | |
2035 | * specified channel. | |
2036 | */ | |
2037 | static enum xp_retval | |
2038 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
bd3e64c1 | 2039 | struct xpc_msg_sn2 **address_of_msg) |
33ba3c77 | 2040 | { |
a47d5dac | 2041 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2042 | struct xpc_msg_sn2 *msg; |
33ba3c77 DN |
2043 | enum xp_retval ret; |
2044 | s64 put; | |
2045 | ||
33ba3c77 DN |
2046 | /* |
2047 | * Get the next available message entry from the local message queue. | |
2048 | * If none are available, we'll make sure that we grab the latest | |
2049 | * GP values. | |
2050 | */ | |
2051 | ret = xpTimeout; | |
2052 | ||
2053 | while (1) { | |
2054 | ||
a47d5dac | 2055 | put = ch_sn2->w_local_GP.put; |
33ba3c77 | 2056 | rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2057 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2058 | |
2059 | /* There are available message entries. We need to try | |
2060 | * to secure one for ourselves. We'll do this by trying | |
2061 | * to increment w_local_GP.put as long as someone else | |
2062 | * doesn't beat us to it. If they do, we'll have to | |
2063 | * try again. | |
2064 | */ | |
a47d5dac DN |
2065 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2066 | put) { | |
33ba3c77 DN |
2067 | /* we got the entry referenced by put */ |
2068 | break; | |
2069 | } | |
2070 | continue; /* try again */ | |
2071 | } | |
2072 | ||
2073 | /* | |
2074 | * There aren't any available msg entries at this time. | |
2075 | * | |
2076 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2077 | * we set a timeout in case the other side is not sending |
2078 | * completion interrupts. This lets us fake a notify IRQ | |
2079 | * that will cause the notify IRQ handler to fetch the latest | |
2080 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2081 | */ |
2082 | if (ret == xpTimeout) | |
7fb5e59d | 2083 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2084 | |
97bf1aa1 | 2085 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2086 | return xpNoWait; |
33ba3c77 DN |
2087 | |
2088 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2089 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2090 | return ret; |
33ba3c77 DN |
2091 | } |
2092 | ||
2093 | /* get the message's address and initialize it */ | |
bd3e64c1 DN |
2094 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue + |
2095 | (put % ch->local_nentries) * | |
2096 | ch->entry_size); | |
33ba3c77 DN |
2097 | |
2098 | DBUG_ON(msg->flags != 0); | |
2099 | msg->number = put; | |
2100 | ||
2101 | dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, " | |
2102 | "msg_number=%ld, partid=%d, channel=%d\n", put + 1, | |
2103 | (void *)msg, msg->number, ch->partid, ch->number); | |
2104 | ||
2105 | *address_of_msg = msg; | |
33ba3c77 DN |
2106 | return xpSuccess; |
2107 | } | |
2108 | ||
2109 | /* | |
2110 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2111 | * local message queue's Put value and sends a chctl msgrequest to the |
2112 | * partition the message is being sent to. | |
33ba3c77 DN |
2113 | */ |
2114 | static enum xp_retval | |
bd3e64c1 DN |
2115 | xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2116 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2117 | void *key) | |
33ba3c77 DN |
2118 | { |
2119 | enum xp_retval ret = xpSuccess; | |
5b8669df | 2120 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 DN |
2121 | struct xpc_msg_sn2 *msg = msg; |
2122 | struct xpc_notify_sn2 *notify = notify; | |
97bf1aa1 DN |
2123 | s64 msg_number; |
2124 | s64 put; | |
33ba3c77 DN |
2125 | |
2126 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 | 2127 | |
bd3e64c1 | 2128 | if (XPC_MSG_SIZE(payload_size) > ch->entry_size) |
97bf1aa1 DN |
2129 | return xpPayloadTooBig; |
2130 | ||
2131 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2132 | |
2133 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2134 | ret = ch->reason; |
2135 | goto out_1; | |
2136 | } | |
2137 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2138 | ret = xpNotConnected; | |
2139 | goto out_1; | |
33ba3c77 DN |
2140 | } |
2141 | ||
97bf1aa1 DN |
2142 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2143 | if (ret != xpSuccess) | |
2144 | goto out_1; | |
2145 | ||
2146 | msg_number = msg->number; | |
2147 | ||
33ba3c77 DN |
2148 | if (notify_type != 0) { |
2149 | /* | |
2150 | * Tell the remote side to send an ACK interrupt when the | |
2151 | * message has been delivered. | |
2152 | */ | |
bd3e64c1 | 2153 | msg->flags |= XPC_M_SN2_INTERRUPT; |
33ba3c77 DN |
2154 | |
2155 | atomic_inc(&ch->n_to_notify); | |
2156 | ||
5b8669df | 2157 | notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries]; |
33ba3c77 DN |
2158 | notify->func = func; |
2159 | notify->key = key; | |
2160 | notify->type = notify_type; | |
2161 | ||
ea57f80c | 2162 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2163 | |
2164 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2165 | /* | |
2166 | * An error occurred between our last error check and | |
2167 | * this one. We will try to clear the type field from | |
2168 | * the notify entry. If we succeed then | |
2169 | * xpc_disconnect_channel() didn't already process | |
2170 | * the notify entry. | |
2171 | */ | |
2172 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2173 | notify_type) { | |
2174 | atomic_dec(&ch->n_to_notify); | |
2175 | ret = ch->reason; | |
2176 | } | |
97bf1aa1 | 2177 | goto out_1; |
33ba3c77 DN |
2178 | } |
2179 | } | |
2180 | ||
97bf1aa1 DN |
2181 | memcpy(&msg->payload, payload, payload_size); |
2182 | ||
bd3e64c1 | 2183 | msg->flags |= XPC_M_SN2_READY; |
33ba3c77 DN |
2184 | |
2185 | /* | |
2186 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2187 | * load of local_GP->put. |
33ba3c77 DN |
2188 | */ |
2189 | mb(); | |
2190 | ||
2191 | /* see if the message is next in line to be sent, if so send it */ | |
2192 | ||
5b8669df | 2193 | put = ch_sn2->local_GP->put; |
33ba3c77 DN |
2194 | if (put == msg_number) |
2195 | xpc_send_msgs_sn2(ch, put); | |
2196 | ||
97bf1aa1 | 2197 | out_1: |
33ba3c77 DN |
2198 | xpc_msgqueue_deref(ch); |
2199 | return ret; | |
2200 | } | |
2201 | ||
2202 | /* | |
2203 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2204 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2205 | * send a chctl msgrequest to the message sender's partition. |
bd3e64c1 DN |
2206 | * |
2207 | * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition | |
2208 | * that sent the message. | |
33ba3c77 DN |
2209 | */ |
2210 | static void | |
2211 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2212 | { | |
a47d5dac | 2213 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
bd3e64c1 | 2214 | struct xpc_msg_sn2 *msg; |
33ba3c77 | 2215 | s64 get = initial_get + 1; |
7fb5e59d | 2216 | int send_msgrequest = 0; |
33ba3c77 DN |
2217 | |
2218 | while (1) { | |
2219 | ||
2220 | while (1) { | |
a47d5dac | 2221 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2222 | break; |
2223 | ||
bd3e64c1 DN |
2224 | msg = (struct xpc_msg_sn2 *)((u64)ch_sn2-> |
2225 | remote_msgqueue + (get % | |
2226 | ch->remote_nentries) * | |
2227 | ch->entry_size); | |
33ba3c77 | 2228 | |
bd3e64c1 | 2229 | if (!(msg->flags & XPC_M_SN2_DONE)) |
33ba3c77 DN |
2230 | break; |
2231 | ||
2232 | msg_flags |= msg->flags; | |
2233 | get++; | |
2234 | } | |
2235 | ||
2236 | if (get == initial_get) { | |
2237 | /* nothing's changed */ | |
2238 | break; | |
2239 | } | |
2240 | ||
a47d5dac | 2241 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2242 | initial_get) { |
2243 | /* someone else beat us to it */ | |
a47d5dac | 2244 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2245 | break; |
2246 | } | |
2247 | ||
2248 | /* we just set the new value of local_GP->get */ | |
2249 | ||
2250 | dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, " | |
2251 | "channel=%d\n", get, ch->partid, ch->number); | |
2252 | ||
bd3e64c1 | 2253 | send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT); |
33ba3c77 DN |
2254 | |
2255 | /* | |
2256 | * We need to ensure that the message referenced by | |
bd3e64c1 | 2257 | * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get |
33ba3c77 DN |
2258 | * equals w_local_GP.get, so we'll go have a look. |
2259 | */ | |
2260 | initial_get = get; | |
2261 | } | |
2262 | ||
7fb5e59d DN |
2263 | if (send_msgrequest) |
2264 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2265 | } |
2266 | ||
2267 | static void | |
bd3e64c1 | 2268 | xpc_received_payload_sn2(struct xpc_channel *ch, void *payload) |
33ba3c77 | 2269 | { |
bd3e64c1 DN |
2270 | struct xpc_msg_sn2 *msg; |
2271 | s64 msg_number; | |
33ba3c77 | 2272 | s64 get; |
bd3e64c1 DN |
2273 | |
2274 | msg = container_of(payload, struct xpc_msg_sn2, payload); | |
2275 | msg_number = msg->number; | |
33ba3c77 DN |
2276 | |
2277 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n", | |
2278 | (void *)msg, msg_number, ch->partid, ch->number); | |
2279 | ||
bd3e64c1 | 2280 | DBUG_ON((((u64)msg - (u64)ch->remote_msgqueue) / ch->entry_size) != |
33ba3c77 | 2281 | msg_number % ch->remote_nentries); |
bd3e64c1 | 2282 | DBUG_ON(msg->flags & XPC_M_SN2_DONE); |
33ba3c77 | 2283 | |
bd3e64c1 | 2284 | msg->flags |= XPC_M_SN2_DONE; |
33ba3c77 DN |
2285 | |
2286 | /* | |
2287 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2288 | * load of local_GP->get. |
33ba3c77 DN |
2289 | */ |
2290 | mb(); | |
2291 | ||
2292 | /* | |
2293 | * See if this message is next in line to be acknowledged as having | |
2294 | * been delivered. | |
2295 | */ | |
a47d5dac | 2296 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2297 | if (get == msg_number) |
2298 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2299 | } | |
2300 | ||
6e41017a | 2301 | int |
94bd2708 DN |
2302 | xpc_init_sn2(void) |
2303 | { | |
6e41017a | 2304 | int ret; |
ee6665e3 | 2305 | size_t buf_size; |
6e41017a | 2306 | |
5b8669df | 2307 | xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2; |
261f3b49 | 2308 | xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2; |
5b8669df | 2309 | xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2; |
33ba3c77 DN |
2310 | xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; |
2311 | xpc_offline_heartbeat = xpc_offline_heartbeat_sn2; | |
2312 | xpc_online_heartbeat = xpc_online_heartbeat_sn2; | |
2313 | xpc_heartbeat_init = xpc_heartbeat_init_sn2; | |
2314 | xpc_heartbeat_exit = xpc_heartbeat_exit_sn2; | |
61deb86e | 2315 | xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_sn2; |
33ba3c77 | 2316 | |
a47d5dac DN |
2317 | xpc_request_partition_activation = xpc_request_partition_activation_sn2; |
2318 | xpc_request_partition_reactivation = | |
2319 | xpc_request_partition_reactivation_sn2; | |
2320 | xpc_request_partition_deactivation = | |
2321 | xpc_request_partition_deactivation_sn2; | |
2322 | xpc_cancel_partition_deactivation_request = | |
2323 | xpc_cancel_partition_deactivation_request_sn2; | |
2324 | ||
6e41017a | 2325 | xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2; |
5b8669df DN |
2326 | xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_sn2; |
2327 | xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_sn2; | |
e17d416b | 2328 | xpc_make_first_contact = xpc_make_first_contact_sn2; |
5b8669df | 2329 | |
7fb5e59d | 2330 | xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2; |
5b8669df DN |
2331 | xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2; |
2332 | xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2; | |
2333 | xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2; | |
2334 | xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2; | |
2335 | ||
2336 | xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2; | |
2337 | ||
2338 | xpc_setup_msg_structures = xpc_setup_msg_structures_sn2; | |
2339 | xpc_teardown_msg_structures = xpc_teardown_msg_structures_sn2; | |
2340 | ||
a47d5dac | 2341 | xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2; |
7fb5e59d | 2342 | xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2; |
bd3e64c1 DN |
2343 | xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2; |
2344 | xpc_get_deliverable_payload = xpc_get_deliverable_payload_sn2; | |
33ba3c77 | 2345 | |
a47d5dac | 2346 | xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2; |
a47d5dac DN |
2347 | xpc_indicate_partition_disengaged = |
2348 | xpc_indicate_partition_disengaged_sn2; | |
5b8669df DN |
2349 | xpc_partition_engaged = xpc_partition_engaged_sn2; |
2350 | xpc_any_partition_engaged = xpc_any_partition_engaged_sn2; | |
a47d5dac DN |
2351 | xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2; |
2352 | ||
bd3e64c1 DN |
2353 | xpc_send_payload = xpc_send_payload_sn2; |
2354 | xpc_received_payload = xpc_received_payload_sn2; | |
2355 | ||
2356 | if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) { | |
2357 | dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is " | |
2358 | "larger than %d\n", XPC_MSG_HDR_MAX_SIZE); | |
2359 | return -E2BIG; | |
2360 | } | |
6e41017a | 2361 | |
ee6665e3 DN |
2362 | buf_size = max(XPC_RP_VARS_SIZE, |
2363 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2364 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2365 | GFP_KERNEL, | |
2366 | &xpc_remote_copy_buffer_base_sn2); | |
2367 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2368 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2369 | return -ENOMEM; | |
2370 | } | |
2371 | ||
c39838ce | 2372 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2373 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2374 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2375 | |
2376 | /* | |
2377 | * This is safe to do before the xpc_hb_checker thread has started | |
2378 | * because the handler releases a wait queue. If an interrupt is | |
2379 | * received before the thread is waiting, it will not go to sleep, | |
2380 | * but rather immediately process the interrupt. | |
2381 | */ | |
2382 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2383 | "xpc hb", NULL); | |
2384 | if (ret != 0) { | |
2385 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2386 | "errno=%d\n", -ret); | |
2387 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2388 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2389 | } |
2390 | return ret; | |
94bd2708 DN |
2391 | } |
2392 | ||
2393 | void | |
2394 | xpc_exit_sn2(void) | |
2395 | { | |
6e41017a DN |
2396 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2397 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2398 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2399 | } |