]>
Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | |
7 | */ | |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
e17d416b | 16 | #include <linux/delay.h> |
94bd2708 | 17 | #include <asm/uncached.h> |
261f3b49 | 18 | #include <asm/sn/mspec.h> |
94bd2708 DN |
19 | #include <asm/sn/sn_sal.h> |
20 | #include "xpc.h" | |
21 | ||
ee6665e3 DN |
22 | /* |
23 | * Define the number of u64s required to represent all the C-brick nasids | |
24 | * as a bitmap. The cross-partition kernel modules deal only with | |
25 | * C-brick nasids, thus the need for bitmaps which don't account for | |
26 | * odd-numbered (non C-brick) nasids. | |
27 | */ | |
28 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
29 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
30 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
31 | ||
32 | /* | |
33 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
34 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
35 | * for cached references and an alternate TLB handler to never provide a | |
36 | * cacheable mapping for the entire region. This will prevent speculative | |
37 | * reading of cached copies of our lines from being issued which will cause | |
38 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
39 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
40 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
41 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
42 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
43 | * local XPC and 1 amo variable to request partition deactivation. | |
44 | */ | |
45 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
46 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
47 | XP_MAX_NPARTITIONS_SN2) | |
48 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
49 | XP_NASID_MASK_WORDS_SN2) | |
50 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
51 | ||
52 | /* | |
53 | * Buffer used to store a local copy of portions of a remote partition's | |
54 | * reserved page (either its header and part_nasids mask, or its vars). | |
55 | */ | |
56 | static char *xpc_remote_copy_buffer_sn2; | |
57 | static void *xpc_remote_copy_buffer_base_sn2; | |
58 | ||
8e85c23e DN |
59 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
60 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 61 | |
6e41017a | 62 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
63 | static u64 xpc_sh1_IPI_access_sn2; |
64 | static u64 xpc_sh2_IPI_access0_sn2; | |
65 | static u64 xpc_sh2_IPI_access1_sn2; | |
66 | static u64 xpc_sh2_IPI_access2_sn2; | |
67 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
68 | |
69 | /* | |
70 | * Change protections to allow IPI operations. | |
71 | */ | |
72 | static void | |
73 | xpc_allow_IPI_ops_sn2(void) | |
74 | { | |
75 | int node; | |
76 | int nasid; | |
77 | ||
ea57f80c | 78 | /* !!! The following should get moved into SAL. */ |
6e41017a | 79 | if (is_shub2()) { |
8e85c23e | 80 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 81 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 82 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 83 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 84 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 85 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 86 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
87 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
88 | ||
89 | for_each_online_node(node) { | |
90 | nasid = cnodeid_to_nasid(node); | |
91 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
92 | -1UL); | |
93 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
94 | -1UL); | |
95 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
96 | -1UL); | |
97 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
98 | -1UL); | |
99 | } | |
100 | } else { | |
8e85c23e | 101 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
102 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
103 | ||
104 | for_each_online_node(node) { | |
105 | nasid = cnodeid_to_nasid(node); | |
106 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
107 | -1UL); | |
108 | } | |
109 | } | |
110 | } | |
111 | ||
112 | /* | |
113 | * Restrict protections to disallow IPI operations. | |
114 | */ | |
115 | static void | |
116 | xpc_disallow_IPI_ops_sn2(void) | |
117 | { | |
118 | int node; | |
119 | int nasid; | |
120 | ||
ea57f80c | 121 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
122 | if (is_shub2()) { |
123 | for_each_online_node(node) { | |
124 | nasid = cnodeid_to_nasid(node); | |
125 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 126 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 127 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 128 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 129 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 130 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 131 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 132 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
133 | } |
134 | } else { | |
135 | for_each_online_node(node) { | |
136 | nasid = cnodeid_to_nasid(node); | |
137 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 138 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
139 | } |
140 | } | |
141 | } | |
142 | ||
33ba3c77 | 143 | /* |
7fb5e59d DN |
144 | * The following set of functions are used for the sending and receiving of |
145 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
146 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
147 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
148 | */ |
149 | ||
150 | static u64 | |
c39838ce | 151 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
152 | { |
153 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
154 | } | |
155 | ||
156 | static enum xp_retval | |
c39838ce DN |
157 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
158 | int vector) | |
33ba3c77 DN |
159 | { |
160 | int ret = 0; | |
161 | unsigned long irq_flags; | |
162 | ||
163 | local_irq_save(irq_flags); | |
164 | ||
165 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
166 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
167 | ||
168 | /* | |
169 | * We must always use the nofault function regardless of whether we | |
170 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
171 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 172 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
173 | */ |
174 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
175 | xp_nofault_PIOR_target)); | |
176 | ||
177 | local_irq_restore(irq_flags); | |
178 | ||
261f3b49 | 179 | return (ret == 0) ? xpSuccess : xpPioReadError; |
33ba3c77 DN |
180 | } |
181 | ||
c39838ce | 182 | static struct amo * |
7fb5e59d | 183 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 184 | { |
8e85c23e | 185 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 186 | |
c39838ce | 187 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
188 | return amo; |
189 | } | |
190 | ||
191 | /* | |
7fb5e59d | 192 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
193 | */ |
194 | ||
6e41017a DN |
195 | /* |
196 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
197 | */ | |
198 | static irqreturn_t | |
199 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
200 | { | |
201 | atomic_inc(&xpc_activate_IRQ_rcvd); | |
202 | wake_up_interruptible(&xpc_activate_IRQ_wq); | |
203 | return IRQ_HANDLED; | |
204 | } | |
205 | ||
33ba3c77 | 206 | /* |
c39838ce | 207 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
208 | */ |
209 | static void | |
a812dcc3 DN |
210 | xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid, |
211 | int to_nasid, int to_phys_cpuid) | |
33ba3c77 | 212 | { |
c39838ce | 213 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 214 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 215 | sizeof(struct amo))); |
33ba3c77 | 216 | |
04de7418 DN |
217 | (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)], |
218 | BIT_MASK(from_nasid / 2), to_nasid, | |
33ba3c77 DN |
219 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
220 | } | |
221 | ||
222 | static void | |
7fb5e59d | 223 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 | 224 | { |
8e85c23e | 225 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 226 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 227 | sizeof(struct amo))); |
33ba3c77 DN |
228 | |
229 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
04de7418 DN |
230 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable), |
231 | FETCHOP_OR, BIT_MASK(from_nasid / 2)); | |
232 | ||
6e41017a DN |
233 | atomic_inc(&xpc_activate_IRQ_rcvd); |
234 | wake_up_interruptible(&xpc_activate_IRQ_wq); | |
33ba3c77 DN |
235 | } |
236 | ||
a47d5dac | 237 | /* |
7fb5e59d | 238 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 239 | */ |
33ba3c77 | 240 | |
a47d5dac | 241 | /* |
7fb5e59d | 242 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 243 | */ |
33ba3c77 | 244 | static void |
7fb5e59d | 245 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 246 | { |
7fb5e59d | 247 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 248 | unsigned long irq_flags; |
33ba3c77 | 249 | |
7fb5e59d DN |
250 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
251 | local_chctl_amo_va); | |
252 | if (chctl.all_flags == 0) | |
a47d5dac DN |
253 | return; |
254 | ||
7fb5e59d DN |
255 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
256 | part->chctl.all_flags |= chctl.all_flags; | |
257 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 258 | |
7fb5e59d DN |
259 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
260 | "0x%lx\n", XPC_PARTID(part), chctl.all_flags); | |
a47d5dac DN |
261 | |
262 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
263 | } |
264 | ||
a47d5dac DN |
265 | /* |
266 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
267 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 268 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 269 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
270 | * associated kthread to handle it. |
271 | * | |
7fb5e59d | 272 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
273 | * running on other partitions. |
274 | * | |
275 | * Noteworthy Arguments: | |
276 | * | |
277 | * irq - Interrupt ReQuest number. NOT USED. | |
278 | * | |
7fb5e59d | 279 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
280 | */ |
281 | static irqreturn_t | |
282 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 283 | { |
a47d5dac DN |
284 | short partid = (short)(u64)dev_id; |
285 | struct xpc_partition *part = &xpc_partitions[partid]; | |
286 | ||
261f3b49 | 287 | DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2); |
a47d5dac DN |
288 | |
289 | if (xpc_part_ref(part)) { | |
7fb5e59d | 290 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
291 | |
292 | xpc_part_deref(part); | |
293 | } | |
294 | return IRQ_HANDLED; | |
33ba3c77 DN |
295 | } |
296 | ||
297 | /* | |
7fb5e59d DN |
298 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
299 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 300 | * was received. |
33ba3c77 | 301 | */ |
a47d5dac | 302 | static void |
7fb5e59d | 303 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
304 | { |
305 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
306 | ||
307 | if (xpc_part_ref(part)) { | |
7fb5e59d | 308 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
309 | |
310 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 311 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
312 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
313 | xpc_part_deref(part); | |
314 | } | |
315 | } | |
33ba3c77 DN |
316 | |
317 | /* | |
7fb5e59d | 318 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
319 | * specified channel. |
320 | */ | |
321 | static void | |
7fb5e59d DN |
322 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
323 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
324 | { |
325 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 326 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 327 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
328 | enum xp_retval ret; |
329 | ||
330 | if (likely(part->act_state != XPC_P_DEACTIVATING)) { | |
7fb5e59d DN |
331 | chctl.flags[ch->number] = chctl_flag; |
332 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
333 | chctl.all_flags, | |
334 | part_sn2->notify_IRQ_nasid, | |
335 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
336 | SGI_XPC_NOTIFY); |
337 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 338 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
339 | if (unlikely(ret != xpSuccess)) { |
340 | if (irq_flags != NULL) | |
341 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
342 | XPC_DEACTIVATE_PARTITION(part, ret); | |
343 | if (irq_flags != NULL) | |
344 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
345 | } | |
346 | } | |
347 | } | |
348 | ||
7fb5e59d DN |
349 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
350 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
351 | |
352 | /* | |
353 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
354 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
355 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
356 | */ |
357 | static void | |
7fb5e59d DN |
358 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
359 | char *chctl_flag_string) | |
33ba3c77 DN |
360 | { |
361 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 362 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 363 | |
7fb5e59d DN |
364 | chctl.flags[ch->number] = chctl_flag; |
365 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
366 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 367 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 368 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
369 | } |
370 | ||
7fb5e59d DN |
371 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
372 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
373 | |
374 | static void | |
7fb5e59d DN |
375 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
376 | unsigned long *irq_flags) | |
33ba3c77 DN |
377 | { |
378 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
379 | ||
380 | args->reason = ch->reason; | |
7fb5e59d | 381 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
382 | } |
383 | ||
384 | static void | |
7fb5e59d | 385 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 386 | { |
7fb5e59d | 387 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
388 | } |
389 | ||
390 | static void | |
7fb5e59d | 391 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 DN |
392 | { |
393 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
394 | ||
395 | args->msg_size = ch->msg_size; | |
396 | args->local_nentries = ch->local_nentries; | |
7fb5e59d | 397 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
398 | } |
399 | ||
400 | static void | |
7fb5e59d | 401 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 DN |
402 | { |
403 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
404 | ||
405 | args->remote_nentries = ch->remote_nentries; | |
406 | args->local_nentries = ch->local_nentries; | |
a812dcc3 | 407 | args->local_msgqueue_pa = xp_pa(ch->local_msgqueue); |
7fb5e59d | 408 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
409 | } |
410 | ||
411 | static void | |
7fb5e59d | 412 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 413 | { |
7fb5e59d | 414 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
415 | } |
416 | ||
417 | static void | |
7fb5e59d | 418 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 419 | { |
7fb5e59d | 420 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
421 | } |
422 | ||
423 | /* | |
424 | * This next set of functions are used to keep track of when a partition is | |
425 | * potentially engaged in accessing memory belonging to another partition. | |
426 | */ | |
427 | ||
428 | static void | |
a47d5dac | 429 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
430 | { |
431 | unsigned long irq_flags; | |
c39838ce | 432 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 433 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 434 | sizeof(struct amo))); |
33ba3c77 DN |
435 | |
436 | local_irq_save(irq_flags); | |
437 | ||
c39838ce | 438 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 439 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
440 | BIT(sn_partition_id)); |
441 | ||
33ba3c77 DN |
442 | /* |
443 | * We must always use the nofault function regardless of whether we | |
444 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
445 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 446 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
447 | */ |
448 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
449 | variable), | |
450 | xp_nofault_PIOR_target)); | |
451 | ||
452 | local_irq_restore(irq_flags); | |
453 | } | |
454 | ||
455 | static void | |
a47d5dac | 456 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 457 | { |
a47d5dac | 458 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 459 | unsigned long irq_flags; |
c39838ce | 460 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 461 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 462 | sizeof(struct amo))); |
33ba3c77 DN |
463 | |
464 | local_irq_save(irq_flags); | |
465 | ||
c39838ce | 466 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 | 467 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
468 | ~BIT(sn_partition_id)); |
469 | ||
33ba3c77 DN |
470 | /* |
471 | * We must always use the nofault function regardless of whether we | |
472 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
473 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 474 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
475 | */ |
476 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
477 | variable), | |
478 | xp_nofault_PIOR_target)); | |
479 | ||
480 | local_irq_restore(irq_flags); | |
33ba3c77 | 481 | |
33ba3c77 | 482 | /* |
a47d5dac | 483 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 484 | * bit in their engaged partitions amo. |
33ba3c77 | 485 | */ |
7fb5e59d | 486 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
487 | cnodeid_to_nasid(0), |
488 | part_sn2->activate_IRQ_nasid, | |
489 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
490 | } |
491 | ||
a47d5dac DN |
492 | static int |
493 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 494 | { |
8e85c23e DN |
495 | struct amo *amo = xpc_vars_sn2->amos_page + |
496 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 497 | |
c39838ce | 498 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 499 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 500 | BIT(partid)) != 0; |
33ba3c77 DN |
501 | } |
502 | ||
a47d5dac DN |
503 | static int |
504 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 505 | { |
8e85c23e DN |
506 | struct amo *amo = xpc_vars_sn2->amos_page + |
507 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 508 | |
c39838ce | 509 | /* our partition's amo variable */ |
a47d5dac | 510 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
511 | } |
512 | ||
513 | static void | |
a47d5dac | 514 | xpc_assume_partition_disengaged_sn2(short partid) |
33ba3c77 | 515 | { |
8e85c23e DN |
516 | struct amo *amo = xpc_vars_sn2->amos_page + |
517 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 518 | |
c39838ce | 519 | /* clear bit(s) based on partid mask in our partition's amo */ |
33ba3c77 | 520 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 | 521 | ~BIT(partid)); |
33ba3c77 DN |
522 | } |
523 | ||
6e41017a DN |
524 | /* original protection values for each node */ |
525 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
526 | ||
527 | /* | |
c39838ce | 528 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
529 | */ |
530 | static enum xp_retval | |
c39838ce | 531 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a DN |
532 | { |
533 | u64 nasid_array = 0; | |
534 | int ret; | |
535 | ||
536 | /* | |
537 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
538 | * collides with memory operations. On those systems we call | |
c39838ce | 539 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a DN |
540 | */ |
541 | if (!enable_shub_wars_1_1()) { | |
542 | ret = sn_change_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE, | |
543 | SN_MEMPROT_ACCESS_CLASS_1, | |
544 | &nasid_array); | |
545 | if (ret != 0) | |
546 | return xpSalError; | |
547 | } | |
548 | return xpSuccess; | |
549 | } | |
550 | ||
551 | /* | |
c39838ce | 552 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
553 | */ |
554 | static void | |
c39838ce | 555 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
556 | { |
557 | int node; | |
558 | int nasid; | |
559 | ||
560 | if (!enable_shub_wars_1_1()) | |
561 | return; | |
562 | ||
563 | for_each_online_node(node) { | |
564 | nasid = cnodeid_to_nasid(node); | |
565 | /* save current protection values */ | |
566 | xpc_prot_vec_sn2[node] = | |
567 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
568 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
569 | /* open up everything */ | |
570 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
571 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
572 | -1UL); | |
573 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
574 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
575 | -1UL); | |
576 | } | |
577 | } | |
578 | ||
261f3b49 | 579 | static enum xp_retval |
a812dcc3 | 580 | xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa, |
261f3b49 DN |
581 | size_t *len) |
582 | { | |
583 | s64 status; | |
584 | enum xp_retval ret; | |
585 | ||
a812dcc3 | 586 | status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len); |
261f3b49 DN |
587 | if (status == SALRET_OK) |
588 | ret = xpSuccess; | |
589 | else if (status == SALRET_MORE_PASSES) | |
590 | ret = xpNeedMoreInfo; | |
591 | else | |
592 | ret = xpSalError; | |
593 | ||
594 | return ret; | |
595 | } | |
596 | ||
597 | ||
94bd2708 DN |
598 | static enum xp_retval |
599 | xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp) | |
600 | { | |
c39838ce | 601 | struct amo *amos_page; |
94bd2708 DN |
602 | int i; |
603 | int ret; | |
604 | ||
8e85c23e | 605 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 606 | |
a812dcc3 | 607 | rp->sn.vars_pa = xp_pa(xpc_vars_sn2); |
94bd2708 | 608 | |
e17d416b | 609 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
610 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
611 | XPC_RP_VARS_SIZE); | |
e17d416b | 612 | |
94bd2708 | 613 | /* |
8e85c23e DN |
614 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
615 | * previously allocated. If not we'll need to allocate one and set | |
616 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 617 | * |
c39838ce | 618 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 619 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 620 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 621 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 622 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
623 | * memory protections are never restricted. |
624 | */ | |
8e85c23e | 625 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 626 | if (amos_page == NULL) { |
c39838ce | 627 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 628 | if (amos_page == NULL) { |
c39838ce | 629 | dev_err(xpc_part, "can't allocate page of amos\n"); |
94bd2708 DN |
630 | return xpNoMemory; |
631 | } | |
632 | ||
633 | /* | |
c39838ce DN |
634 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
635 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 636 | */ |
c39838ce | 637 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 638 | if (ret != xpSuccess) { |
c39838ce | 639 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
640 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
641 | TO_PHYS((u64)amos_page), 1); | |
642 | return ret; | |
94bd2708 DN |
643 | } |
644 | } | |
645 | ||
8e85c23e DN |
646 | /* clear xpc_vars_sn2 */ |
647 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 648 | |
8e85c23e DN |
649 | xpc_vars_sn2->version = XPC_V_VERSION; |
650 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
651 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
a812dcc3 | 652 | xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2); |
8e85c23e DN |
653 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); |
654 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 655 | |
8e85c23e DN |
656 | /* clear xpc_vars_part_sn2 */ |
657 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
261f3b49 | 658 | XP_MAX_NPARTITIONS_SN2); |
94bd2708 | 659 | |
c39838ce | 660 | /* initialize the activate IRQ related amo variables */ |
04de7418 | 661 | for (i = 0; i < xpc_nasid_mask_nlongs; i++) |
ee6665e3 | 662 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); |
94bd2708 | 663 | |
c39838ce | 664 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
665 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
666 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 DN |
667 | |
668 | return xpSuccess; | |
669 | } | |
670 | ||
671 | static void | |
672 | xpc_increment_heartbeat_sn2(void) | |
673 | { | |
8e85c23e | 674 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
675 | } |
676 | ||
677 | static void | |
678 | xpc_offline_heartbeat_sn2(void) | |
679 | { | |
680 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 681 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
682 | } |
683 | ||
684 | static void | |
685 | xpc_online_heartbeat_sn2(void) | |
686 | { | |
687 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 688 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
689 | } |
690 | ||
691 | static void | |
692 | xpc_heartbeat_init_sn2(void) | |
693 | { | |
8e85c23e | 694 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 695 | |
8e85c23e DN |
696 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
697 | xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0]; | |
33ba3c77 DN |
698 | xpc_online_heartbeat_sn2(); |
699 | } | |
700 | ||
701 | static void | |
702 | xpc_heartbeat_exit_sn2(void) | |
703 | { | |
704 | xpc_offline_heartbeat_sn2(); | |
705 | } | |
706 | ||
707 | /* | |
708 | * At periodic intervals, scan through all active partitions and ensure | |
709 | * their heartbeat is still active. If not, the partition is deactivated. | |
710 | */ | |
711 | static void | |
712 | xpc_check_remote_hb_sn2(void) | |
713 | { | |
714 | struct xpc_vars_sn2 *remote_vars; | |
715 | struct xpc_partition *part; | |
716 | short partid; | |
717 | enum xp_retval ret; | |
718 | ||
ee6665e3 | 719 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 | 720 | |
261f3b49 | 721 | for (partid = 0; partid < XP_MAX_NPARTITIONS_SN2; partid++) { |
33ba3c77 DN |
722 | |
723 | if (xpc_exiting) | |
724 | break; | |
725 | ||
726 | if (partid == sn_partition_id) | |
727 | continue; | |
728 | ||
729 | part = &xpc_partitions[partid]; | |
730 | ||
731 | if (part->act_state == XPC_P_INACTIVE || | |
732 | part->act_state == XPC_P_DEACTIVATING) { | |
733 | continue; | |
734 | } | |
735 | ||
736 | /* pull the remote_hb cache line */ | |
a812dcc3 DN |
737 | ret = xp_remote_memcpy(xp_pa(remote_vars), |
738 | part->sn.sn2.remote_vars_pa, | |
33ba3c77 DN |
739 | XPC_RP_VARS_SIZE); |
740 | if (ret != xpSuccess) { | |
741 | XPC_DEACTIVATE_PARTITION(part, ret); | |
742 | continue; | |
743 | } | |
744 | ||
745 | dev_dbg(xpc_part, "partid = %d, heartbeat = %ld, last_heartbeat" | |
746 | " = %ld, heartbeat_offline = %ld, HB_mask[0] = 0x%lx\n", | |
747 | partid, remote_vars->heartbeat, part->last_heartbeat, | |
748 | remote_vars->heartbeat_offline, | |
749 | remote_vars->heartbeating_to_mask[0]); | |
750 | ||
751 | if (((remote_vars->heartbeat == part->last_heartbeat) && | |
752 | (remote_vars->heartbeat_offline == 0)) || | |
753 | !xpc_hb_allowed(sn_partition_id, | |
754 | &remote_vars->heartbeating_to_mask)) { | |
755 | ||
756 | XPC_DEACTIVATE_PARTITION(part, xpNoHeartbeat); | |
757 | continue; | |
758 | } | |
759 | ||
760 | part->last_heartbeat = remote_vars->heartbeat; | |
761 | } | |
762 | } | |
763 | ||
764 | /* | |
765 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
766 | * | |
767 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
768 | * assumed to be of size XPC_RP_VARS_SIZE. | |
769 | */ | |
770 | static enum xp_retval | |
a812dcc3 DN |
771 | xpc_get_remote_vars_sn2(unsigned long remote_vars_pa, |
772 | struct xpc_vars_sn2 *remote_vars) | |
33ba3c77 DN |
773 | { |
774 | enum xp_retval ret; | |
775 | ||
776 | if (remote_vars_pa == 0) | |
777 | return xpVarsNotSet; | |
778 | ||
779 | /* pull over the cross partition variables */ | |
a812dcc3 | 780 | ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa, |
33ba3c77 DN |
781 | XPC_RP_VARS_SIZE); |
782 | if (ret != xpSuccess) | |
783 | return ret; | |
784 | ||
785 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
786 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
787 | return xpBadVersion; | |
788 | } | |
94bd2708 DN |
789 | |
790 | return xpSuccess; | |
791 | } | |
792 | ||
33ba3c77 | 793 | static void |
a47d5dac | 794 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
a812dcc3 | 795 | unsigned long remote_rp_pa, int nasid) |
33ba3c77 | 796 | { |
7fb5e59d | 797 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
798 | } |
799 | ||
800 | static void | |
801 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
802 | { | |
7fb5e59d | 803 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
804 | } |
805 | ||
806 | static void | |
807 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
808 | { | |
809 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
810 | unsigned long irq_flags; | |
c39838ce | 811 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 812 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 813 | sizeof(struct amo))); |
a47d5dac DN |
814 | |
815 | local_irq_save(irq_flags); | |
816 | ||
c39838ce | 817 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 818 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
04de7418 DN |
819 | BIT(sn_partition_id)); |
820 | ||
a47d5dac DN |
821 | /* |
822 | * We must always use the nofault function regardless of whether we | |
823 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
824 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 825 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
826 | */ |
827 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
828 | variable), | |
829 | xp_nofault_PIOR_target)); | |
830 | ||
831 | local_irq_restore(irq_flags); | |
832 | ||
833 | /* | |
834 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 835 | * bit in their deactivate request amo. |
a47d5dac | 836 | */ |
7fb5e59d | 837 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
838 | cnodeid_to_nasid(0), |
839 | part_sn2->activate_IRQ_nasid, | |
840 | part_sn2->activate_IRQ_phys_cpuid); | |
841 | } | |
842 | ||
843 | static void | |
844 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
845 | { | |
846 | unsigned long irq_flags; | |
c39838ce | 847 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 848 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 849 | sizeof(struct amo))); |
a47d5dac DN |
850 | |
851 | local_irq_save(irq_flags); | |
852 | ||
c39838ce | 853 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac | 854 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
04de7418 DN |
855 | ~BIT(sn_partition_id)); |
856 | ||
a47d5dac DN |
857 | /* |
858 | * We must always use the nofault function regardless of whether we | |
859 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
860 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 861 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
862 | */ |
863 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
864 | variable), | |
865 | xp_nofault_PIOR_target)); | |
866 | ||
867 | local_irq_restore(irq_flags); | |
868 | } | |
869 | ||
870 | static int | |
871 | xpc_partition_deactivation_requested_sn2(short partid) | |
872 | { | |
8e85c23e DN |
873 | struct amo *amo = xpc_vars_sn2->amos_page + |
874 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 875 | |
c39838ce | 876 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac | 877 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
04de7418 | 878 | BIT(partid)) != 0; |
33ba3c77 DN |
879 | } |
880 | ||
881 | /* | |
882 | * Update the remote partition's info. | |
883 | */ | |
884 | static void | |
885 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
81fe7883 | 886 | unsigned long *remote_rp_ts_jiffies, |
a812dcc3 DN |
887 | unsigned long remote_rp_pa, |
888 | unsigned long remote_vars_pa, | |
33ba3c77 DN |
889 | struct xpc_vars_sn2 *remote_vars) |
890 | { | |
a47d5dac DN |
891 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
892 | ||
33ba3c77 DN |
893 | part->remote_rp_version = remote_rp_version; |
894 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
895 | part->remote_rp_version); | |
896 | ||
81fe7883 DN |
897 | part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies; |
898 | dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n", | |
899 | part->remote_rp_ts_jiffies); | |
33ba3c77 DN |
900 | |
901 | part->remote_rp_pa = remote_rp_pa; | |
902 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
903 | ||
a47d5dac | 904 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 905 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 906 | part_sn2->remote_vars_pa); |
33ba3c77 DN |
907 | |
908 | part->last_heartbeat = remote_vars->heartbeat; | |
909 | dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n", | |
910 | part->last_heartbeat); | |
911 | ||
a47d5dac | 912 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 913 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 914 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 915 | |
a47d5dac DN |
916 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
917 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
918 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 919 | |
a47d5dac DN |
920 | part_sn2->activate_IRQ_phys_cpuid = |
921 | remote_vars->activate_IRQ_phys_cpuid; | |
922 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
923 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 924 | |
a47d5dac | 925 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 926 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 927 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 928 | |
a47d5dac | 929 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 930 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 931 | part_sn2->remote_vars_version); |
33ba3c77 DN |
932 | } |
933 | ||
934 | /* | |
7fb5e59d DN |
935 | * Prior code has determined the nasid which generated a activate IRQ. |
936 | * Inspect that nasid to determine if its partition needs to be activated | |
937 | * or deactivated. | |
33ba3c77 | 938 | * |
7fb5e59d | 939 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
940 | * flags indicate it is not active and it has a heartbeat. A |
941 | * partition is considered "awaiting deactivation" if our partition | |
942 | * flags indicate it is active but it has no heartbeat or it is not | |
943 | * sending its heartbeat to us. | |
944 | * | |
945 | * To determine the heartbeat, the remote nasid must have a properly | |
946 | * initialized reserved page. | |
947 | */ | |
948 | static void | |
6e41017a | 949 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
950 | { |
951 | struct xpc_rsvd_page *remote_rp; | |
952 | struct xpc_vars_sn2 *remote_vars; | |
a812dcc3 DN |
953 | unsigned long remote_rp_pa; |
954 | unsigned long remote_vars_pa; | |
33ba3c77 DN |
955 | int remote_rp_version; |
956 | int reactivate = 0; | |
81fe7883 | 957 | unsigned long remote_rp_ts_jiffies = 0; |
33ba3c77 DN |
958 | short partid; |
959 | struct xpc_partition *part; | |
a47d5dac | 960 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
961 | enum xp_retval ret; |
962 | ||
963 | /* pull over the reserved page structure */ | |
964 | ||
ee6665e3 | 965 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
966 | |
967 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
968 | if (ret != xpSuccess) { | |
969 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
970 | "which sent interrupt, reason=%d\n", nasid, ret); | |
971 | return; | |
972 | } | |
973 | ||
974 | remote_vars_pa = remote_rp->sn.vars_pa; | |
975 | remote_rp_version = remote_rp->version; | |
81fe7883 | 976 | remote_rp_ts_jiffies = remote_rp->ts_jiffies; |
33ba3c77 DN |
977 | |
978 | partid = remote_rp->SAL_partid; | |
979 | part = &xpc_partitions[partid]; | |
a47d5dac | 980 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
981 | |
982 | /* pull over the cross partition variables */ | |
983 | ||
ee6665e3 | 984 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
985 | |
986 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
987 | if (ret != xpSuccess) { | |
33ba3c77 DN |
988 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
989 | "which sent interrupt, reason=%d\n", nasid, ret); | |
990 | ||
991 | XPC_DEACTIVATE_PARTITION(part, ret); | |
992 | return; | |
993 | } | |
994 | ||
6e41017a | 995 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
996 | |
997 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
6e41017a | 998 | "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd, |
33ba3c77 DN |
999 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
1000 | ||
1001 | if (xpc_partition_disengaged(part) && | |
1002 | part->act_state == XPC_P_INACTIVE) { | |
1003 | ||
1004 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1005 | &remote_rp_ts_jiffies, |
1006 | remote_rp_pa, remote_vars_pa, | |
1007 | remote_vars); | |
33ba3c77 | 1008 | |
a47d5dac DN |
1009 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
1010 | /* | |
1011 | * Other side is waiting on us to deactivate even though | |
1012 | * we already have. | |
1013 | */ | |
1014 | return; | |
33ba3c77 DN |
1015 | } |
1016 | ||
1017 | xpc_activate_partition(part); | |
1018 | return; | |
1019 | } | |
1020 | ||
1021 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 1022 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 1023 | |
81fe7883 | 1024 | if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) { |
33ba3c77 | 1025 | |
a47d5dac | 1026 | /* the other side rebooted */ |
33ba3c77 | 1027 | |
a47d5dac DN |
1028 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1029 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1030 | |
1031 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
81fe7883 DN |
1032 | &remote_rp_ts_jiffies, |
1033 | remote_rp_pa, remote_vars_pa, | |
1034 | remote_vars); | |
33ba3c77 | 1035 | reactivate = 1; |
33ba3c77 DN |
1036 | } |
1037 | ||
a47d5dac | 1038 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1039 | /* still waiting on other side to disengage from us */ |
1040 | return; | |
1041 | } | |
1042 | ||
a47d5dac | 1043 | if (reactivate) |
33ba3c77 | 1044 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1045 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1046 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1047 | } |
1048 | ||
1049 | /* | |
c39838ce | 1050 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1051 | * which are set. Each bit indicates a nasid sending a partition |
1052 | * activation or deactivation request. | |
1053 | * | |
1054 | * Return #of IRQs detected. | |
1055 | */ | |
1056 | int | |
6e41017a | 1057 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 | 1058 | { |
04de7418 DN |
1059 | int l; |
1060 | int b; | |
1061 | unsigned long nasid_mask_long; | |
33ba3c77 DN |
1062 | u64 nasid; /* remote nasid */ |
1063 | int n_IRQs_detected = 0; | |
c39838ce | 1064 | struct amo *act_amos; |
33ba3c77 | 1065 | |
8e85c23e | 1066 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1067 | |
04de7418 DN |
1068 | /* scan through activate amo variables looking for non-zero entries */ |
1069 | for (l = 0; l < xpc_nasid_mask_nlongs; l++) { | |
33ba3c77 DN |
1070 | |
1071 | if (xpc_exiting) | |
1072 | break; | |
1073 | ||
04de7418 DN |
1074 | nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]); |
1075 | ||
1076 | b = find_first_bit(&nasid_mask_long, BITS_PER_LONG); | |
1077 | if (b >= BITS_PER_LONG) { | |
1078 | /* no IRQs from nasids in this amo variable */ | |
33ba3c77 DN |
1079 | continue; |
1080 | } | |
1081 | ||
04de7418 DN |
1082 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l, |
1083 | nasid_mask_long); | |
33ba3c77 DN |
1084 | |
1085 | /* | |
1086 | * If this nasid has been added to the machine since | |
1087 | * our partition was reset, this will retain the | |
1088 | * remote nasid in our reserved pages machine mask. | |
1089 | * This is used in the event of module reload. | |
1090 | */ | |
04de7418 | 1091 | xpc_mach_nasids[l] |= nasid_mask_long; |
33ba3c77 DN |
1092 | |
1093 | /* locate the nasid(s) which sent interrupts */ | |
1094 | ||
04de7418 DN |
1095 | do { |
1096 | n_IRQs_detected++; | |
1097 | nasid = (l * BITS_PER_LONG + b) * 2; | |
1098 | dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid); | |
1099 | xpc_identify_activate_IRQ_req_sn2(nasid); | |
1100 | ||
1101 | b = find_next_bit(&nasid_mask_long, BITS_PER_LONG, | |
1102 | b + 1); | |
1103 | } while (b < BITS_PER_LONG); | |
33ba3c77 DN |
1104 | } |
1105 | return n_IRQs_detected; | |
1106 | } | |
1107 | ||
1108 | static void | |
6e41017a | 1109 | xpc_process_activate_IRQ_rcvd_sn2(int n_IRQs_expected) |
33ba3c77 DN |
1110 | { |
1111 | int n_IRQs_detected; | |
1112 | ||
6e41017a | 1113 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1114 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1115 | /* retry once to help avoid missing amo */ |
6e41017a | 1116 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1117 | } |
1118 | } | |
1119 | ||
185c3a1b DN |
1120 | /* |
1121 | * Guarantee that the kzalloc'd memory is cacheline aligned. | |
1122 | */ | |
1123 | static void * | |
1124 | xpc_kzalloc_cacheline_aligned_sn2(size_t size, gfp_t flags, void **base) | |
1125 | { | |
1126 | /* see if kzalloc will give us cachline aligned memory by default */ | |
1127 | *base = kzalloc(size, flags); | |
1128 | if (*base == NULL) | |
1129 | return NULL; | |
1130 | ||
1131 | if ((u64)*base == L1_CACHE_ALIGN((u64)*base)) | |
1132 | return *base; | |
1133 | ||
1134 | kfree(*base); | |
1135 | ||
1136 | /* nope, we'll have to do it ourselves */ | |
1137 | *base = kzalloc(size + L1_CACHE_BYTES, flags); | |
1138 | if (*base == NULL) | |
1139 | return NULL; | |
1140 | ||
1141 | return (void *)L1_CACHE_ALIGN((u64)*base); | |
1142 | } | |
1143 | ||
e17d416b DN |
1144 | /* |
1145 | * Setup the infrastructure necessary to support XPartition Communication | |
1146 | * between the specified remote partition and the local one. | |
1147 | */ | |
1148 | static enum xp_retval | |
1149 | xpc_setup_infrastructure_sn2(struct xpc_partition *part) | |
1150 | { | |
a47d5dac | 1151 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1152 | enum xp_retval retval; |
1153 | int ret; | |
1154 | int cpuid; | |
1155 | int ch_number; | |
1156 | struct xpc_channel *ch; | |
1157 | struct timer_list *timer; | |
1158 | short partid = XPC_PARTID(part); | |
1159 | ||
1160 | /* | |
1161 | * Allocate all of the channel structures as a contiguous chunk of | |
1162 | * memory. | |
1163 | */ | |
1164 | DBUG_ON(part->channels != NULL); | |
1165 | part->channels = kzalloc(sizeof(struct xpc_channel) * XPC_MAX_NCHANNELS, | |
1166 | GFP_KERNEL); | |
1167 | if (part->channels == NULL) { | |
1168 | dev_err(xpc_chan, "can't get memory for channels\n"); | |
1169 | return xpNoMemory; | |
1170 | } | |
1171 | ||
1172 | /* allocate all the required GET/PUT values */ | |
1173 | ||
185c3a1b DN |
1174 | part_sn2->local_GPs = |
1175 | xpc_kzalloc_cacheline_aligned_sn2(XPC_GP_SIZE, GFP_KERNEL, | |
1176 | &part_sn2->local_GPs_base); | |
a47d5dac | 1177 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1178 | dev_err(xpc_chan, "can't get memory for local get/put " |
1179 | "values\n"); | |
1180 | retval = xpNoMemory; | |
1181 | goto out_1; | |
1182 | } | |
1183 | ||
185c3a1b DN |
1184 | part_sn2->remote_GPs = |
1185 | xpc_kzalloc_cacheline_aligned_sn2(XPC_GP_SIZE, GFP_KERNEL, | |
1186 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1187 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1188 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1189 | "values\n"); | |
1190 | retval = xpNoMemory; | |
1191 | goto out_2; | |
1192 | } | |
1193 | ||
a47d5dac | 1194 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1195 | |
1196 | /* allocate all the required open and close args */ | |
1197 | ||
1198 | part->local_openclose_args = | |
185c3a1b DN |
1199 | xpc_kzalloc_cacheline_aligned_sn2(XPC_OPENCLOSE_ARGS_SIZE, |
1200 | GFP_KERNEL, | |
1201 | &part->local_openclose_args_base); | |
e17d416b DN |
1202 | if (part->local_openclose_args == NULL) { |
1203 | dev_err(xpc_chan, "can't get memory for local connect args\n"); | |
1204 | retval = xpNoMemory; | |
1205 | goto out_3; | |
1206 | } | |
1207 | ||
1208 | part->remote_openclose_args = | |
185c3a1b DN |
1209 | xpc_kzalloc_cacheline_aligned_sn2(XPC_OPENCLOSE_ARGS_SIZE, |
1210 | GFP_KERNEL, | |
1211 | &part->remote_openclose_args_base); | |
e17d416b DN |
1212 | if (part->remote_openclose_args == NULL) { |
1213 | dev_err(xpc_chan, "can't get memory for remote connect args\n"); | |
1214 | retval = xpNoMemory; | |
1215 | goto out_4; | |
1216 | } | |
1217 | ||
a47d5dac | 1218 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1219 | |
7fb5e59d DN |
1220 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
1221 | part->chctl.all_flags = 0; | |
1222 | spin_lock_init(&part->chctl_lock); | |
e17d416b | 1223 | |
7fb5e59d DN |
1224 | part_sn2->notify_IRQ_nasid = 0; |
1225 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1226 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b DN |
1227 | |
1228 | atomic_set(&part->channel_mgr_requests, 1); | |
1229 | init_waitqueue_head(&part->channel_mgr_wq); | |
1230 | ||
7fb5e59d | 1231 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1232 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1233 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1234 | (void *)(u64)partid); |
e17d416b DN |
1235 | if (ret != 0) { |
1236 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1237 | "errno=%d\n", -ret); | |
1238 | retval = xpLackOfResources; | |
1239 | goto out_5; | |
1240 | } | |
1241 | ||
7fb5e59d | 1242 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1243 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1244 | init_timer(timer); |
a47d5dac | 1245 | timer->function = |
7fb5e59d | 1246 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1247 | timer->data = (unsigned long)part; |
7fb5e59d | 1248 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1249 | add_timer(timer); |
1250 | ||
1251 | part->nchannels = XPC_MAX_NCHANNELS; | |
1252 | ||
1253 | atomic_set(&part->nchannels_active, 0); | |
1254 | atomic_set(&part->nchannels_engaged, 0); | |
1255 | ||
1256 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { | |
1257 | ch = &part->channels[ch_number]; | |
1258 | ||
1259 | ch->partid = partid; | |
1260 | ch->number = ch_number; | |
1261 | ch->flags = XPC_C_DISCONNECTED; | |
1262 | ||
a47d5dac | 1263 | ch->sn.sn2.local_GP = &part_sn2->local_GPs[ch_number]; |
e17d416b DN |
1264 | ch->local_openclose_args = |
1265 | &part->local_openclose_args[ch_number]; | |
1266 | ||
1267 | atomic_set(&ch->kthreads_assigned, 0); | |
1268 | atomic_set(&ch->kthreads_idle, 0); | |
1269 | atomic_set(&ch->kthreads_active, 0); | |
1270 | ||
1271 | atomic_set(&ch->references, 0); | |
1272 | atomic_set(&ch->n_to_notify, 0); | |
1273 | ||
1274 | spin_lock_init(&ch->lock); | |
a47d5dac | 1275 | mutex_init(&ch->sn.sn2.msg_to_pull_mutex); |
e17d416b DN |
1276 | init_completion(&ch->wdisconnect_wait); |
1277 | ||
1278 | atomic_set(&ch->n_on_msg_allocate_wq, 0); | |
1279 | init_waitqueue_head(&ch->msg_allocate_wq); | |
1280 | init_waitqueue_head(&ch->idle_wq); | |
1281 | } | |
1282 | ||
1283 | /* | |
1284 | * With the setting of the partition setup_state to XPC_P_SETUP, we're | |
1285 | * declaring that this partition is ready to go. | |
1286 | */ | |
1287 | part->setup_state = XPC_P_SETUP; | |
1288 | ||
1289 | /* | |
1290 | * Setup the per partition specific variables required by the | |
1291 | * remote partition to establish channel connections with us. | |
1292 | * | |
1293 | * The setting of the magic # indicates that these per partition | |
1294 | * specific variables are ready to be used. | |
1295 | */ | |
a812dcc3 | 1296 | xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs); |
8e85c23e | 1297 | xpc_vars_part_sn2[partid].openclose_args_pa = |
a812dcc3 | 1298 | xp_pa(part->local_openclose_args); |
8e85c23e | 1299 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
a812dcc3 | 1300 | xp_pa(part_sn2->local_chctl_amo_va); |
e17d416b | 1301 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1302 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1303 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1304 | cpu_physical_id(cpuid); | |
1305 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
1306 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1; | |
e17d416b DN |
1307 | |
1308 | return xpSuccess; | |
1309 | ||
1310 | /* setup of infrastructure failed */ | |
1311 | out_5: | |
1312 | kfree(part->remote_openclose_args_base); | |
1313 | part->remote_openclose_args = NULL; | |
1314 | out_4: | |
1315 | kfree(part->local_openclose_args_base); | |
1316 | part->local_openclose_args = NULL; | |
1317 | out_3: | |
a47d5dac DN |
1318 | kfree(part_sn2->remote_GPs_base); |
1319 | part_sn2->remote_GPs = NULL; | |
e17d416b | 1320 | out_2: |
a47d5dac DN |
1321 | kfree(part_sn2->local_GPs_base); |
1322 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1323 | out_1: |
1324 | kfree(part->channels); | |
1325 | part->channels = NULL; | |
1326 | return retval; | |
1327 | } | |
1328 | ||
1329 | /* | |
1330 | * Teardown the infrastructure necessary to support XPartition Communication | |
1331 | * between the specified remote partition and the local one. | |
1332 | */ | |
1333 | static void | |
1334 | xpc_teardown_infrastructure_sn2(struct xpc_partition *part) | |
1335 | { | |
a47d5dac | 1336 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1337 | short partid = XPC_PARTID(part); |
1338 | ||
1339 | /* | |
1340 | * We start off by making this partition inaccessible to local | |
1341 | * processes by marking it as no longer setup. Then we make it | |
1342 | * inaccessible to remote processes by clearing the XPC per partition | |
1343 | * specific variable's magic # (which indicates that these variables | |
7fb5e59d | 1344 | * are no longer valid) and by ignoring all XPC notify IRQs sent to |
e17d416b DN |
1345 | * this partition. |
1346 | */ | |
1347 | ||
1348 | DBUG_ON(atomic_read(&part->nchannels_engaged) != 0); | |
1349 | DBUG_ON(atomic_read(&part->nchannels_active) != 0); | |
1350 | DBUG_ON(part->setup_state != XPC_P_SETUP); | |
1351 | part->setup_state = XPC_P_WTEARDOWN; | |
1352 | ||
8e85c23e | 1353 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b DN |
1354 | |
1355 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); | |
1356 | ||
1357 | /* | |
1358 | * Before proceeding with the teardown we have to wait until all | |
1359 | * existing references cease. | |
1360 | */ | |
1361 | wait_event(part->teardown_wq, (atomic_read(&part->references) == 0)); | |
1362 | ||
1363 | /* now we can begin tearing down the infrastructure */ | |
1364 | ||
1365 | part->setup_state = XPC_P_TORNDOWN; | |
1366 | ||
1367 | /* in case we've still got outstanding timers registered... */ | |
a47d5dac | 1368 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
e17d416b DN |
1369 | |
1370 | kfree(part->remote_openclose_args_base); | |
1371 | part->remote_openclose_args = NULL; | |
1372 | kfree(part->local_openclose_args_base); | |
1373 | part->local_openclose_args = NULL; | |
a47d5dac DN |
1374 | kfree(part_sn2->remote_GPs_base); |
1375 | part_sn2->remote_GPs = NULL; | |
1376 | kfree(part_sn2->local_GPs_base); | |
1377 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1378 | kfree(part->channels); |
1379 | part->channels = NULL; | |
7fb5e59d | 1380 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1381 | } |
1382 | ||
1383 | /* | |
1384 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1385 | * (or multiple cachelines) from a remote partition. | |
1386 | * | |
a812dcc3 | 1387 | * src_pa must be a cacheline aligned physical address on the remote partition. |
e17d416b DN |
1388 | * dst must be a cacheline aligned virtual address on this partition. |
1389 | * cnt must be cacheline sized | |
1390 | */ | |
ea57f80c | 1391 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1392 | static enum xp_retval |
1393 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
a812dcc3 | 1394 | const unsigned long src_pa, size_t cnt) |
e17d416b DN |
1395 | { |
1396 | enum xp_retval ret; | |
1397 | ||
a812dcc3 DN |
1398 | DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa)); |
1399 | DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst)); | |
e17d416b DN |
1400 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); |
1401 | ||
1402 | if (part->act_state == XPC_P_DEACTIVATING) | |
1403 | return part->reason; | |
1404 | ||
a812dcc3 | 1405 | ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt); |
e17d416b DN |
1406 | if (ret != xpSuccess) { |
1407 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1408 | " ret=%d\n", XPC_PARTID(part), ret); | |
1409 | } | |
1410 | return ret; | |
1411 | } | |
1412 | ||
1413 | /* | |
1414 | * Pull the remote per partition specific variables from the specified | |
1415 | * partition. | |
1416 | */ | |
1417 | static enum xp_retval | |
1418 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1419 | { | |
a47d5dac | 1420 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1421 | u8 buffer[L1_CACHE_BYTES * 2]; |
1422 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1423 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1424 | struct xpc_vars_part_sn2 *pulled_entry; | |
a812dcc3 DN |
1425 | unsigned long remote_entry_cacheline_pa; |
1426 | unsigned long remote_entry_pa; | |
e17d416b DN |
1427 | short partid = XPC_PARTID(part); |
1428 | enum xp_retval ret; | |
1429 | ||
1430 | /* pull the cacheline that contains the variables we're interested in */ | |
1431 | ||
a47d5dac DN |
1432 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1433 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1434 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1435 | ||
a47d5dac | 1436 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1437 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1438 | ||
1439 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1440 | ||
1441 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1442 | + (remote_entry_pa & | |
1443 | (L1_CACHE_BYTES - 1))); | |
1444 | ||
1445 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
a812dcc3 | 1446 | remote_entry_cacheline_pa, |
e17d416b DN |
1447 | L1_CACHE_BYTES); |
1448 | if (ret != xpSuccess) { | |
1449 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1450 | "partition %d, ret=%d\n", partid, ret); | |
1451 | return ret; | |
1452 | } | |
1453 | ||
1454 | /* see if they've been set up yet */ | |
1455 | ||
1456 | if (pulled_entry->magic != XPC_VP_MAGIC1 && | |
1457 | pulled_entry->magic != XPC_VP_MAGIC2) { | |
1458 | ||
1459 | if (pulled_entry->magic != 0) { | |
1460 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
1461 | "partition %d has bad magic value (=0x%lx)\n", | |
1462 | partid, sn_partition_id, pulled_entry->magic); | |
1463 | return xpBadMagic; | |
1464 | } | |
1465 | ||
1466 | /* they've not been initialized yet */ | |
1467 | return xpRetry; | |
1468 | } | |
1469 | ||
8e85c23e | 1470 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1) { |
e17d416b DN |
1471 | |
1472 | /* validate the variables */ | |
1473 | ||
1474 | if (pulled_entry->GPs_pa == 0 || | |
1475 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1476 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1477 | |
1478 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1479 | "partition %d are not valid\n", partid, | |
1480 | sn_partition_id); | |
1481 | return xpInvalidAddress; | |
1482 | } | |
1483 | ||
1484 | /* the variables we imported look to be valid */ | |
1485 | ||
a47d5dac DN |
1486 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1487 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1488 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1489 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1490 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1491 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1492 | part_sn2->notify_IRQ_phys_cpuid = | |
1493 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1494 | |
1495 | if (part->nchannels > pulled_entry->nchannels) | |
1496 | part->nchannels = pulled_entry->nchannels; | |
1497 | ||
1498 | /* let the other side know that we've pulled their variables */ | |
1499 | ||
8e85c23e | 1500 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2; |
e17d416b DN |
1501 | } |
1502 | ||
1503 | if (pulled_entry->magic == XPC_VP_MAGIC1) | |
1504 | return xpRetry; | |
1505 | ||
1506 | return xpSuccess; | |
1507 | } | |
1508 | ||
1509 | /* | |
1510 | * Establish first contact with the remote partititon. This involves pulling | |
1511 | * the XPC per partition variables from the remote partition and waiting for | |
1512 | * the remote partition to pull ours. | |
1513 | */ | |
1514 | static enum xp_retval | |
1515 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1516 | { | |
a47d5dac | 1517 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1518 | enum xp_retval ret; |
1519 | ||
33ba3c77 | 1520 | /* |
c39838ce | 1521 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1522 | * and cleanup errors within that address range should the remote |
1523 | * partition go down. We don't unregister this range because it is | |
1524 | * difficult to tell when outstanding writes to the remote partition | |
1525 | * are finished and thus when it is safe to unregister. This should | |
1526 | * not result in wasted space in the SAL xp_addr_region table because | |
1527 | * we should get the same page for remote_amos_page_pa after module | |
1528 | * reloads and system reboots. | |
1529 | */ | |
a47d5dac | 1530 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1531 | PAGE_SIZE, 1) < 0) { |
1532 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1533 | "xp_addr region\n", XPC_PARTID(part)); | |
1534 | ||
1535 | ret = xpPhysAddrRegFailed; | |
1536 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1537 | return ret; | |
1538 | } | |
1539 | ||
a47d5dac DN |
1540 | /* |
1541 | * Send activate IRQ to get other side to activate if they've not | |
1542 | * already begun to do so. | |
1543 | */ | |
7fb5e59d | 1544 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1545 | cnodeid_to_nasid(0), |
1546 | part_sn2->activate_IRQ_nasid, | |
1547 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1548 | |
e17d416b DN |
1549 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1550 | if (ret != xpRetry) { | |
1551 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1552 | return ret; | |
1553 | } | |
1554 | ||
1555 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1556 | "partition %d\n", XPC_PARTID(part)); | |
1557 | ||
1558 | /* wait a 1/4 of a second or so */ | |
1559 | (void)msleep_interruptible(250); | |
1560 | ||
1561 | if (part->act_state == XPC_P_DEACTIVATING) | |
1562 | return part->reason; | |
1563 | } | |
1564 | ||
1565 | return xpSuccess; | |
1566 | } | |
1567 | ||
1568 | /* | |
7fb5e59d | 1569 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1570 | */ |
1571 | static u64 | |
7fb5e59d | 1572 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1573 | { |
a47d5dac | 1574 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1575 | unsigned long irq_flags; |
7fb5e59d | 1576 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1577 | enum xp_retval ret; |
1578 | ||
1579 | /* | |
7fb5e59d | 1580 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1581 | */ |
1582 | ||
7fb5e59d DN |
1583 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1584 | chctl = part->chctl; | |
1585 | if (chctl.all_flags != 0) | |
1586 | part->chctl.all_flags = 0; | |
e17d416b | 1587 | |
7fb5e59d | 1588 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1589 | |
7fb5e59d | 1590 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1591 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1592 | remote_openclose_args, | |
a812dcc3 | 1593 | part_sn2-> |
e17d416b DN |
1594 | remote_openclose_args_pa, |
1595 | XPC_OPENCLOSE_ARGS_SIZE); | |
1596 | if (ret != xpSuccess) { | |
1597 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1598 | ||
1599 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1600 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1601 | ret); | |
1602 | ||
7fb5e59d DN |
1603 | /* don't bother processing chctl flags anymore */ |
1604 | chctl.all_flags = 0; | |
e17d416b DN |
1605 | } |
1606 | } | |
1607 | ||
7fb5e59d | 1608 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac | 1609 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
a812dcc3 | 1610 | part_sn2->remote_GPs_pa, |
e17d416b DN |
1611 | XPC_GP_SIZE); |
1612 | if (ret != xpSuccess) { | |
1613 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1614 | ||
1615 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1616 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1617 | ||
7fb5e59d DN |
1618 | /* don't bother processing chctl flags anymore */ |
1619 | chctl.all_flags = 0; | |
e17d416b DN |
1620 | } |
1621 | } | |
1622 | ||
7fb5e59d | 1623 | return chctl.all_flags; |
e17d416b DN |
1624 | } |
1625 | ||
185c3a1b DN |
1626 | /* |
1627 | * Allocate the local message queue and the notify queue. | |
1628 | */ | |
1629 | static enum xp_retval | |
1630 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1631 | { | |
1632 | unsigned long irq_flags; | |
1633 | int nentries; | |
1634 | size_t nbytes; | |
1635 | ||
1636 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1637 | ||
1638 | nbytes = nentries * ch->msg_size; | |
1639 | ch->local_msgqueue = | |
1640 | xpc_kzalloc_cacheline_aligned_sn2(nbytes, GFP_KERNEL, | |
1641 | &ch->local_msgqueue_base); | |
1642 | if (ch->local_msgqueue == NULL) | |
1643 | continue; | |
1644 | ||
1645 | nbytes = nentries * sizeof(struct xpc_notify); | |
1646 | ch->notify_queue = kzalloc(nbytes, GFP_KERNEL); | |
1647 | if (ch->notify_queue == NULL) { | |
1648 | kfree(ch->local_msgqueue_base); | |
1649 | ch->local_msgqueue = NULL; | |
1650 | continue; | |
1651 | } | |
1652 | ||
1653 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1654 | if (nentries < ch->local_nentries) { | |
1655 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1656 | "partid=%d, channel=%d\n", nentries, | |
1657 | ch->local_nentries, ch->partid, ch->number); | |
1658 | ||
1659 | ch->local_nentries = nentries; | |
1660 | } | |
1661 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1662 | return xpSuccess; | |
1663 | } | |
1664 | ||
1665 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1666 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1667 | return xpNoMemory; | |
1668 | } | |
1669 | ||
1670 | /* | |
1671 | * Allocate the cached remote message queue. | |
1672 | */ | |
1673 | static enum xp_retval | |
1674 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1675 | { | |
1676 | unsigned long irq_flags; | |
1677 | int nentries; | |
1678 | size_t nbytes; | |
1679 | ||
1680 | DBUG_ON(ch->remote_nentries <= 0); | |
1681 | ||
1682 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1683 | ||
1684 | nbytes = nentries * ch->msg_size; | |
1685 | ch->remote_msgqueue = | |
1686 | xpc_kzalloc_cacheline_aligned_sn2(nbytes, GFP_KERNEL, | |
1687 | &ch->remote_msgqueue_base); | |
1688 | if (ch->remote_msgqueue == NULL) | |
1689 | continue; | |
1690 | ||
1691 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1692 | if (nentries < ch->remote_nentries) { | |
1693 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1694 | "partid=%d, channel=%d\n", nentries, | |
1695 | ch->remote_nentries, ch->partid, ch->number); | |
1696 | ||
1697 | ch->remote_nentries = nentries; | |
1698 | } | |
1699 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1700 | return xpSuccess; | |
1701 | } | |
1702 | ||
1703 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1704 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1705 | return xpNoMemory; | |
1706 | } | |
1707 | ||
1708 | /* | |
1709 | * Allocate message queues and other stuff associated with a channel. | |
1710 | * | |
1711 | * Note: Assumes all of the channel sizes are filled in. | |
1712 | */ | |
1713 | static enum xp_retval | |
1714 | xpc_allocate_msgqueues_sn2(struct xpc_channel *ch) | |
1715 | { | |
1716 | enum xp_retval ret; | |
1717 | ||
1718 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1719 | ||
1720 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1721 | if (ret == xpSuccess) { | |
1722 | ||
1723 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1724 | if (ret != xpSuccess) { | |
1725 | kfree(ch->local_msgqueue_base); | |
1726 | ch->local_msgqueue = NULL; | |
1727 | kfree(ch->notify_queue); | |
1728 | ch->notify_queue = NULL; | |
1729 | } | |
1730 | } | |
1731 | return ret; | |
1732 | } | |
1733 | ||
1734 | /* | |
1735 | * Free up message queues and other stuff that were allocated for the specified | |
1736 | * channel. | |
1737 | * | |
1738 | * Note: ch->reason and ch->reason_line are left set for debugging purposes, | |
1739 | * they're cleared when XPC_C_DISCONNECTED is cleared. | |
1740 | */ | |
1741 | static void | |
1742 | xpc_free_msgqueues_sn2(struct xpc_channel *ch) | |
1743 | { | |
1744 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1745 | ||
1746 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
1747 | DBUG_ON(atomic_read(&ch->n_to_notify) != 0); | |
1748 | ||
1749 | ch->remote_msgqueue_pa = 0; | |
1750 | ch->func = NULL; | |
1751 | ch->key = NULL; | |
1752 | ch->msg_size = 0; | |
1753 | ch->local_nentries = 0; | |
1754 | ch->remote_nentries = 0; | |
1755 | ch->kthreads_assigned_limit = 0; | |
1756 | ch->kthreads_idle_limit = 0; | |
1757 | ||
1758 | ch_sn2->local_GP->get = 0; | |
1759 | ch_sn2->local_GP->put = 0; | |
1760 | ch_sn2->remote_GP.get = 0; | |
1761 | ch_sn2->remote_GP.put = 0; | |
1762 | ch_sn2->w_local_GP.get = 0; | |
1763 | ch_sn2->w_local_GP.put = 0; | |
1764 | ch_sn2->w_remote_GP.get = 0; | |
1765 | ch_sn2->w_remote_GP.put = 0; | |
1766 | ch_sn2->next_msg_to_pull = 0; | |
1767 | ||
1768 | if (ch->flags & XPC_C_SETUP) { | |
1769 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1770 | ch->flags, ch->partid, ch->number); | |
1771 | ||
1772 | kfree(ch->local_msgqueue_base); | |
1773 | ch->local_msgqueue = NULL; | |
1774 | kfree(ch->remote_msgqueue_base); | |
1775 | ch->remote_msgqueue = NULL; | |
1776 | kfree(ch->notify_queue); | |
1777 | ch->notify_queue = NULL; | |
1778 | } | |
1779 | } | |
1780 | ||
a47d5dac DN |
1781 | /* |
1782 | * Notify those who wanted to be notified upon delivery of their message. | |
1783 | */ | |
1784 | static void | |
1785 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1786 | { | |
1787 | struct xpc_notify *notify; | |
1788 | u8 notify_type; | |
1789 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1790 | ||
1791 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1792 | ||
1793 | notify = &ch->notify_queue[get % ch->local_nentries]; | |
1794 | ||
1795 | /* | |
1796 | * See if the notify entry indicates it was associated with | |
1797 | * a message who's sender wants to be notified. It is possible | |
1798 | * that it is, but someone else is doing or has done the | |
1799 | * notification. | |
1800 | */ | |
1801 | notify_type = notify->type; | |
1802 | if (notify_type == 0 || | |
1803 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1804 | continue; | |
1805 | } | |
1806 | ||
1807 | DBUG_ON(notify_type != XPC_N_CALL); | |
1808 | ||
1809 | atomic_dec(&ch->n_to_notify); | |
1810 | ||
1811 | if (notify->func != NULL) { | |
1812 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p, " | |
1813 | "msg_number=%ld, partid=%d, channel=%d\n", | |
1814 | (void *)notify, get, ch->partid, ch->number); | |
1815 | ||
1816 | notify->func(reason, ch->partid, ch->number, | |
1817 | notify->key); | |
1818 | ||
1819 | dev_dbg(xpc_chan, "notify->func() returned, " | |
1820 | "notify=0x%p, msg_number=%ld, partid=%d, " | |
1821 | "channel=%d\n", (void *)notify, get, | |
1822 | ch->partid, ch->number); | |
1823 | } | |
1824 | } | |
1825 | } | |
1826 | ||
1827 | static void | |
1828 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1829 | { | |
1830 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1831 | } | |
1832 | ||
1833 | /* | |
1834 | * Clear some of the msg flags in the local message queue. | |
1835 | */ | |
1836 | static inline void | |
1837 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1838 | { | |
1839 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1840 | struct xpc_msg *msg; | |
1841 | s64 get; | |
1842 | ||
1843 | get = ch_sn2->w_remote_GP.get; | |
1844 | do { | |
1845 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
1846 | (get % ch->local_nentries) * | |
1847 | ch->msg_size); | |
1848 | msg->flags = 0; | |
1849 | } while (++get < ch_sn2->remote_GP.get); | |
1850 | } | |
1851 | ||
1852 | /* | |
1853 | * Clear some of the msg flags in the remote message queue. | |
1854 | */ | |
1855 | static inline void | |
1856 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1857 | { | |
1858 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1859 | struct xpc_msg *msg; | |
1860 | s64 put; | |
1861 | ||
1862 | put = ch_sn2->w_remote_GP.put; | |
1863 | do { | |
1864 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + | |
1865 | (put % ch->remote_nentries) * | |
1866 | ch->msg_size); | |
1867 | msg->flags = 0; | |
1868 | } while (++put < ch_sn2->remote_GP.put); | |
1869 | } | |
1870 | ||
1871 | static void | |
7fb5e59d | 1872 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1873 | { |
1874 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1875 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1876 | int nmsgs_sent; | |
1877 | ||
1878 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1879 | ||
1880 | /* See what, if anything, has changed for each connected channel */ | |
1881 | ||
1882 | xpc_msgqueue_ref(ch); | |
1883 | ||
1884 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1885 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1886 | /* nothing changed since GPs were last pulled */ | |
1887 | xpc_msgqueue_deref(ch); | |
1888 | return; | |
1889 | } | |
1890 | ||
1891 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1892 | xpc_msgqueue_deref(ch); | |
1893 | return; | |
1894 | } | |
1895 | ||
1896 | /* | |
1897 | * First check to see if messages recently sent by us have been | |
1898 | * received by the other side. (The remote GET value will have | |
1899 | * changed since we last looked at it.) | |
1900 | */ | |
1901 | ||
1902 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1903 | ||
1904 | /* | |
1905 | * We need to notify any senders that want to be notified | |
1906 | * that their sent messages have been received by their | |
1907 | * intended recipients. We need to do this before updating | |
1908 | * w_remote_GP.get so that we don't allocate the same message | |
1909 | * queue entries prematurely (see xpc_allocate_msg()). | |
1910 | */ | |
1911 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1912 | /* | |
1913 | * Notify senders that messages sent have been | |
1914 | * received and delivered by the other side. | |
1915 | */ | |
1916 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1917 | ch_sn2->remote_GP.get); | |
1918 | } | |
1919 | ||
1920 | /* | |
1921 | * Clear msg->flags in previously sent messages, so that | |
1922 | * they're ready for xpc_allocate_msg(). | |
1923 | */ | |
1924 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1925 | ||
1926 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1927 | ||
1928 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, " | |
1929 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, | |
1930 | ch->number); | |
1931 | ||
1932 | /* | |
1933 | * If anyone was waiting for message queue entries to become | |
1934 | * available, wake them up. | |
1935 | */ | |
1936 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1937 | wake_up(&ch->msg_allocate_wq); | |
1938 | } | |
1939 | ||
1940 | /* | |
1941 | * Now check for newly sent messages by the other side. (The remote | |
1942 | * PUT value will have changed since we last looked at it.) | |
1943 | */ | |
1944 | ||
1945 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1946 | /* | |
1947 | * Clear msg->flags in previously received messages, so that | |
1948 | * they're ready for xpc_get_deliverable_msg(). | |
1949 | */ | |
1950 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1951 | ||
1952 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; | |
1953 | ||
1954 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, " | |
1955 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, | |
1956 | ch->number); | |
1957 | ||
1958 | nmsgs_sent = ch_sn2->w_remote_GP.put - ch_sn2->w_local_GP.get; | |
1959 | if (nmsgs_sent > 0) { | |
1960 | dev_dbg(xpc_chan, "msgs waiting to be copied and " | |
1961 | "delivered=%d, partid=%d, channel=%d\n", | |
1962 | nmsgs_sent, ch->partid, ch->number); | |
1963 | ||
1964 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
1965 | xpc_activate_kthreads(ch, nmsgs_sent); | |
1966 | } | |
1967 | } | |
1968 | ||
1969 | xpc_msgqueue_deref(ch); | |
1970 | } | |
1971 | ||
e17d416b DN |
1972 | static struct xpc_msg * |
1973 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) | |
1974 | { | |
1975 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1976 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
a812dcc3 DN |
1977 | unsigned long remote_msg_pa; |
1978 | struct xpc_msg *msg; | |
1979 | u32 msg_index; | |
1980 | u32 nmsgs; | |
e17d416b DN |
1981 | u64 msg_offset; |
1982 | enum xp_retval ret; | |
1983 | ||
a47d5dac | 1984 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1985 | /* we were interrupted by a signal */ |
1986 | return NULL; | |
1987 | } | |
1988 | ||
a47d5dac | 1989 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1990 | |
1991 | /* pull as many messages as are ready and able to be pulled */ | |
1992 | ||
a47d5dac | 1993 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1994 | |
a47d5dac DN |
1995 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1996 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1997 | if (msg_index + nmsgs > ch->remote_nentries) { |
1998 | /* ignore the ones that wrap the msg queue for now */ | |
1999 | nmsgs = ch->remote_nentries - msg_index; | |
2000 | } | |
2001 | ||
2002 | msg_offset = msg_index * ch->msg_size; | |
2003 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + msg_offset); | |
a812dcc3 | 2004 | remote_msg_pa = ch->remote_msgqueue_pa + msg_offset; |
e17d416b | 2005 | |
a812dcc3 | 2006 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa, |
e17d416b DN |
2007 | nmsgs * ch->msg_size); |
2008 | if (ret != xpSuccess) { | |
2009 | ||
2010 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
2011 | " msg %ld from partition %d, channel=%d, " | |
a47d5dac | 2012 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
2013 | ch->partid, ch->number, ret); |
2014 | ||
2015 | XPC_DEACTIVATE_PARTITION(part, ret); | |
2016 | ||
a47d5dac | 2017 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
2018 | return NULL; |
2019 | } | |
2020 | ||
a47d5dac | 2021 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
2022 | } |
2023 | ||
a47d5dac | 2024 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
2025 | |
2026 | /* return the message we were looking for */ | |
2027 | msg_offset = (get % ch->remote_nentries) * ch->msg_size; | |
2028 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + msg_offset); | |
2029 | ||
2030 | return msg; | |
2031 | } | |
2032 | ||
a47d5dac DN |
2033 | static int |
2034 | xpc_n_of_deliverable_msgs_sn2(struct xpc_channel *ch) | |
2035 | { | |
2036 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
2037 | } | |
2038 | ||
e17d416b DN |
2039 | /* |
2040 | * Get a message to be delivered. | |
2041 | */ | |
2042 | static struct xpc_msg * | |
2043 | xpc_get_deliverable_msg_sn2(struct xpc_channel *ch) | |
2044 | { | |
a47d5dac | 2045 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
e17d416b DN |
2046 | struct xpc_msg *msg = NULL; |
2047 | s64 get; | |
2048 | ||
2049 | do { | |
2050 | if (ch->flags & XPC_C_DISCONNECTING) | |
2051 | break; | |
2052 | ||
a47d5dac | 2053 | get = ch_sn2->w_local_GP.get; |
e17d416b | 2054 | rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 2055 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
2056 | break; |
2057 | ||
2058 | /* There are messages waiting to be pulled and delivered. | |
2059 | * We need to try to secure one for ourselves. We'll do this | |
2060 | * by trying to increment w_local_GP.get and hope that no one | |
2061 | * else beats us to it. If they do, we'll we'll simply have | |
2062 | * to try again for the next one. | |
2063 | */ | |
2064 | ||
a47d5dac | 2065 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
2066 | /* we got the entry referenced by get */ |
2067 | ||
2068 | dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, " | |
2069 | "partid=%d, channel=%d\n", get + 1, | |
2070 | ch->partid, ch->number); | |
2071 | ||
2072 | /* pull the message from the remote partition */ | |
2073 | ||
2074 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
2075 | ||
2076 | DBUG_ON(msg != NULL && msg->number != get); | |
2077 | DBUG_ON(msg != NULL && (msg->flags & XPC_M_DONE)); | |
2078 | DBUG_ON(msg != NULL && !(msg->flags & XPC_M_READY)); | |
2079 | ||
2080 | break; | |
2081 | } | |
2082 | ||
2083 | } while (1); | |
2084 | ||
2085 | return msg; | |
2086 | } | |
2087 | ||
33ba3c77 DN |
2088 | /* |
2089 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
2090 | * the local message queue's Put value and then send a chctl msgrequest to the |
2091 | * recipient partition. | |
33ba3c77 DN |
2092 | */ |
2093 | static void | |
2094 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
2095 | { | |
a47d5dac | 2096 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2097 | struct xpc_msg *msg; |
2098 | s64 put = initial_put + 1; | |
7fb5e59d | 2099 | int send_msgrequest = 0; |
33ba3c77 DN |
2100 | |
2101 | while (1) { | |
2102 | ||
2103 | while (1) { | |
a47d5dac | 2104 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
2105 | break; |
2106 | ||
2107 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
2108 | (put % ch->local_nentries) * | |
2109 | ch->msg_size); | |
2110 | ||
2111 | if (!(msg->flags & XPC_M_READY)) | |
2112 | break; | |
2113 | ||
2114 | put++; | |
2115 | } | |
2116 | ||
2117 | if (put == initial_put) { | |
2118 | /* nothing's changed */ | |
2119 | break; | |
2120 | } | |
2121 | ||
a47d5dac | 2122 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2123 | initial_put) { |
2124 | /* someone else beat us to it */ | |
a47d5dac | 2125 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2126 | break; |
2127 | } | |
2128 | ||
2129 | /* we just set the new value of local_GP->put */ | |
2130 | ||
2131 | dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, " | |
2132 | "channel=%d\n", put, ch->partid, ch->number); | |
2133 | ||
7fb5e59d | 2134 | send_msgrequest = 1; |
33ba3c77 DN |
2135 | |
2136 | /* | |
2137 | * We need to ensure that the message referenced by | |
2138 | * local_GP->put is not XPC_M_READY or that local_GP->put | |
2139 | * equals w_local_GP.put, so we'll go have a look. | |
2140 | */ | |
2141 | initial_put = put; | |
2142 | } | |
2143 | ||
7fb5e59d DN |
2144 | if (send_msgrequest) |
2145 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2146 | } |
2147 | ||
2148 | /* | |
2149 | * Allocate an entry for a message from the message queue associated with the | |
2150 | * specified channel. | |
2151 | */ | |
2152 | static enum xp_retval | |
2153 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
2154 | struct xpc_msg **address_of_msg) | |
2155 | { | |
a47d5dac | 2156 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2157 | struct xpc_msg *msg; |
2158 | enum xp_retval ret; | |
2159 | s64 put; | |
2160 | ||
33ba3c77 DN |
2161 | /* |
2162 | * Get the next available message entry from the local message queue. | |
2163 | * If none are available, we'll make sure that we grab the latest | |
2164 | * GP values. | |
2165 | */ | |
2166 | ret = xpTimeout; | |
2167 | ||
2168 | while (1) { | |
2169 | ||
a47d5dac | 2170 | put = ch_sn2->w_local_GP.put; |
33ba3c77 | 2171 | rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2172 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2173 | |
2174 | /* There are available message entries. We need to try | |
2175 | * to secure one for ourselves. We'll do this by trying | |
2176 | * to increment w_local_GP.put as long as someone else | |
2177 | * doesn't beat us to it. If they do, we'll have to | |
2178 | * try again. | |
2179 | */ | |
a47d5dac DN |
2180 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2181 | put) { | |
33ba3c77 DN |
2182 | /* we got the entry referenced by put */ |
2183 | break; | |
2184 | } | |
2185 | continue; /* try again */ | |
2186 | } | |
2187 | ||
2188 | /* | |
2189 | * There aren't any available msg entries at this time. | |
2190 | * | |
2191 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2192 | * we set a timeout in case the other side is not sending |
2193 | * completion interrupts. This lets us fake a notify IRQ | |
2194 | * that will cause the notify IRQ handler to fetch the latest | |
2195 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2196 | */ |
2197 | if (ret == xpTimeout) | |
7fb5e59d | 2198 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2199 | |
97bf1aa1 | 2200 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2201 | return xpNoWait; |
33ba3c77 DN |
2202 | |
2203 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2204 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2205 | return ret; |
33ba3c77 DN |
2206 | } |
2207 | ||
2208 | /* get the message's address and initialize it */ | |
2209 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
2210 | (put % ch->local_nentries) * ch->msg_size); | |
2211 | ||
2212 | DBUG_ON(msg->flags != 0); | |
2213 | msg->number = put; | |
2214 | ||
2215 | dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, " | |
2216 | "msg_number=%ld, partid=%d, channel=%d\n", put + 1, | |
2217 | (void *)msg, msg->number, ch->partid, ch->number); | |
2218 | ||
2219 | *address_of_msg = msg; | |
33ba3c77 DN |
2220 | return xpSuccess; |
2221 | } | |
2222 | ||
2223 | /* | |
2224 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2225 | * local message queue's Put value and sends a chctl msgrequest to the |
2226 | * partition the message is being sent to. | |
33ba3c77 DN |
2227 | */ |
2228 | static enum xp_retval | |
97bf1aa1 DN |
2229 | xpc_send_msg_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2230 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2231 | void *key) | |
33ba3c77 DN |
2232 | { |
2233 | enum xp_retval ret = xpSuccess; | |
97bf1aa1 | 2234 | struct xpc_msg *msg = msg; |
33ba3c77 | 2235 | struct xpc_notify *notify = notify; |
97bf1aa1 DN |
2236 | s64 msg_number; |
2237 | s64 put; | |
33ba3c77 DN |
2238 | |
2239 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 DN |
2240 | |
2241 | if (XPC_MSG_SIZE(payload_size) > ch->msg_size) | |
2242 | return xpPayloadTooBig; | |
2243 | ||
2244 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2245 | |
2246 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2247 | ret = ch->reason; |
2248 | goto out_1; | |
2249 | } | |
2250 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2251 | ret = xpNotConnected; | |
2252 | goto out_1; | |
33ba3c77 DN |
2253 | } |
2254 | ||
97bf1aa1 DN |
2255 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2256 | if (ret != xpSuccess) | |
2257 | goto out_1; | |
2258 | ||
2259 | msg_number = msg->number; | |
2260 | ||
33ba3c77 DN |
2261 | if (notify_type != 0) { |
2262 | /* | |
2263 | * Tell the remote side to send an ACK interrupt when the | |
2264 | * message has been delivered. | |
2265 | */ | |
2266 | msg->flags |= XPC_M_INTERRUPT; | |
2267 | ||
2268 | atomic_inc(&ch->n_to_notify); | |
2269 | ||
2270 | notify = &ch->notify_queue[msg_number % ch->local_nentries]; | |
2271 | notify->func = func; | |
2272 | notify->key = key; | |
2273 | notify->type = notify_type; | |
2274 | ||
ea57f80c | 2275 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2276 | |
2277 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2278 | /* | |
2279 | * An error occurred between our last error check and | |
2280 | * this one. We will try to clear the type field from | |
2281 | * the notify entry. If we succeed then | |
2282 | * xpc_disconnect_channel() didn't already process | |
2283 | * the notify entry. | |
2284 | */ | |
2285 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2286 | notify_type) { | |
2287 | atomic_dec(&ch->n_to_notify); | |
2288 | ret = ch->reason; | |
2289 | } | |
97bf1aa1 | 2290 | goto out_1; |
33ba3c77 DN |
2291 | } |
2292 | } | |
2293 | ||
97bf1aa1 DN |
2294 | memcpy(&msg->payload, payload, payload_size); |
2295 | ||
33ba3c77 DN |
2296 | msg->flags |= XPC_M_READY; |
2297 | ||
2298 | /* | |
2299 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2300 | * load of local_GP->put. |
33ba3c77 DN |
2301 | */ |
2302 | mb(); | |
2303 | ||
2304 | /* see if the message is next in line to be sent, if so send it */ | |
2305 | ||
a47d5dac | 2306 | put = ch->sn.sn2.local_GP->put; |
33ba3c77 DN |
2307 | if (put == msg_number) |
2308 | xpc_send_msgs_sn2(ch, put); | |
2309 | ||
97bf1aa1 | 2310 | out_1: |
33ba3c77 DN |
2311 | xpc_msgqueue_deref(ch); |
2312 | return ret; | |
2313 | } | |
2314 | ||
2315 | /* | |
2316 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2317 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2318 | * send a chctl msgrequest to the message sender's partition. |
33ba3c77 DN |
2319 | */ |
2320 | static void | |
2321 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2322 | { | |
a47d5dac | 2323 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2324 | struct xpc_msg *msg; |
2325 | s64 get = initial_get + 1; | |
7fb5e59d | 2326 | int send_msgrequest = 0; |
33ba3c77 DN |
2327 | |
2328 | while (1) { | |
2329 | ||
2330 | while (1) { | |
a47d5dac | 2331 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2332 | break; |
2333 | ||
2334 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + | |
2335 | (get % ch->remote_nentries) * | |
2336 | ch->msg_size); | |
2337 | ||
2338 | if (!(msg->flags & XPC_M_DONE)) | |
2339 | break; | |
2340 | ||
2341 | msg_flags |= msg->flags; | |
2342 | get++; | |
2343 | } | |
2344 | ||
2345 | if (get == initial_get) { | |
2346 | /* nothing's changed */ | |
2347 | break; | |
2348 | } | |
2349 | ||
a47d5dac | 2350 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2351 | initial_get) { |
2352 | /* someone else beat us to it */ | |
a47d5dac | 2353 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2354 | break; |
2355 | } | |
2356 | ||
2357 | /* we just set the new value of local_GP->get */ | |
2358 | ||
2359 | dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, " | |
2360 | "channel=%d\n", get, ch->partid, ch->number); | |
2361 | ||
7fb5e59d | 2362 | send_msgrequest = (msg_flags & XPC_M_INTERRUPT); |
33ba3c77 DN |
2363 | |
2364 | /* | |
2365 | * We need to ensure that the message referenced by | |
2366 | * local_GP->get is not XPC_M_DONE or that local_GP->get | |
2367 | * equals w_local_GP.get, so we'll go have a look. | |
2368 | */ | |
2369 | initial_get = get; | |
2370 | } | |
2371 | ||
7fb5e59d DN |
2372 | if (send_msgrequest) |
2373 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2374 | } |
2375 | ||
2376 | static void | |
2377 | xpc_received_msg_sn2(struct xpc_channel *ch, struct xpc_msg *msg) | |
2378 | { | |
2379 | s64 get; | |
2380 | s64 msg_number = msg->number; | |
2381 | ||
2382 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n", | |
2383 | (void *)msg, msg_number, ch->partid, ch->number); | |
2384 | ||
2385 | DBUG_ON((((u64)msg - (u64)ch->remote_msgqueue) / ch->msg_size) != | |
2386 | msg_number % ch->remote_nentries); | |
2387 | DBUG_ON(msg->flags & XPC_M_DONE); | |
2388 | ||
2389 | msg->flags |= XPC_M_DONE; | |
2390 | ||
2391 | /* | |
2392 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2393 | * load of local_GP->get. |
33ba3c77 DN |
2394 | */ |
2395 | mb(); | |
2396 | ||
2397 | /* | |
2398 | * See if this message is next in line to be acknowledged as having | |
2399 | * been delivered. | |
2400 | */ | |
a47d5dac | 2401 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2402 | if (get == msg_number) |
2403 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2404 | } | |
2405 | ||
6e41017a | 2406 | int |
94bd2708 DN |
2407 | xpc_init_sn2(void) |
2408 | { | |
6e41017a | 2409 | int ret; |
ee6665e3 | 2410 | size_t buf_size; |
6e41017a | 2411 | |
261f3b49 | 2412 | xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2; |
94bd2708 | 2413 | xpc_rsvd_page_init = xpc_rsvd_page_init_sn2; |
33ba3c77 DN |
2414 | xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; |
2415 | xpc_offline_heartbeat = xpc_offline_heartbeat_sn2; | |
2416 | xpc_online_heartbeat = xpc_online_heartbeat_sn2; | |
2417 | xpc_heartbeat_init = xpc_heartbeat_init_sn2; | |
2418 | xpc_heartbeat_exit = xpc_heartbeat_exit_sn2; | |
2419 | xpc_check_remote_hb = xpc_check_remote_hb_sn2; | |
2420 | ||
a47d5dac DN |
2421 | xpc_request_partition_activation = xpc_request_partition_activation_sn2; |
2422 | xpc_request_partition_reactivation = | |
2423 | xpc_request_partition_reactivation_sn2; | |
2424 | xpc_request_partition_deactivation = | |
2425 | xpc_request_partition_deactivation_sn2; | |
2426 | xpc_cancel_partition_deactivation_request = | |
2427 | xpc_cancel_partition_deactivation_request_sn2; | |
2428 | ||
6e41017a | 2429 | xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2; |
e17d416b DN |
2430 | xpc_setup_infrastructure = xpc_setup_infrastructure_sn2; |
2431 | xpc_teardown_infrastructure = xpc_teardown_infrastructure_sn2; | |
2432 | xpc_make_first_contact = xpc_make_first_contact_sn2; | |
7fb5e59d | 2433 | xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2; |
185c3a1b DN |
2434 | xpc_allocate_msgqueues = xpc_allocate_msgqueues_sn2; |
2435 | xpc_free_msgqueues = xpc_free_msgqueues_sn2; | |
a47d5dac | 2436 | xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2; |
7fb5e59d | 2437 | xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2; |
a47d5dac | 2438 | xpc_n_of_deliverable_msgs = xpc_n_of_deliverable_msgs_sn2; |
e17d416b | 2439 | xpc_get_deliverable_msg = xpc_get_deliverable_msg_sn2; |
33ba3c77 | 2440 | |
a47d5dac | 2441 | xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2; |
33ba3c77 | 2442 | xpc_partition_engaged = xpc_partition_engaged_sn2; |
a47d5dac DN |
2443 | xpc_any_partition_engaged = xpc_any_partition_engaged_sn2; |
2444 | xpc_indicate_partition_disengaged = | |
2445 | xpc_indicate_partition_disengaged_sn2; | |
2446 | xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2; | |
2447 | ||
7fb5e59d DN |
2448 | xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2; |
2449 | xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2; | |
2450 | xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2; | |
2451 | xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2; | |
33ba3c77 | 2452 | |
33ba3c77 DN |
2453 | xpc_send_msg = xpc_send_msg_sn2; |
2454 | xpc_received_msg = xpc_received_msg_sn2; | |
6e41017a | 2455 | |
ee6665e3 DN |
2456 | buf_size = max(XPC_RP_VARS_SIZE, |
2457 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2458 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2459 | GFP_KERNEL, | |
2460 | &xpc_remote_copy_buffer_base_sn2); | |
2461 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2462 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2463 | return -ENOMEM; | |
2464 | } | |
2465 | ||
c39838ce | 2466 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2467 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2468 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2469 | |
2470 | /* | |
2471 | * This is safe to do before the xpc_hb_checker thread has started | |
2472 | * because the handler releases a wait queue. If an interrupt is | |
2473 | * received before the thread is waiting, it will not go to sleep, | |
2474 | * but rather immediately process the interrupt. | |
2475 | */ | |
2476 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2477 | "xpc hb", NULL); | |
2478 | if (ret != 0) { | |
2479 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2480 | "errno=%d\n", -ret); | |
2481 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2482 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2483 | } |
2484 | return ret; | |
94bd2708 DN |
2485 | } |
2486 | ||
2487 | void | |
2488 | xpc_exit_sn2(void) | |
2489 | { | |
6e41017a DN |
2490 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2491 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2492 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2493 | } |