]>
Commit | Line | Data |
---|---|---|
94bd2708 DN |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | |
7 | */ | |
8 | ||
9 | /* | |
10 | * Cross Partition Communication (XPC) sn2-based functions. | |
11 | * | |
12 | * Architecture specific implementation of common functions. | |
13 | * | |
14 | */ | |
15 | ||
16 | #include <linux/kernel.h> | |
e17d416b | 17 | #include <linux/delay.h> |
94bd2708 DN |
18 | #include <asm/uncached.h> |
19 | #include <asm/sn/sn_sal.h> | |
20 | #include "xpc.h" | |
21 | ||
ee6665e3 DN |
22 | /* |
23 | * Define the number of u64s required to represent all the C-brick nasids | |
24 | * as a bitmap. The cross-partition kernel modules deal only with | |
25 | * C-brick nasids, thus the need for bitmaps which don't account for | |
26 | * odd-numbered (non C-brick) nasids. | |
27 | */ | |
28 | #define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2) | |
29 | #define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8) | |
30 | #define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64) | |
31 | ||
32 | /* | |
33 | * Memory for XPC's amo variables is allocated by the MSPEC driver. These | |
34 | * pages are located in the lowest granule. The lowest granule uses 4k pages | |
35 | * for cached references and an alternate TLB handler to never provide a | |
36 | * cacheable mapping for the entire region. This will prevent speculative | |
37 | * reading of cached copies of our lines from being issued which will cause | |
38 | * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64 | |
39 | * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of | |
40 | * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify | |
41 | * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote | |
42 | * partitions (i.e., XPCs) consider themselves currently engaged with the | |
43 | * local XPC and 1 amo variable to request partition deactivation. | |
44 | */ | |
45 | #define XPC_NOTIFY_IRQ_AMOS_SN2 0 | |
46 | #define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \ | |
47 | XP_MAX_NPARTITIONS_SN2) | |
48 | #define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \ | |
49 | XP_NASID_MASK_WORDS_SN2) | |
50 | #define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1) | |
51 | ||
52 | /* | |
53 | * Buffer used to store a local copy of portions of a remote partition's | |
54 | * reserved page (either its header and part_nasids mask, or its vars). | |
55 | */ | |
56 | static char *xpc_remote_copy_buffer_sn2; | |
57 | static void *xpc_remote_copy_buffer_base_sn2; | |
58 | ||
8e85c23e DN |
59 | static struct xpc_vars_sn2 *xpc_vars_sn2; |
60 | static struct xpc_vars_part_sn2 *xpc_vars_part_sn2; | |
94bd2708 | 61 | |
6e41017a | 62 | /* SH_IPI_ACCESS shub register value on startup */ |
8e85c23e DN |
63 | static u64 xpc_sh1_IPI_access_sn2; |
64 | static u64 xpc_sh2_IPI_access0_sn2; | |
65 | static u64 xpc_sh2_IPI_access1_sn2; | |
66 | static u64 xpc_sh2_IPI_access2_sn2; | |
67 | static u64 xpc_sh2_IPI_access3_sn2; | |
6e41017a DN |
68 | |
69 | /* | |
70 | * Change protections to allow IPI operations. | |
71 | */ | |
72 | static void | |
73 | xpc_allow_IPI_ops_sn2(void) | |
74 | { | |
75 | int node; | |
76 | int nasid; | |
77 | ||
ea57f80c | 78 | /* !!! The following should get moved into SAL. */ |
6e41017a | 79 | if (is_shub2()) { |
8e85c23e | 80 | xpc_sh2_IPI_access0_sn2 = |
6e41017a | 81 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0)); |
8e85c23e | 82 | xpc_sh2_IPI_access1_sn2 = |
6e41017a | 83 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1)); |
8e85c23e | 84 | xpc_sh2_IPI_access2_sn2 = |
6e41017a | 85 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2)); |
8e85c23e | 86 | xpc_sh2_IPI_access3_sn2 = |
6e41017a DN |
87 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3)); |
88 | ||
89 | for_each_online_node(node) { | |
90 | nasid = cnodeid_to_nasid(node); | |
91 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
92 | -1UL); | |
93 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), | |
94 | -1UL); | |
95 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), | |
96 | -1UL); | |
97 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), | |
98 | -1UL); | |
99 | } | |
100 | } else { | |
8e85c23e | 101 | xpc_sh1_IPI_access_sn2 = |
6e41017a DN |
102 | (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS)); |
103 | ||
104 | for_each_online_node(node) { | |
105 | nasid = cnodeid_to_nasid(node); | |
106 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
107 | -1UL); | |
108 | } | |
109 | } | |
110 | } | |
111 | ||
112 | /* | |
113 | * Restrict protections to disallow IPI operations. | |
114 | */ | |
115 | static void | |
116 | xpc_disallow_IPI_ops_sn2(void) | |
117 | { | |
118 | int node; | |
119 | int nasid; | |
120 | ||
ea57f80c | 121 | /* !!! The following should get moved into SAL. */ |
6e41017a DN |
122 | if (is_shub2()) { |
123 | for_each_online_node(node) { | |
124 | nasid = cnodeid_to_nasid(node); | |
125 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0), | |
8e85c23e | 126 | xpc_sh2_IPI_access0_sn2); |
6e41017a | 127 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1), |
8e85c23e | 128 | xpc_sh2_IPI_access1_sn2); |
6e41017a | 129 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2), |
8e85c23e | 130 | xpc_sh2_IPI_access2_sn2); |
6e41017a | 131 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3), |
8e85c23e | 132 | xpc_sh2_IPI_access3_sn2); |
6e41017a DN |
133 | } |
134 | } else { | |
135 | for_each_online_node(node) { | |
136 | nasid = cnodeid_to_nasid(node); | |
137 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS), | |
8e85c23e | 138 | xpc_sh1_IPI_access_sn2); |
6e41017a DN |
139 | } |
140 | } | |
141 | } | |
142 | ||
33ba3c77 | 143 | /* |
7fb5e59d DN |
144 | * The following set of functions are used for the sending and receiving of |
145 | * IRQs (also known as IPIs). There are two flavors of IRQs, one that is | |
146 | * associated with partition activity (SGI_XPC_ACTIVATE) and the other that | |
147 | * is associated with channel activity (SGI_XPC_NOTIFY). | |
33ba3c77 DN |
148 | */ |
149 | ||
150 | static u64 | |
c39838ce | 151 | xpc_receive_IRQ_amo_sn2(struct amo *amo) |
33ba3c77 DN |
152 | { |
153 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR); | |
154 | } | |
155 | ||
156 | static enum xp_retval | |
c39838ce DN |
157 | xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid, |
158 | int vector) | |
33ba3c77 DN |
159 | { |
160 | int ret = 0; | |
161 | unsigned long irq_flags; | |
162 | ||
163 | local_irq_save(irq_flags); | |
164 | ||
165 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag); | |
166 | sn_send_IPI_phys(nasid, phys_cpuid, vector, 0); | |
167 | ||
168 | /* | |
169 | * We must always use the nofault function regardless of whether we | |
170 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
171 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 172 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
173 | */ |
174 | ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable), | |
175 | xp_nofault_PIOR_target)); | |
176 | ||
177 | local_irq_restore(irq_flags); | |
178 | ||
179 | return ((ret == 0) ? xpSuccess : xpPioReadError); | |
180 | } | |
181 | ||
c39838ce | 182 | static struct amo * |
7fb5e59d | 183 | xpc_init_IRQ_amo_sn2(int index) |
33ba3c77 | 184 | { |
8e85c23e | 185 | struct amo *amo = xpc_vars_sn2->amos_page + index; |
33ba3c77 | 186 | |
c39838ce | 187 | (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */ |
33ba3c77 DN |
188 | return amo; |
189 | } | |
190 | ||
191 | /* | |
7fb5e59d | 192 | * Functions associated with SGI_XPC_ACTIVATE IRQ. |
33ba3c77 DN |
193 | */ |
194 | ||
6e41017a DN |
195 | /* |
196 | * Notify the heartbeat check thread that an activate IRQ has been received. | |
197 | */ | |
198 | static irqreturn_t | |
199 | xpc_handle_activate_IRQ_sn2(int irq, void *dev_id) | |
200 | { | |
201 | atomic_inc(&xpc_activate_IRQ_rcvd); | |
202 | wake_up_interruptible(&xpc_activate_IRQ_wq); | |
203 | return IRQ_HANDLED; | |
204 | } | |
205 | ||
33ba3c77 | 206 | /* |
c39838ce | 207 | * Flag the appropriate amo variable and send an IRQ to the specified node. |
33ba3c77 DN |
208 | */ |
209 | static void | |
7fb5e59d DN |
210 | xpc_send_activate_IRQ_sn2(u64 amos_page_pa, int from_nasid, int to_nasid, |
211 | int to_phys_cpuid) | |
33ba3c77 DN |
212 | { |
213 | int w_index = XPC_NASID_W_INDEX(from_nasid); | |
214 | int b_index = XPC_NASID_B_INDEX(from_nasid); | |
c39838ce | 215 | struct amo *amos = (struct amo *)__va(amos_page_pa + |
ee6665e3 | 216 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 217 | sizeof(struct amo))); |
33ba3c77 | 218 | |
7fb5e59d | 219 | (void)xpc_send_IRQ_sn2(&amos[w_index], (1UL << b_index), to_nasid, |
33ba3c77 DN |
220 | to_phys_cpuid, SGI_XPC_ACTIVATE); |
221 | } | |
222 | ||
223 | static void | |
7fb5e59d | 224 | xpc_send_local_activate_IRQ_sn2(int from_nasid) |
33ba3c77 DN |
225 | { |
226 | int w_index = XPC_NASID_W_INDEX(from_nasid); | |
227 | int b_index = XPC_NASID_B_INDEX(from_nasid); | |
8e85c23e | 228 | struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa + |
ee6665e3 | 229 | (XPC_ACTIVATE_IRQ_AMOS_SN2 * |
c39838ce | 230 | sizeof(struct amo))); |
33ba3c77 DN |
231 | |
232 | /* fake the sending and receipt of an activate IRQ from remote nasid */ | |
233 | FETCHOP_STORE_OP(TO_AMO((u64)&amos[w_index].variable), FETCHOP_OR, | |
234 | (1UL << b_index)); | |
6e41017a DN |
235 | atomic_inc(&xpc_activate_IRQ_rcvd); |
236 | wake_up_interruptible(&xpc_activate_IRQ_wq); | |
33ba3c77 DN |
237 | } |
238 | ||
a47d5dac | 239 | /* |
7fb5e59d | 240 | * Functions associated with SGI_XPC_NOTIFY IRQ. |
a47d5dac | 241 | */ |
33ba3c77 | 242 | |
a47d5dac | 243 | /* |
7fb5e59d | 244 | * Check to see if any chctl flags were sent from the specified partition. |
a47d5dac | 245 | */ |
33ba3c77 | 246 | static void |
7fb5e59d | 247 | xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part) |
33ba3c77 | 248 | { |
7fb5e59d | 249 | union xpc_channel_ctl_flags chctl; |
a47d5dac | 250 | unsigned long irq_flags; |
33ba3c77 | 251 | |
7fb5e59d DN |
252 | chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2. |
253 | local_chctl_amo_va); | |
254 | if (chctl.all_flags == 0) | |
a47d5dac DN |
255 | return; |
256 | ||
7fb5e59d DN |
257 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
258 | part->chctl.all_flags |= chctl.all_flags; | |
259 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); | |
a47d5dac | 260 | |
7fb5e59d DN |
261 | dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags=" |
262 | "0x%lx\n", XPC_PARTID(part), chctl.all_flags); | |
a47d5dac DN |
263 | |
264 | xpc_wakeup_channel_mgr(part); | |
33ba3c77 DN |
265 | } |
266 | ||
a47d5dac DN |
267 | /* |
268 | * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified | |
269 | * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more | |
c39838ce | 270 | * than one partition, we use an amo structure per partition to indicate |
7fb5e59d | 271 | * whether a partition has sent an IRQ or not. If it has, then wake up the |
a47d5dac DN |
272 | * associated kthread to handle it. |
273 | * | |
7fb5e59d | 274 | * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC |
a47d5dac DN |
275 | * running on other partitions. |
276 | * | |
277 | * Noteworthy Arguments: | |
278 | * | |
279 | * irq - Interrupt ReQuest number. NOT USED. | |
280 | * | |
7fb5e59d | 281 | * dev_id - partid of IRQ's potential sender. |
a47d5dac DN |
282 | */ |
283 | static irqreturn_t | |
284 | xpc_handle_notify_IRQ_sn2(int irq, void *dev_id) | |
33ba3c77 | 285 | { |
a47d5dac DN |
286 | short partid = (short)(u64)dev_id; |
287 | struct xpc_partition *part = &xpc_partitions[partid]; | |
288 | ||
289 | DBUG_ON(partid < 0 || partid >= xp_max_npartitions); | |
290 | ||
291 | if (xpc_part_ref(part)) { | |
7fb5e59d | 292 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
293 | |
294 | xpc_part_deref(part); | |
295 | } | |
296 | return IRQ_HANDLED; | |
33ba3c77 DN |
297 | } |
298 | ||
299 | /* | |
7fb5e59d DN |
300 | * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor |
301 | * because the write to their associated amo variable completed after the IRQ | |
a47d5dac | 302 | * was received. |
33ba3c77 | 303 | */ |
a47d5dac | 304 | static void |
7fb5e59d | 305 | xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part) |
a47d5dac DN |
306 | { |
307 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
308 | ||
309 | if (xpc_part_ref(part)) { | |
7fb5e59d | 310 | xpc_check_for_sent_chctl_flags_sn2(part); |
a47d5dac DN |
311 | |
312 | part_sn2->dropped_notify_IRQ_timer.expires = jiffies + | |
7fb5e59d | 313 | XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
a47d5dac DN |
314 | add_timer(&part_sn2->dropped_notify_IRQ_timer); |
315 | xpc_part_deref(part); | |
316 | } | |
317 | } | |
33ba3c77 DN |
318 | |
319 | /* | |
7fb5e59d | 320 | * Send a notify IRQ to the remote partition that is associated with the |
33ba3c77 DN |
321 | * specified channel. |
322 | */ | |
323 | static void | |
7fb5e59d DN |
324 | xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
325 | char *chctl_flag_string, unsigned long *irq_flags) | |
33ba3c77 DN |
326 | { |
327 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 328 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
7fb5e59d | 329 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 DN |
330 | enum xp_retval ret; |
331 | ||
332 | if (likely(part->act_state != XPC_P_DEACTIVATING)) { | |
7fb5e59d DN |
333 | chctl.flags[ch->number] = chctl_flag; |
334 | ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va, | |
335 | chctl.all_flags, | |
336 | part_sn2->notify_IRQ_nasid, | |
337 | part_sn2->notify_IRQ_phys_cpuid, | |
33ba3c77 DN |
338 | SGI_XPC_NOTIFY); |
339 | dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n", | |
7fb5e59d | 340 | chctl_flag_string, ch->partid, ch->number, ret); |
33ba3c77 DN |
341 | if (unlikely(ret != xpSuccess)) { |
342 | if (irq_flags != NULL) | |
343 | spin_unlock_irqrestore(&ch->lock, *irq_flags); | |
344 | XPC_DEACTIVATE_PARTITION(part, ret); | |
345 | if (irq_flags != NULL) | |
346 | spin_lock_irqsave(&ch->lock, *irq_flags); | |
347 | } | |
348 | } | |
349 | } | |
350 | ||
7fb5e59d DN |
351 | #define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \ |
352 | xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f) | |
33ba3c77 DN |
353 | |
354 | /* | |
355 | * Make it look like the remote partition, which is associated with the | |
7fb5e59d DN |
356 | * specified channel, sent us a notify IRQ. This faked IRQ will be handled |
357 | * by xpc_check_for_dropped_notify_IRQ_sn2(). | |
33ba3c77 DN |
358 | */ |
359 | static void | |
7fb5e59d DN |
360 | xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag, |
361 | char *chctl_flag_string) | |
33ba3c77 DN |
362 | { |
363 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
7fb5e59d | 364 | union xpc_channel_ctl_flags chctl = { 0 }; |
33ba3c77 | 365 | |
7fb5e59d DN |
366 | chctl.flags[ch->number] = chctl_flag; |
367 | FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va-> | |
368 | variable), FETCHOP_OR, chctl.all_flags); | |
33ba3c77 | 369 | dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n", |
7fb5e59d | 370 | chctl_flag_string, ch->partid, ch->number); |
33ba3c77 DN |
371 | } |
372 | ||
7fb5e59d DN |
373 | #define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \ |
374 | xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f) | |
33ba3c77 DN |
375 | |
376 | static void | |
7fb5e59d DN |
377 | xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch, |
378 | unsigned long *irq_flags) | |
33ba3c77 DN |
379 | { |
380 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
381 | ||
382 | args->reason = ch->reason; | |
7fb5e59d | 383 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags); |
33ba3c77 DN |
384 | } |
385 | ||
386 | static void | |
7fb5e59d | 387 | xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 | 388 | { |
7fb5e59d | 389 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags); |
33ba3c77 DN |
390 | } |
391 | ||
392 | static void | |
7fb5e59d | 393 | xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 DN |
394 | { |
395 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
396 | ||
397 | args->msg_size = ch->msg_size; | |
398 | args->local_nentries = ch->local_nentries; | |
7fb5e59d | 399 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags); |
33ba3c77 DN |
400 | } |
401 | ||
402 | static void | |
7fb5e59d | 403 | xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags) |
33ba3c77 DN |
404 | { |
405 | struct xpc_openclose_args *args = ch->local_openclose_args; | |
406 | ||
407 | args->remote_nentries = ch->remote_nentries; | |
408 | args->local_nentries = ch->local_nentries; | |
409 | args->local_msgqueue_pa = __pa(ch->local_msgqueue); | |
7fb5e59d | 410 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags); |
33ba3c77 DN |
411 | } |
412 | ||
413 | static void | |
7fb5e59d | 414 | xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 415 | { |
7fb5e59d | 416 | XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL); |
33ba3c77 DN |
417 | } |
418 | ||
419 | static void | |
7fb5e59d | 420 | xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch) |
33ba3c77 | 421 | { |
7fb5e59d | 422 | XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); |
33ba3c77 DN |
423 | } |
424 | ||
425 | /* | |
426 | * This next set of functions are used to keep track of when a partition is | |
427 | * potentially engaged in accessing memory belonging to another partition. | |
428 | */ | |
429 | ||
430 | static void | |
a47d5dac | 431 | xpc_indicate_partition_engaged_sn2(struct xpc_partition *part) |
33ba3c77 DN |
432 | { |
433 | unsigned long irq_flags; | |
c39838ce | 434 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 435 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 436 | sizeof(struct amo))); |
33ba3c77 DN |
437 | |
438 | local_irq_save(irq_flags); | |
439 | ||
c39838ce | 440 | /* set bit corresponding to our partid in remote partition's amo */ |
33ba3c77 DN |
441 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
442 | (1UL << sn_partition_id)); | |
443 | /* | |
444 | * We must always use the nofault function regardless of whether we | |
445 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
446 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 447 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
448 | */ |
449 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
450 | variable), | |
451 | xp_nofault_PIOR_target)); | |
452 | ||
453 | local_irq_restore(irq_flags); | |
454 | } | |
455 | ||
456 | static void | |
a47d5dac | 457 | xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part) |
33ba3c77 | 458 | { |
a47d5dac | 459 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
33ba3c77 | 460 | unsigned long irq_flags; |
c39838ce | 461 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 462 | (XPC_ENGAGED_PARTITIONS_AMO_SN2 * |
c39838ce | 463 | sizeof(struct amo))); |
33ba3c77 DN |
464 | |
465 | local_irq_save(irq_flags); | |
466 | ||
c39838ce | 467 | /* clear bit corresponding to our partid in remote partition's amo */ |
33ba3c77 DN |
468 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
469 | ~(1UL << sn_partition_id)); | |
470 | /* | |
471 | * We must always use the nofault function regardless of whether we | |
472 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
473 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 474 | * keep sending IRQs and amos to it until the heartbeat times out. |
33ba3c77 DN |
475 | */ |
476 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
477 | variable), | |
478 | xp_nofault_PIOR_target)); | |
479 | ||
480 | local_irq_restore(irq_flags); | |
33ba3c77 | 481 | |
33ba3c77 | 482 | /* |
a47d5dac | 483 | * Send activate IRQ to get other side to see that we've cleared our |
c39838ce | 484 | * bit in their engaged partitions amo. |
33ba3c77 | 485 | */ |
7fb5e59d | 486 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
487 | cnodeid_to_nasid(0), |
488 | part_sn2->activate_IRQ_nasid, | |
489 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 DN |
490 | } |
491 | ||
a47d5dac DN |
492 | static int |
493 | xpc_partition_engaged_sn2(short partid) | |
33ba3c77 | 494 | { |
8e85c23e DN |
495 | struct amo *amo = xpc_vars_sn2->amos_page + |
496 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 497 | |
c39838ce | 498 | /* our partition's amo variable ANDed with partid mask */ |
33ba3c77 | 499 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
a47d5dac | 500 | (1UL << partid)) != 0; |
33ba3c77 DN |
501 | } |
502 | ||
a47d5dac DN |
503 | static int |
504 | xpc_any_partition_engaged_sn2(void) | |
33ba3c77 | 505 | { |
8e85c23e DN |
506 | struct amo *amo = xpc_vars_sn2->amos_page + |
507 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 508 | |
c39838ce | 509 | /* our partition's amo variable */ |
a47d5dac | 510 | return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0; |
33ba3c77 DN |
511 | } |
512 | ||
513 | static void | |
a47d5dac | 514 | xpc_assume_partition_disengaged_sn2(short partid) |
33ba3c77 | 515 | { |
8e85c23e DN |
516 | struct amo *amo = xpc_vars_sn2->amos_page + |
517 | XPC_ENGAGED_PARTITIONS_AMO_SN2; | |
33ba3c77 | 518 | |
c39838ce | 519 | /* clear bit(s) based on partid mask in our partition's amo */ |
33ba3c77 | 520 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
a47d5dac | 521 | ~(1UL << partid)); |
33ba3c77 DN |
522 | } |
523 | ||
6e41017a DN |
524 | /* original protection values for each node */ |
525 | static u64 xpc_prot_vec_sn2[MAX_NUMNODES]; | |
526 | ||
527 | /* | |
c39838ce | 528 | * Change protections to allow amo operations on non-Shub 1.1 systems. |
6e41017a DN |
529 | */ |
530 | static enum xp_retval | |
c39838ce | 531 | xpc_allow_amo_ops_sn2(struct amo *amos_page) |
6e41017a DN |
532 | { |
533 | u64 nasid_array = 0; | |
534 | int ret; | |
535 | ||
536 | /* | |
537 | * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST | |
538 | * collides with memory operations. On those systems we call | |
c39838ce | 539 | * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead. |
6e41017a DN |
540 | */ |
541 | if (!enable_shub_wars_1_1()) { | |
542 | ret = sn_change_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE, | |
543 | SN_MEMPROT_ACCESS_CLASS_1, | |
544 | &nasid_array); | |
545 | if (ret != 0) | |
546 | return xpSalError; | |
547 | } | |
548 | return xpSuccess; | |
549 | } | |
550 | ||
551 | /* | |
c39838ce | 552 | * Change protections to allow amo operations on Shub 1.1 systems. |
6e41017a DN |
553 | */ |
554 | static void | |
c39838ce | 555 | xpc_allow_amo_ops_shub_wars_1_1_sn2(void) |
6e41017a DN |
556 | { |
557 | int node; | |
558 | int nasid; | |
559 | ||
560 | if (!enable_shub_wars_1_1()) | |
561 | return; | |
562 | ||
563 | for_each_online_node(node) { | |
564 | nasid = cnodeid_to_nasid(node); | |
565 | /* save current protection values */ | |
566 | xpc_prot_vec_sn2[node] = | |
567 | (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid, | |
568 | SH1_MD_DQLP_MMR_DIR_PRIVEC0)); | |
569 | /* open up everything */ | |
570 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
571 | SH1_MD_DQLP_MMR_DIR_PRIVEC0), | |
572 | -1UL); | |
573 | HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, | |
574 | SH1_MD_DQRP_MMR_DIR_PRIVEC0), | |
575 | -1UL); | |
576 | } | |
577 | } | |
578 | ||
94bd2708 DN |
579 | static enum xp_retval |
580 | xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp) | |
581 | { | |
c39838ce | 582 | struct amo *amos_page; |
94bd2708 DN |
583 | int i; |
584 | int ret; | |
585 | ||
8e85c23e | 586 | xpc_vars_sn2 = XPC_RP_VARS(rp); |
94bd2708 | 587 | |
8e85c23e | 588 | rp->sn.vars_pa = __pa(xpc_vars_sn2); |
94bd2708 | 589 | |
e17d416b | 590 | /* vars_part array follows immediately after vars */ |
8e85c23e DN |
591 | xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) + |
592 | XPC_RP_VARS_SIZE); | |
e17d416b | 593 | |
94bd2708 | 594 | /* |
8e85c23e DN |
595 | * Before clearing xpc_vars_sn2, see if a page of amos had been |
596 | * previously allocated. If not we'll need to allocate one and set | |
597 | * permissions so that cross-partition amos are allowed. | |
94bd2708 | 598 | * |
c39838ce | 599 | * The allocated amo page needs MCA reporting to remain disabled after |
94bd2708 | 600 | * XPC has unloaded. To make this work, we keep a copy of the pointer |
8e85c23e | 601 | * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure, |
94bd2708 | 602 | * which is pointed to by the reserved page, and re-use that saved copy |
c39838ce | 603 | * on subsequent loads of XPC. This amo page is never freed, and its |
94bd2708 DN |
604 | * memory protections are never restricted. |
605 | */ | |
8e85c23e | 606 | amos_page = xpc_vars_sn2->amos_page; |
94bd2708 | 607 | if (amos_page == NULL) { |
c39838ce | 608 | amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1)); |
94bd2708 | 609 | if (amos_page == NULL) { |
c39838ce | 610 | dev_err(xpc_part, "can't allocate page of amos\n"); |
94bd2708 DN |
611 | return xpNoMemory; |
612 | } | |
613 | ||
614 | /* | |
c39838ce DN |
615 | * Open up amo-R/W to cpu. This is done on Shub 1.1 systems |
616 | * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called. | |
94bd2708 | 617 | */ |
c39838ce | 618 | ret = xpc_allow_amo_ops_sn2(amos_page); |
6e41017a | 619 | if (ret != xpSuccess) { |
c39838ce | 620 | dev_err(xpc_part, "can't allow amo operations\n"); |
6e41017a DN |
621 | uncached_free_page(__IA64_UNCACHED_OFFSET | |
622 | TO_PHYS((u64)amos_page), 1); | |
623 | return ret; | |
94bd2708 DN |
624 | } |
625 | } | |
626 | ||
8e85c23e DN |
627 | /* clear xpc_vars_sn2 */ |
628 | memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2)); | |
94bd2708 | 629 | |
8e85c23e DN |
630 | xpc_vars_sn2->version = XPC_V_VERSION; |
631 | xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0); | |
632 | xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0); | |
633 | xpc_vars_sn2->vars_part_pa = __pa(xpc_vars_part_sn2); | |
634 | xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page); | |
635 | xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */ | |
94bd2708 | 636 | |
8e85c23e DN |
637 | /* clear xpc_vars_part_sn2 */ |
638 | memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) * | |
94bd2708 DN |
639 | xp_max_npartitions); |
640 | ||
c39838ce | 641 | /* initialize the activate IRQ related amo variables */ |
ee6665e3 DN |
642 | for (i = 0; i < xpc_nasid_mask_words; i++) |
643 | (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i); | |
94bd2708 | 644 | |
c39838ce | 645 | /* initialize the engaged remote partitions related amo variables */ |
ee6665e3 DN |
646 | (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2); |
647 | (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2); | |
33ba3c77 DN |
648 | |
649 | return xpSuccess; | |
650 | } | |
651 | ||
652 | static void | |
653 | xpc_increment_heartbeat_sn2(void) | |
654 | { | |
8e85c23e | 655 | xpc_vars_sn2->heartbeat++; |
33ba3c77 DN |
656 | } |
657 | ||
658 | static void | |
659 | xpc_offline_heartbeat_sn2(void) | |
660 | { | |
661 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 662 | xpc_vars_sn2->heartbeat_offline = 1; |
33ba3c77 DN |
663 | } |
664 | ||
665 | static void | |
666 | xpc_online_heartbeat_sn2(void) | |
667 | { | |
668 | xpc_increment_heartbeat_sn2(); | |
8e85c23e | 669 | xpc_vars_sn2->heartbeat_offline = 0; |
33ba3c77 DN |
670 | } |
671 | ||
672 | static void | |
673 | xpc_heartbeat_init_sn2(void) | |
674 | { | |
8e85c23e | 675 | DBUG_ON(xpc_vars_sn2 == NULL); |
33ba3c77 | 676 | |
8e85c23e DN |
677 | bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2); |
678 | xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0]; | |
33ba3c77 DN |
679 | xpc_online_heartbeat_sn2(); |
680 | } | |
681 | ||
682 | static void | |
683 | xpc_heartbeat_exit_sn2(void) | |
684 | { | |
685 | xpc_offline_heartbeat_sn2(); | |
686 | } | |
687 | ||
688 | /* | |
689 | * At periodic intervals, scan through all active partitions and ensure | |
690 | * their heartbeat is still active. If not, the partition is deactivated. | |
691 | */ | |
692 | static void | |
693 | xpc_check_remote_hb_sn2(void) | |
694 | { | |
695 | struct xpc_vars_sn2 *remote_vars; | |
696 | struct xpc_partition *part; | |
697 | short partid; | |
698 | enum xp_retval ret; | |
699 | ||
ee6665e3 | 700 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
701 | |
702 | for (partid = 0; partid < xp_max_npartitions; partid++) { | |
703 | ||
704 | if (xpc_exiting) | |
705 | break; | |
706 | ||
707 | if (partid == sn_partition_id) | |
708 | continue; | |
709 | ||
710 | part = &xpc_partitions[partid]; | |
711 | ||
712 | if (part->act_state == XPC_P_INACTIVE || | |
713 | part->act_state == XPC_P_DEACTIVATING) { | |
714 | continue; | |
715 | } | |
716 | ||
717 | /* pull the remote_hb cache line */ | |
718 | ret = xp_remote_memcpy(remote_vars, | |
a47d5dac | 719 | (void *)part->sn.sn2.remote_vars_pa, |
33ba3c77 DN |
720 | XPC_RP_VARS_SIZE); |
721 | if (ret != xpSuccess) { | |
722 | XPC_DEACTIVATE_PARTITION(part, ret); | |
723 | continue; | |
724 | } | |
725 | ||
726 | dev_dbg(xpc_part, "partid = %d, heartbeat = %ld, last_heartbeat" | |
727 | " = %ld, heartbeat_offline = %ld, HB_mask[0] = 0x%lx\n", | |
728 | partid, remote_vars->heartbeat, part->last_heartbeat, | |
729 | remote_vars->heartbeat_offline, | |
730 | remote_vars->heartbeating_to_mask[0]); | |
731 | ||
732 | if (((remote_vars->heartbeat == part->last_heartbeat) && | |
733 | (remote_vars->heartbeat_offline == 0)) || | |
734 | !xpc_hb_allowed(sn_partition_id, | |
735 | &remote_vars->heartbeating_to_mask)) { | |
736 | ||
737 | XPC_DEACTIVATE_PARTITION(part, xpNoHeartbeat); | |
738 | continue; | |
739 | } | |
740 | ||
741 | part->last_heartbeat = remote_vars->heartbeat; | |
742 | } | |
743 | } | |
744 | ||
745 | /* | |
746 | * Get a copy of the remote partition's XPC variables from the reserved page. | |
747 | * | |
748 | * remote_vars points to a buffer that is cacheline aligned for BTE copies and | |
749 | * assumed to be of size XPC_RP_VARS_SIZE. | |
750 | */ | |
751 | static enum xp_retval | |
752 | xpc_get_remote_vars_sn2(u64 remote_vars_pa, struct xpc_vars_sn2 *remote_vars) | |
753 | { | |
754 | enum xp_retval ret; | |
755 | ||
756 | if (remote_vars_pa == 0) | |
757 | return xpVarsNotSet; | |
758 | ||
759 | /* pull over the cross partition variables */ | |
760 | ret = xp_remote_memcpy(remote_vars, (void *)remote_vars_pa, | |
761 | XPC_RP_VARS_SIZE); | |
762 | if (ret != xpSuccess) | |
763 | return ret; | |
764 | ||
765 | if (XPC_VERSION_MAJOR(remote_vars->version) != | |
766 | XPC_VERSION_MAJOR(XPC_V_VERSION)) { | |
767 | return xpBadVersion; | |
768 | } | |
94bd2708 DN |
769 | |
770 | return xpSuccess; | |
771 | } | |
772 | ||
33ba3c77 | 773 | static void |
a47d5dac DN |
774 | xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp, |
775 | u64 remote_rp_pa, int nasid) | |
33ba3c77 | 776 | { |
7fb5e59d | 777 | xpc_send_local_activate_IRQ_sn2(nasid); |
a47d5dac DN |
778 | } |
779 | ||
780 | static void | |
781 | xpc_request_partition_reactivation_sn2(struct xpc_partition *part) | |
782 | { | |
7fb5e59d | 783 | xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid); |
a47d5dac DN |
784 | } |
785 | ||
786 | static void | |
787 | xpc_request_partition_deactivation_sn2(struct xpc_partition *part) | |
788 | { | |
789 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; | |
790 | unsigned long irq_flags; | |
c39838ce | 791 | struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa + |
ee6665e3 | 792 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 793 | sizeof(struct amo))); |
a47d5dac DN |
794 | |
795 | local_irq_save(irq_flags); | |
796 | ||
c39838ce | 797 | /* set bit corresponding to our partid in remote partition's amo */ |
a47d5dac DN |
798 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, |
799 | (1UL << sn_partition_id)); | |
800 | /* | |
801 | * We must always use the nofault function regardless of whether we | |
802 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
803 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 804 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
805 | */ |
806 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
807 | variable), | |
808 | xp_nofault_PIOR_target)); | |
809 | ||
810 | local_irq_restore(irq_flags); | |
811 | ||
812 | /* | |
813 | * Send activate IRQ to get other side to see that we've set our | |
c39838ce | 814 | * bit in their deactivate request amo. |
a47d5dac | 815 | */ |
7fb5e59d | 816 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
817 | cnodeid_to_nasid(0), |
818 | part_sn2->activate_IRQ_nasid, | |
819 | part_sn2->activate_IRQ_phys_cpuid); | |
820 | } | |
821 | ||
822 | static void | |
823 | xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part) | |
824 | { | |
825 | unsigned long irq_flags; | |
c39838ce | 826 | struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa + |
ee6665e3 | 827 | (XPC_DEACTIVATE_REQUEST_AMO_SN2 * |
c39838ce | 828 | sizeof(struct amo))); |
a47d5dac DN |
829 | |
830 | local_irq_save(irq_flags); | |
831 | ||
c39838ce | 832 | /* clear bit corresponding to our partid in remote partition's amo */ |
a47d5dac DN |
833 | FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND, |
834 | ~(1UL << sn_partition_id)); | |
835 | /* | |
836 | * We must always use the nofault function regardless of whether we | |
837 | * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we | |
838 | * didn't, we'd never know that the other partition is down and would | |
c39838ce | 839 | * keep sending IRQs and amos to it until the heartbeat times out. |
a47d5dac DN |
840 | */ |
841 | (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo-> | |
842 | variable), | |
843 | xp_nofault_PIOR_target)); | |
844 | ||
845 | local_irq_restore(irq_flags); | |
846 | } | |
847 | ||
848 | static int | |
849 | xpc_partition_deactivation_requested_sn2(short partid) | |
850 | { | |
8e85c23e DN |
851 | struct amo *amo = xpc_vars_sn2->amos_page + |
852 | XPC_DEACTIVATE_REQUEST_AMO_SN2; | |
a47d5dac | 853 | |
c39838ce | 854 | /* our partition's amo variable ANDed with partid mask */ |
a47d5dac DN |
855 | return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) & |
856 | (1UL << partid)) != 0; | |
33ba3c77 DN |
857 | } |
858 | ||
859 | /* | |
860 | * Update the remote partition's info. | |
861 | */ | |
862 | static void | |
863 | xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version, | |
aaa3cd69 DN |
864 | unsigned long *remote_rp_stamp, u64 remote_rp_pa, |
865 | u64 remote_vars_pa, | |
33ba3c77 DN |
866 | struct xpc_vars_sn2 *remote_vars) |
867 | { | |
a47d5dac DN |
868 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
869 | ||
33ba3c77 DN |
870 | part->remote_rp_version = remote_rp_version; |
871 | dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n", | |
872 | part->remote_rp_version); | |
873 | ||
874 | part->remote_rp_stamp = *remote_rp_stamp; | |
aaa3cd69 DN |
875 | dev_dbg(xpc_part, " remote_rp_stamp = 0x%016lx\n", |
876 | part->remote_rp_stamp); | |
33ba3c77 DN |
877 | |
878 | part->remote_rp_pa = remote_rp_pa; | |
879 | dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa); | |
880 | ||
a47d5dac | 881 | part_sn2->remote_vars_pa = remote_vars_pa; |
33ba3c77 | 882 | dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n", |
a47d5dac | 883 | part_sn2->remote_vars_pa); |
33ba3c77 DN |
884 | |
885 | part->last_heartbeat = remote_vars->heartbeat; | |
886 | dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n", | |
887 | part->last_heartbeat); | |
888 | ||
a47d5dac | 889 | part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa; |
33ba3c77 | 890 | dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n", |
a47d5dac | 891 | part_sn2->remote_vars_part_pa); |
33ba3c77 | 892 | |
a47d5dac DN |
893 | part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid; |
894 | dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n", | |
895 | part_sn2->activate_IRQ_nasid); | |
33ba3c77 | 896 | |
a47d5dac DN |
897 | part_sn2->activate_IRQ_phys_cpuid = |
898 | remote_vars->activate_IRQ_phys_cpuid; | |
899 | dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n", | |
900 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 901 | |
a47d5dac | 902 | part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa; |
33ba3c77 | 903 | dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n", |
a47d5dac | 904 | part_sn2->remote_amos_page_pa); |
33ba3c77 | 905 | |
a47d5dac | 906 | part_sn2->remote_vars_version = remote_vars->version; |
33ba3c77 | 907 | dev_dbg(xpc_part, " remote_vars_version = 0x%x\n", |
a47d5dac | 908 | part_sn2->remote_vars_version); |
33ba3c77 DN |
909 | } |
910 | ||
911 | /* | |
7fb5e59d DN |
912 | * Prior code has determined the nasid which generated a activate IRQ. |
913 | * Inspect that nasid to determine if its partition needs to be activated | |
914 | * or deactivated. | |
33ba3c77 | 915 | * |
7fb5e59d | 916 | * A partition is considered "awaiting activation" if our partition |
33ba3c77 DN |
917 | * flags indicate it is not active and it has a heartbeat. A |
918 | * partition is considered "awaiting deactivation" if our partition | |
919 | * flags indicate it is active but it has no heartbeat or it is not | |
920 | * sending its heartbeat to us. | |
921 | * | |
922 | * To determine the heartbeat, the remote nasid must have a properly | |
923 | * initialized reserved page. | |
924 | */ | |
925 | static void | |
6e41017a | 926 | xpc_identify_activate_IRQ_req_sn2(int nasid) |
33ba3c77 DN |
927 | { |
928 | struct xpc_rsvd_page *remote_rp; | |
929 | struct xpc_vars_sn2 *remote_vars; | |
930 | u64 remote_rp_pa; | |
931 | u64 remote_vars_pa; | |
932 | int remote_rp_version; | |
933 | int reactivate = 0; | |
aaa3cd69 | 934 | unsigned long remote_rp_stamp = 0; |
33ba3c77 DN |
935 | short partid; |
936 | struct xpc_partition *part; | |
a47d5dac | 937 | struct xpc_partition_sn2 *part_sn2; |
33ba3c77 DN |
938 | enum xp_retval ret; |
939 | ||
940 | /* pull over the reserved page structure */ | |
941 | ||
ee6665e3 | 942 | remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
943 | |
944 | ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa); | |
945 | if (ret != xpSuccess) { | |
946 | dev_warn(xpc_part, "unable to get reserved page from nasid %d, " | |
947 | "which sent interrupt, reason=%d\n", nasid, ret); | |
948 | return; | |
949 | } | |
950 | ||
951 | remote_vars_pa = remote_rp->sn.vars_pa; | |
952 | remote_rp_version = remote_rp->version; | |
a47d5dac | 953 | remote_rp_stamp = remote_rp->stamp; |
33ba3c77 DN |
954 | |
955 | partid = remote_rp->SAL_partid; | |
956 | part = &xpc_partitions[partid]; | |
a47d5dac | 957 | part_sn2 = &part->sn.sn2; |
33ba3c77 DN |
958 | |
959 | /* pull over the cross partition variables */ | |
960 | ||
ee6665e3 | 961 | remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2; |
33ba3c77 DN |
962 | |
963 | ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars); | |
964 | if (ret != xpSuccess) { | |
33ba3c77 DN |
965 | dev_warn(xpc_part, "unable to get XPC variables from nasid %d, " |
966 | "which sent interrupt, reason=%d\n", nasid, ret); | |
967 | ||
968 | XPC_DEACTIVATE_PARTITION(part, ret); | |
969 | return; | |
970 | } | |
971 | ||
6e41017a | 972 | part->activate_IRQ_rcvd++; |
33ba3c77 DN |
973 | |
974 | dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = " | |
6e41017a | 975 | "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd, |
33ba3c77 DN |
976 | remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]); |
977 | ||
978 | if (xpc_partition_disengaged(part) && | |
979 | part->act_state == XPC_P_INACTIVE) { | |
980 | ||
981 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
982 | &remote_rp_stamp, remote_rp_pa, | |
983 | remote_vars_pa, remote_vars); | |
984 | ||
a47d5dac DN |
985 | if (xpc_partition_deactivation_requested_sn2(partid)) { |
986 | /* | |
987 | * Other side is waiting on us to deactivate even though | |
988 | * we already have. | |
989 | */ | |
990 | return; | |
33ba3c77 DN |
991 | } |
992 | ||
993 | xpc_activate_partition(part); | |
994 | return; | |
995 | } | |
996 | ||
997 | DBUG_ON(part->remote_rp_version == 0); | |
a47d5dac | 998 | DBUG_ON(part_sn2->remote_vars_version == 0); |
33ba3c77 | 999 | |
a47d5dac | 1000 | if (remote_rp_stamp != part->remote_rp_stamp) { |
33ba3c77 | 1001 | |
a47d5dac | 1002 | /* the other side rebooted */ |
33ba3c77 | 1003 | |
a47d5dac DN |
1004 | DBUG_ON(xpc_partition_engaged_sn2(partid)); |
1005 | DBUG_ON(xpc_partition_deactivation_requested_sn2(partid)); | |
33ba3c77 DN |
1006 | |
1007 | xpc_update_partition_info_sn2(part, remote_rp_version, | |
1008 | &remote_rp_stamp, remote_rp_pa, | |
1009 | remote_vars_pa, remote_vars); | |
1010 | reactivate = 1; | |
33ba3c77 DN |
1011 | } |
1012 | ||
a47d5dac | 1013 | if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) { |
33ba3c77 DN |
1014 | /* still waiting on other side to disengage from us */ |
1015 | return; | |
1016 | } | |
1017 | ||
a47d5dac | 1018 | if (reactivate) |
33ba3c77 | 1019 | XPC_DEACTIVATE_PARTITION(part, xpReactivating); |
a47d5dac | 1020 | else if (xpc_partition_deactivation_requested_sn2(partid)) |
33ba3c77 | 1021 | XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown); |
33ba3c77 DN |
1022 | } |
1023 | ||
1024 | /* | |
c39838ce | 1025 | * Loop through the activation amo variables and process any bits |
33ba3c77 DN |
1026 | * which are set. Each bit indicates a nasid sending a partition |
1027 | * activation or deactivation request. | |
1028 | * | |
1029 | * Return #of IRQs detected. | |
1030 | */ | |
1031 | int | |
6e41017a | 1032 | xpc_identify_activate_IRQ_sender_sn2(void) |
33ba3c77 DN |
1033 | { |
1034 | int word, bit; | |
1035 | u64 nasid_mask; | |
1036 | u64 nasid; /* remote nasid */ | |
1037 | int n_IRQs_detected = 0; | |
c39838ce | 1038 | struct amo *act_amos; |
33ba3c77 | 1039 | |
8e85c23e | 1040 | act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2; |
33ba3c77 | 1041 | |
c39838ce | 1042 | /* scan through act amo variable looking for non-zero entries */ |
ee6665e3 | 1043 | for (word = 0; word < xpc_nasid_mask_words; word++) { |
33ba3c77 DN |
1044 | |
1045 | if (xpc_exiting) | |
1046 | break; | |
1047 | ||
7fb5e59d | 1048 | nasid_mask = xpc_receive_IRQ_amo_sn2(&act_amos[word]); |
33ba3c77 DN |
1049 | if (nasid_mask == 0) { |
1050 | /* no IRQs from nasids in this variable */ | |
1051 | continue; | |
1052 | } | |
1053 | ||
c39838ce | 1054 | dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", word, |
33ba3c77 DN |
1055 | nasid_mask); |
1056 | ||
1057 | /* | |
1058 | * If this nasid has been added to the machine since | |
1059 | * our partition was reset, this will retain the | |
1060 | * remote nasid in our reserved pages machine mask. | |
1061 | * This is used in the event of module reload. | |
1062 | */ | |
1063 | xpc_mach_nasids[word] |= nasid_mask; | |
1064 | ||
1065 | /* locate the nasid(s) which sent interrupts */ | |
1066 | ||
1067 | for (bit = 0; bit < (8 * sizeof(u64)); bit++) { | |
1068 | if (nasid_mask & (1UL << bit)) { | |
1069 | n_IRQs_detected++; | |
1070 | nasid = XPC_NASID_FROM_W_B(word, bit); | |
1071 | dev_dbg(xpc_part, "interrupt from nasid %ld\n", | |
1072 | nasid); | |
6e41017a | 1073 | xpc_identify_activate_IRQ_req_sn2(nasid); |
33ba3c77 DN |
1074 | } |
1075 | } | |
1076 | } | |
1077 | return n_IRQs_detected; | |
1078 | } | |
1079 | ||
1080 | static void | |
6e41017a | 1081 | xpc_process_activate_IRQ_rcvd_sn2(int n_IRQs_expected) |
33ba3c77 DN |
1082 | { |
1083 | int n_IRQs_detected; | |
1084 | ||
6e41017a | 1085 | n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 | 1086 | if (n_IRQs_detected < n_IRQs_expected) { |
c39838ce | 1087 | /* retry once to help avoid missing amo */ |
6e41017a | 1088 | (void)xpc_identify_activate_IRQ_sender_sn2(); |
33ba3c77 DN |
1089 | } |
1090 | } | |
1091 | ||
185c3a1b DN |
1092 | /* |
1093 | * Guarantee that the kzalloc'd memory is cacheline aligned. | |
1094 | */ | |
1095 | static void * | |
1096 | xpc_kzalloc_cacheline_aligned_sn2(size_t size, gfp_t flags, void **base) | |
1097 | { | |
1098 | /* see if kzalloc will give us cachline aligned memory by default */ | |
1099 | *base = kzalloc(size, flags); | |
1100 | if (*base == NULL) | |
1101 | return NULL; | |
1102 | ||
1103 | if ((u64)*base == L1_CACHE_ALIGN((u64)*base)) | |
1104 | return *base; | |
1105 | ||
1106 | kfree(*base); | |
1107 | ||
1108 | /* nope, we'll have to do it ourselves */ | |
1109 | *base = kzalloc(size + L1_CACHE_BYTES, flags); | |
1110 | if (*base == NULL) | |
1111 | return NULL; | |
1112 | ||
1113 | return (void *)L1_CACHE_ALIGN((u64)*base); | |
1114 | } | |
1115 | ||
e17d416b DN |
1116 | /* |
1117 | * Setup the infrastructure necessary to support XPartition Communication | |
1118 | * between the specified remote partition and the local one. | |
1119 | */ | |
1120 | static enum xp_retval | |
1121 | xpc_setup_infrastructure_sn2(struct xpc_partition *part) | |
1122 | { | |
a47d5dac | 1123 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1124 | enum xp_retval retval; |
1125 | int ret; | |
1126 | int cpuid; | |
1127 | int ch_number; | |
1128 | struct xpc_channel *ch; | |
1129 | struct timer_list *timer; | |
1130 | short partid = XPC_PARTID(part); | |
1131 | ||
1132 | /* | |
1133 | * Allocate all of the channel structures as a contiguous chunk of | |
1134 | * memory. | |
1135 | */ | |
1136 | DBUG_ON(part->channels != NULL); | |
1137 | part->channels = kzalloc(sizeof(struct xpc_channel) * XPC_MAX_NCHANNELS, | |
1138 | GFP_KERNEL); | |
1139 | if (part->channels == NULL) { | |
1140 | dev_err(xpc_chan, "can't get memory for channels\n"); | |
1141 | return xpNoMemory; | |
1142 | } | |
1143 | ||
1144 | /* allocate all the required GET/PUT values */ | |
1145 | ||
185c3a1b DN |
1146 | part_sn2->local_GPs = |
1147 | xpc_kzalloc_cacheline_aligned_sn2(XPC_GP_SIZE, GFP_KERNEL, | |
1148 | &part_sn2->local_GPs_base); | |
a47d5dac | 1149 | if (part_sn2->local_GPs == NULL) { |
e17d416b DN |
1150 | dev_err(xpc_chan, "can't get memory for local get/put " |
1151 | "values\n"); | |
1152 | retval = xpNoMemory; | |
1153 | goto out_1; | |
1154 | } | |
1155 | ||
185c3a1b DN |
1156 | part_sn2->remote_GPs = |
1157 | xpc_kzalloc_cacheline_aligned_sn2(XPC_GP_SIZE, GFP_KERNEL, | |
1158 | &part_sn2->remote_GPs_base); | |
a47d5dac | 1159 | if (part_sn2->remote_GPs == NULL) { |
e17d416b DN |
1160 | dev_err(xpc_chan, "can't get memory for remote get/put " |
1161 | "values\n"); | |
1162 | retval = xpNoMemory; | |
1163 | goto out_2; | |
1164 | } | |
1165 | ||
a47d5dac | 1166 | part_sn2->remote_GPs_pa = 0; |
e17d416b DN |
1167 | |
1168 | /* allocate all the required open and close args */ | |
1169 | ||
1170 | part->local_openclose_args = | |
185c3a1b DN |
1171 | xpc_kzalloc_cacheline_aligned_sn2(XPC_OPENCLOSE_ARGS_SIZE, |
1172 | GFP_KERNEL, | |
1173 | &part->local_openclose_args_base); | |
e17d416b DN |
1174 | if (part->local_openclose_args == NULL) { |
1175 | dev_err(xpc_chan, "can't get memory for local connect args\n"); | |
1176 | retval = xpNoMemory; | |
1177 | goto out_3; | |
1178 | } | |
1179 | ||
1180 | part->remote_openclose_args = | |
185c3a1b DN |
1181 | xpc_kzalloc_cacheline_aligned_sn2(XPC_OPENCLOSE_ARGS_SIZE, |
1182 | GFP_KERNEL, | |
1183 | &part->remote_openclose_args_base); | |
e17d416b DN |
1184 | if (part->remote_openclose_args == NULL) { |
1185 | dev_err(xpc_chan, "can't get memory for remote connect args\n"); | |
1186 | retval = xpNoMemory; | |
1187 | goto out_4; | |
1188 | } | |
1189 | ||
a47d5dac | 1190 | part_sn2->remote_openclose_args_pa = 0; |
e17d416b | 1191 | |
7fb5e59d DN |
1192 | part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid); |
1193 | part->chctl.all_flags = 0; | |
1194 | spin_lock_init(&part->chctl_lock); | |
e17d416b | 1195 | |
7fb5e59d DN |
1196 | part_sn2->notify_IRQ_nasid = 0; |
1197 | part_sn2->notify_IRQ_phys_cpuid = 0; | |
1198 | part_sn2->remote_chctl_amo_va = NULL; | |
e17d416b DN |
1199 | |
1200 | atomic_set(&part->channel_mgr_requests, 1); | |
1201 | init_waitqueue_head(&part->channel_mgr_wq); | |
1202 | ||
7fb5e59d | 1203 | sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid); |
a47d5dac | 1204 | ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2, |
7fb5e59d | 1205 | IRQF_SHARED, part_sn2->notify_IRQ_owner, |
a47d5dac | 1206 | (void *)(u64)partid); |
e17d416b DN |
1207 | if (ret != 0) { |
1208 | dev_err(xpc_chan, "can't register NOTIFY IRQ handler, " | |
1209 | "errno=%d\n", -ret); | |
1210 | retval = xpLackOfResources; | |
1211 | goto out_5; | |
1212 | } | |
1213 | ||
7fb5e59d | 1214 | /* Setup a timer to check for dropped notify IRQs */ |
a47d5dac | 1215 | timer = &part_sn2->dropped_notify_IRQ_timer; |
e17d416b | 1216 | init_timer(timer); |
a47d5dac | 1217 | timer->function = |
7fb5e59d | 1218 | (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2; |
e17d416b | 1219 | timer->data = (unsigned long)part; |
7fb5e59d | 1220 | timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL; |
e17d416b DN |
1221 | add_timer(timer); |
1222 | ||
1223 | part->nchannels = XPC_MAX_NCHANNELS; | |
1224 | ||
1225 | atomic_set(&part->nchannels_active, 0); | |
1226 | atomic_set(&part->nchannels_engaged, 0); | |
1227 | ||
1228 | for (ch_number = 0; ch_number < part->nchannels; ch_number++) { | |
1229 | ch = &part->channels[ch_number]; | |
1230 | ||
1231 | ch->partid = partid; | |
1232 | ch->number = ch_number; | |
1233 | ch->flags = XPC_C_DISCONNECTED; | |
1234 | ||
a47d5dac | 1235 | ch->sn.sn2.local_GP = &part_sn2->local_GPs[ch_number]; |
e17d416b DN |
1236 | ch->local_openclose_args = |
1237 | &part->local_openclose_args[ch_number]; | |
1238 | ||
1239 | atomic_set(&ch->kthreads_assigned, 0); | |
1240 | atomic_set(&ch->kthreads_idle, 0); | |
1241 | atomic_set(&ch->kthreads_active, 0); | |
1242 | ||
1243 | atomic_set(&ch->references, 0); | |
1244 | atomic_set(&ch->n_to_notify, 0); | |
1245 | ||
1246 | spin_lock_init(&ch->lock); | |
a47d5dac | 1247 | mutex_init(&ch->sn.sn2.msg_to_pull_mutex); |
e17d416b DN |
1248 | init_completion(&ch->wdisconnect_wait); |
1249 | ||
1250 | atomic_set(&ch->n_on_msg_allocate_wq, 0); | |
1251 | init_waitqueue_head(&ch->msg_allocate_wq); | |
1252 | init_waitqueue_head(&ch->idle_wq); | |
1253 | } | |
1254 | ||
1255 | /* | |
1256 | * With the setting of the partition setup_state to XPC_P_SETUP, we're | |
1257 | * declaring that this partition is ready to go. | |
1258 | */ | |
1259 | part->setup_state = XPC_P_SETUP; | |
1260 | ||
1261 | /* | |
1262 | * Setup the per partition specific variables required by the | |
1263 | * remote partition to establish channel connections with us. | |
1264 | * | |
1265 | * The setting of the magic # indicates that these per partition | |
1266 | * specific variables are ready to be used. | |
1267 | */ | |
8e85c23e DN |
1268 | xpc_vars_part_sn2[partid].GPs_pa = __pa(part_sn2->local_GPs); |
1269 | xpc_vars_part_sn2[partid].openclose_args_pa = | |
e17d416b | 1270 | __pa(part->local_openclose_args); |
8e85c23e DN |
1271 | xpc_vars_part_sn2[partid].chctl_amo_pa = |
1272 | __pa(part_sn2->local_chctl_amo_va); | |
e17d416b | 1273 | cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */ |
8e85c23e DN |
1274 | xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid); |
1275 | xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid = | |
1276 | cpu_physical_id(cpuid); | |
1277 | xpc_vars_part_sn2[partid].nchannels = part->nchannels; | |
1278 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1; | |
e17d416b DN |
1279 | |
1280 | return xpSuccess; | |
1281 | ||
1282 | /* setup of infrastructure failed */ | |
1283 | out_5: | |
1284 | kfree(part->remote_openclose_args_base); | |
1285 | part->remote_openclose_args = NULL; | |
1286 | out_4: | |
1287 | kfree(part->local_openclose_args_base); | |
1288 | part->local_openclose_args = NULL; | |
1289 | out_3: | |
a47d5dac DN |
1290 | kfree(part_sn2->remote_GPs_base); |
1291 | part_sn2->remote_GPs = NULL; | |
e17d416b | 1292 | out_2: |
a47d5dac DN |
1293 | kfree(part_sn2->local_GPs_base); |
1294 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1295 | out_1: |
1296 | kfree(part->channels); | |
1297 | part->channels = NULL; | |
1298 | return retval; | |
1299 | } | |
1300 | ||
1301 | /* | |
1302 | * Teardown the infrastructure necessary to support XPartition Communication | |
1303 | * between the specified remote partition and the local one. | |
1304 | */ | |
1305 | static void | |
1306 | xpc_teardown_infrastructure_sn2(struct xpc_partition *part) | |
1307 | { | |
a47d5dac | 1308 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1309 | short partid = XPC_PARTID(part); |
1310 | ||
1311 | /* | |
1312 | * We start off by making this partition inaccessible to local | |
1313 | * processes by marking it as no longer setup. Then we make it | |
1314 | * inaccessible to remote processes by clearing the XPC per partition | |
1315 | * specific variable's magic # (which indicates that these variables | |
7fb5e59d | 1316 | * are no longer valid) and by ignoring all XPC notify IRQs sent to |
e17d416b DN |
1317 | * this partition. |
1318 | */ | |
1319 | ||
1320 | DBUG_ON(atomic_read(&part->nchannels_engaged) != 0); | |
1321 | DBUG_ON(atomic_read(&part->nchannels_active) != 0); | |
1322 | DBUG_ON(part->setup_state != XPC_P_SETUP); | |
1323 | part->setup_state = XPC_P_WTEARDOWN; | |
1324 | ||
8e85c23e | 1325 | xpc_vars_part_sn2[partid].magic = 0; |
e17d416b DN |
1326 | |
1327 | free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid); | |
1328 | ||
1329 | /* | |
1330 | * Before proceeding with the teardown we have to wait until all | |
1331 | * existing references cease. | |
1332 | */ | |
1333 | wait_event(part->teardown_wq, (atomic_read(&part->references) == 0)); | |
1334 | ||
1335 | /* now we can begin tearing down the infrastructure */ | |
1336 | ||
1337 | part->setup_state = XPC_P_TORNDOWN; | |
1338 | ||
1339 | /* in case we've still got outstanding timers registered... */ | |
a47d5dac | 1340 | del_timer_sync(&part_sn2->dropped_notify_IRQ_timer); |
e17d416b DN |
1341 | |
1342 | kfree(part->remote_openclose_args_base); | |
1343 | part->remote_openclose_args = NULL; | |
1344 | kfree(part->local_openclose_args_base); | |
1345 | part->local_openclose_args = NULL; | |
a47d5dac DN |
1346 | kfree(part_sn2->remote_GPs_base); |
1347 | part_sn2->remote_GPs = NULL; | |
1348 | kfree(part_sn2->local_GPs_base); | |
1349 | part_sn2->local_GPs = NULL; | |
e17d416b DN |
1350 | kfree(part->channels); |
1351 | part->channels = NULL; | |
7fb5e59d | 1352 | part_sn2->local_chctl_amo_va = NULL; |
e17d416b DN |
1353 | } |
1354 | ||
1355 | /* | |
1356 | * Create a wrapper that hides the underlying mechanism for pulling a cacheline | |
1357 | * (or multiple cachelines) from a remote partition. | |
1358 | * | |
1359 | * src must be a cacheline aligned physical address on the remote partition. | |
1360 | * dst must be a cacheline aligned virtual address on this partition. | |
1361 | * cnt must be cacheline sized | |
1362 | */ | |
ea57f80c | 1363 | /* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */ |
e17d416b DN |
1364 | static enum xp_retval |
1365 | xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst, | |
1366 | const void *src, size_t cnt) | |
1367 | { | |
1368 | enum xp_retval ret; | |
1369 | ||
1370 | DBUG_ON((u64)src != L1_CACHE_ALIGN((u64)src)); | |
1371 | DBUG_ON((u64)dst != L1_CACHE_ALIGN((u64)dst)); | |
1372 | DBUG_ON(cnt != L1_CACHE_ALIGN(cnt)); | |
1373 | ||
1374 | if (part->act_state == XPC_P_DEACTIVATING) | |
1375 | return part->reason; | |
1376 | ||
1377 | ret = xp_remote_memcpy(dst, src, cnt); | |
1378 | if (ret != xpSuccess) { | |
1379 | dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed," | |
1380 | " ret=%d\n", XPC_PARTID(part), ret); | |
1381 | } | |
1382 | return ret; | |
1383 | } | |
1384 | ||
1385 | /* | |
1386 | * Pull the remote per partition specific variables from the specified | |
1387 | * partition. | |
1388 | */ | |
1389 | static enum xp_retval | |
1390 | xpc_pull_remote_vars_part_sn2(struct xpc_partition *part) | |
1391 | { | |
a47d5dac | 1392 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1393 | u8 buffer[L1_CACHE_BYTES * 2]; |
1394 | struct xpc_vars_part_sn2 *pulled_entry_cacheline = | |
1395 | (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer); | |
1396 | struct xpc_vars_part_sn2 *pulled_entry; | |
1397 | u64 remote_entry_cacheline_pa, remote_entry_pa; | |
1398 | short partid = XPC_PARTID(part); | |
1399 | enum xp_retval ret; | |
1400 | ||
1401 | /* pull the cacheline that contains the variables we're interested in */ | |
1402 | ||
a47d5dac DN |
1403 | DBUG_ON(part_sn2->remote_vars_part_pa != |
1404 | L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa)); | |
e17d416b DN |
1405 | DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2); |
1406 | ||
a47d5dac | 1407 | remote_entry_pa = part_sn2->remote_vars_part_pa + |
e17d416b DN |
1408 | sn_partition_id * sizeof(struct xpc_vars_part_sn2); |
1409 | ||
1410 | remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1)); | |
1411 | ||
1412 | pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline | |
1413 | + (remote_entry_pa & | |
1414 | (L1_CACHE_BYTES - 1))); | |
1415 | ||
1416 | ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline, | |
1417 | (void *)remote_entry_cacheline_pa, | |
1418 | L1_CACHE_BYTES); | |
1419 | if (ret != xpSuccess) { | |
1420 | dev_dbg(xpc_chan, "failed to pull XPC vars_part from " | |
1421 | "partition %d, ret=%d\n", partid, ret); | |
1422 | return ret; | |
1423 | } | |
1424 | ||
1425 | /* see if they've been set up yet */ | |
1426 | ||
1427 | if (pulled_entry->magic != XPC_VP_MAGIC1 && | |
1428 | pulled_entry->magic != XPC_VP_MAGIC2) { | |
1429 | ||
1430 | if (pulled_entry->magic != 0) { | |
1431 | dev_dbg(xpc_chan, "partition %d's XPC vars_part for " | |
1432 | "partition %d has bad magic value (=0x%lx)\n", | |
1433 | partid, sn_partition_id, pulled_entry->magic); | |
1434 | return xpBadMagic; | |
1435 | } | |
1436 | ||
1437 | /* they've not been initialized yet */ | |
1438 | return xpRetry; | |
1439 | } | |
1440 | ||
8e85c23e | 1441 | if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1) { |
e17d416b DN |
1442 | |
1443 | /* validate the variables */ | |
1444 | ||
1445 | if (pulled_entry->GPs_pa == 0 || | |
1446 | pulled_entry->openclose_args_pa == 0 || | |
7fb5e59d | 1447 | pulled_entry->chctl_amo_pa == 0) { |
e17d416b DN |
1448 | |
1449 | dev_err(xpc_chan, "partition %d's XPC vars_part for " | |
1450 | "partition %d are not valid\n", partid, | |
1451 | sn_partition_id); | |
1452 | return xpInvalidAddress; | |
1453 | } | |
1454 | ||
1455 | /* the variables we imported look to be valid */ | |
1456 | ||
a47d5dac DN |
1457 | part_sn2->remote_GPs_pa = pulled_entry->GPs_pa; |
1458 | part_sn2->remote_openclose_args_pa = | |
e17d416b | 1459 | pulled_entry->openclose_args_pa; |
7fb5e59d | 1460 | part_sn2->remote_chctl_amo_va = |
c39838ce | 1461 | (struct amo *)__va(pulled_entry->chctl_amo_pa); |
7fb5e59d DN |
1462 | part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid; |
1463 | part_sn2->notify_IRQ_phys_cpuid = | |
1464 | pulled_entry->notify_IRQ_phys_cpuid; | |
e17d416b DN |
1465 | |
1466 | if (part->nchannels > pulled_entry->nchannels) | |
1467 | part->nchannels = pulled_entry->nchannels; | |
1468 | ||
1469 | /* let the other side know that we've pulled their variables */ | |
1470 | ||
8e85c23e | 1471 | xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2; |
e17d416b DN |
1472 | } |
1473 | ||
1474 | if (pulled_entry->magic == XPC_VP_MAGIC1) | |
1475 | return xpRetry; | |
1476 | ||
1477 | return xpSuccess; | |
1478 | } | |
1479 | ||
1480 | /* | |
1481 | * Establish first contact with the remote partititon. This involves pulling | |
1482 | * the XPC per partition variables from the remote partition and waiting for | |
1483 | * the remote partition to pull ours. | |
1484 | */ | |
1485 | static enum xp_retval | |
1486 | xpc_make_first_contact_sn2(struct xpc_partition *part) | |
1487 | { | |
a47d5dac | 1488 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b DN |
1489 | enum xp_retval ret; |
1490 | ||
33ba3c77 | 1491 | /* |
c39838ce | 1492 | * Register the remote partition's amos with SAL so it can handle |
33ba3c77 DN |
1493 | * and cleanup errors within that address range should the remote |
1494 | * partition go down. We don't unregister this range because it is | |
1495 | * difficult to tell when outstanding writes to the remote partition | |
1496 | * are finished and thus when it is safe to unregister. This should | |
1497 | * not result in wasted space in the SAL xp_addr_region table because | |
1498 | * we should get the same page for remote_amos_page_pa after module | |
1499 | * reloads and system reboots. | |
1500 | */ | |
a47d5dac | 1501 | if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa, |
33ba3c77 DN |
1502 | PAGE_SIZE, 1) < 0) { |
1503 | dev_warn(xpc_part, "xpc_activating(%d) failed to register " | |
1504 | "xp_addr region\n", XPC_PARTID(part)); | |
1505 | ||
1506 | ret = xpPhysAddrRegFailed; | |
1507 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1508 | return ret; | |
1509 | } | |
1510 | ||
a47d5dac DN |
1511 | /* |
1512 | * Send activate IRQ to get other side to activate if they've not | |
1513 | * already begun to do so. | |
1514 | */ | |
7fb5e59d | 1515 | xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa, |
a47d5dac DN |
1516 | cnodeid_to_nasid(0), |
1517 | part_sn2->activate_IRQ_nasid, | |
1518 | part_sn2->activate_IRQ_phys_cpuid); | |
33ba3c77 | 1519 | |
e17d416b DN |
1520 | while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) { |
1521 | if (ret != xpRetry) { | |
1522 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1523 | return ret; | |
1524 | } | |
1525 | ||
1526 | dev_dbg(xpc_part, "waiting to make first contact with " | |
1527 | "partition %d\n", XPC_PARTID(part)); | |
1528 | ||
1529 | /* wait a 1/4 of a second or so */ | |
1530 | (void)msleep_interruptible(250); | |
1531 | ||
1532 | if (part->act_state == XPC_P_DEACTIVATING) | |
1533 | return part->reason; | |
1534 | } | |
1535 | ||
1536 | return xpSuccess; | |
1537 | } | |
1538 | ||
1539 | /* | |
7fb5e59d | 1540 | * Get the chctl flags and pull the openclose args and/or remote GPs as needed. |
e17d416b DN |
1541 | */ |
1542 | static u64 | |
7fb5e59d | 1543 | xpc_get_chctl_all_flags_sn2(struct xpc_partition *part) |
e17d416b | 1544 | { |
a47d5dac | 1545 | struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2; |
e17d416b | 1546 | unsigned long irq_flags; |
7fb5e59d | 1547 | union xpc_channel_ctl_flags chctl; |
e17d416b DN |
1548 | enum xp_retval ret; |
1549 | ||
1550 | /* | |
7fb5e59d | 1551 | * See if there are any chctl flags to be handled. |
e17d416b DN |
1552 | */ |
1553 | ||
7fb5e59d DN |
1554 | spin_lock_irqsave(&part->chctl_lock, irq_flags); |
1555 | chctl = part->chctl; | |
1556 | if (chctl.all_flags != 0) | |
1557 | part->chctl.all_flags = 0; | |
e17d416b | 1558 | |
7fb5e59d | 1559 | spin_unlock_irqrestore(&part->chctl_lock, irq_flags); |
e17d416b | 1560 | |
7fb5e59d | 1561 | if (xpc_any_openclose_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1562 | ret = xpc_pull_remote_cachelines_sn2(part, part-> |
1563 | remote_openclose_args, | |
1564 | (void *)part_sn2-> | |
e17d416b DN |
1565 | remote_openclose_args_pa, |
1566 | XPC_OPENCLOSE_ARGS_SIZE); | |
1567 | if (ret != xpSuccess) { | |
1568 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1569 | ||
1570 | dev_dbg(xpc_chan, "failed to pull openclose args from " | |
1571 | "partition %d, ret=%d\n", XPC_PARTID(part), | |
1572 | ret); | |
1573 | ||
7fb5e59d DN |
1574 | /* don't bother processing chctl flags anymore */ |
1575 | chctl.all_flags = 0; | |
e17d416b DN |
1576 | } |
1577 | } | |
1578 | ||
7fb5e59d | 1579 | if (xpc_any_msg_chctl_flags_set(&chctl)) { |
a47d5dac DN |
1580 | ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs, |
1581 | (void *)part_sn2->remote_GPs_pa, | |
e17d416b DN |
1582 | XPC_GP_SIZE); |
1583 | if (ret != xpSuccess) { | |
1584 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1585 | ||
1586 | dev_dbg(xpc_chan, "failed to pull GPs from partition " | |
1587 | "%d, ret=%d\n", XPC_PARTID(part), ret); | |
1588 | ||
7fb5e59d DN |
1589 | /* don't bother processing chctl flags anymore */ |
1590 | chctl.all_flags = 0; | |
e17d416b DN |
1591 | } |
1592 | } | |
1593 | ||
7fb5e59d | 1594 | return chctl.all_flags; |
e17d416b DN |
1595 | } |
1596 | ||
185c3a1b DN |
1597 | /* |
1598 | * Allocate the local message queue and the notify queue. | |
1599 | */ | |
1600 | static enum xp_retval | |
1601 | xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch) | |
1602 | { | |
1603 | unsigned long irq_flags; | |
1604 | int nentries; | |
1605 | size_t nbytes; | |
1606 | ||
1607 | for (nentries = ch->local_nentries; nentries > 0; nentries--) { | |
1608 | ||
1609 | nbytes = nentries * ch->msg_size; | |
1610 | ch->local_msgqueue = | |
1611 | xpc_kzalloc_cacheline_aligned_sn2(nbytes, GFP_KERNEL, | |
1612 | &ch->local_msgqueue_base); | |
1613 | if (ch->local_msgqueue == NULL) | |
1614 | continue; | |
1615 | ||
1616 | nbytes = nentries * sizeof(struct xpc_notify); | |
1617 | ch->notify_queue = kzalloc(nbytes, GFP_KERNEL); | |
1618 | if (ch->notify_queue == NULL) { | |
1619 | kfree(ch->local_msgqueue_base); | |
1620 | ch->local_msgqueue = NULL; | |
1621 | continue; | |
1622 | } | |
1623 | ||
1624 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1625 | if (nentries < ch->local_nentries) { | |
1626 | dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, " | |
1627 | "partid=%d, channel=%d\n", nentries, | |
1628 | ch->local_nentries, ch->partid, ch->number); | |
1629 | ||
1630 | ch->local_nentries = nentries; | |
1631 | } | |
1632 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1633 | return xpSuccess; | |
1634 | } | |
1635 | ||
1636 | dev_dbg(xpc_chan, "can't get memory for local message queue and notify " | |
1637 | "queue, partid=%d, channel=%d\n", ch->partid, ch->number); | |
1638 | return xpNoMemory; | |
1639 | } | |
1640 | ||
1641 | /* | |
1642 | * Allocate the cached remote message queue. | |
1643 | */ | |
1644 | static enum xp_retval | |
1645 | xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch) | |
1646 | { | |
1647 | unsigned long irq_flags; | |
1648 | int nentries; | |
1649 | size_t nbytes; | |
1650 | ||
1651 | DBUG_ON(ch->remote_nentries <= 0); | |
1652 | ||
1653 | for (nentries = ch->remote_nentries; nentries > 0; nentries--) { | |
1654 | ||
1655 | nbytes = nentries * ch->msg_size; | |
1656 | ch->remote_msgqueue = | |
1657 | xpc_kzalloc_cacheline_aligned_sn2(nbytes, GFP_KERNEL, | |
1658 | &ch->remote_msgqueue_base); | |
1659 | if (ch->remote_msgqueue == NULL) | |
1660 | continue; | |
1661 | ||
1662 | spin_lock_irqsave(&ch->lock, irq_flags); | |
1663 | if (nentries < ch->remote_nentries) { | |
1664 | dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, " | |
1665 | "partid=%d, channel=%d\n", nentries, | |
1666 | ch->remote_nentries, ch->partid, ch->number); | |
1667 | ||
1668 | ch->remote_nentries = nentries; | |
1669 | } | |
1670 | spin_unlock_irqrestore(&ch->lock, irq_flags); | |
1671 | return xpSuccess; | |
1672 | } | |
1673 | ||
1674 | dev_dbg(xpc_chan, "can't get memory for cached remote message queue, " | |
1675 | "partid=%d, channel=%d\n", ch->partid, ch->number); | |
1676 | return xpNoMemory; | |
1677 | } | |
1678 | ||
1679 | /* | |
1680 | * Allocate message queues and other stuff associated with a channel. | |
1681 | * | |
1682 | * Note: Assumes all of the channel sizes are filled in. | |
1683 | */ | |
1684 | static enum xp_retval | |
1685 | xpc_allocate_msgqueues_sn2(struct xpc_channel *ch) | |
1686 | { | |
1687 | enum xp_retval ret; | |
1688 | ||
1689 | DBUG_ON(ch->flags & XPC_C_SETUP); | |
1690 | ||
1691 | ret = xpc_allocate_local_msgqueue_sn2(ch); | |
1692 | if (ret == xpSuccess) { | |
1693 | ||
1694 | ret = xpc_allocate_remote_msgqueue_sn2(ch); | |
1695 | if (ret != xpSuccess) { | |
1696 | kfree(ch->local_msgqueue_base); | |
1697 | ch->local_msgqueue = NULL; | |
1698 | kfree(ch->notify_queue); | |
1699 | ch->notify_queue = NULL; | |
1700 | } | |
1701 | } | |
1702 | return ret; | |
1703 | } | |
1704 | ||
1705 | /* | |
1706 | * Free up message queues and other stuff that were allocated for the specified | |
1707 | * channel. | |
1708 | * | |
1709 | * Note: ch->reason and ch->reason_line are left set for debugging purposes, | |
1710 | * they're cleared when XPC_C_DISCONNECTED is cleared. | |
1711 | */ | |
1712 | static void | |
1713 | xpc_free_msgqueues_sn2(struct xpc_channel *ch) | |
1714 | { | |
1715 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1716 | ||
1717 | DBUG_ON(!spin_is_locked(&ch->lock)); | |
1718 | DBUG_ON(atomic_read(&ch->n_to_notify) != 0); | |
1719 | ||
1720 | ch->remote_msgqueue_pa = 0; | |
1721 | ch->func = NULL; | |
1722 | ch->key = NULL; | |
1723 | ch->msg_size = 0; | |
1724 | ch->local_nentries = 0; | |
1725 | ch->remote_nentries = 0; | |
1726 | ch->kthreads_assigned_limit = 0; | |
1727 | ch->kthreads_idle_limit = 0; | |
1728 | ||
1729 | ch_sn2->local_GP->get = 0; | |
1730 | ch_sn2->local_GP->put = 0; | |
1731 | ch_sn2->remote_GP.get = 0; | |
1732 | ch_sn2->remote_GP.put = 0; | |
1733 | ch_sn2->w_local_GP.get = 0; | |
1734 | ch_sn2->w_local_GP.put = 0; | |
1735 | ch_sn2->w_remote_GP.get = 0; | |
1736 | ch_sn2->w_remote_GP.put = 0; | |
1737 | ch_sn2->next_msg_to_pull = 0; | |
1738 | ||
1739 | if (ch->flags & XPC_C_SETUP) { | |
1740 | dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n", | |
1741 | ch->flags, ch->partid, ch->number); | |
1742 | ||
1743 | kfree(ch->local_msgqueue_base); | |
1744 | ch->local_msgqueue = NULL; | |
1745 | kfree(ch->remote_msgqueue_base); | |
1746 | ch->remote_msgqueue = NULL; | |
1747 | kfree(ch->notify_queue); | |
1748 | ch->notify_queue = NULL; | |
1749 | } | |
1750 | } | |
1751 | ||
a47d5dac DN |
1752 | /* |
1753 | * Notify those who wanted to be notified upon delivery of their message. | |
1754 | */ | |
1755 | static void | |
1756 | xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put) | |
1757 | { | |
1758 | struct xpc_notify *notify; | |
1759 | u8 notify_type; | |
1760 | s64 get = ch->sn.sn2.w_remote_GP.get - 1; | |
1761 | ||
1762 | while (++get < put && atomic_read(&ch->n_to_notify) > 0) { | |
1763 | ||
1764 | notify = &ch->notify_queue[get % ch->local_nentries]; | |
1765 | ||
1766 | /* | |
1767 | * See if the notify entry indicates it was associated with | |
1768 | * a message who's sender wants to be notified. It is possible | |
1769 | * that it is, but someone else is doing or has done the | |
1770 | * notification. | |
1771 | */ | |
1772 | notify_type = notify->type; | |
1773 | if (notify_type == 0 || | |
1774 | cmpxchg(¬ify->type, notify_type, 0) != notify_type) { | |
1775 | continue; | |
1776 | } | |
1777 | ||
1778 | DBUG_ON(notify_type != XPC_N_CALL); | |
1779 | ||
1780 | atomic_dec(&ch->n_to_notify); | |
1781 | ||
1782 | if (notify->func != NULL) { | |
1783 | dev_dbg(xpc_chan, "notify->func() called, notify=0x%p, " | |
1784 | "msg_number=%ld, partid=%d, channel=%d\n", | |
1785 | (void *)notify, get, ch->partid, ch->number); | |
1786 | ||
1787 | notify->func(reason, ch->partid, ch->number, | |
1788 | notify->key); | |
1789 | ||
1790 | dev_dbg(xpc_chan, "notify->func() returned, " | |
1791 | "notify=0x%p, msg_number=%ld, partid=%d, " | |
1792 | "channel=%d\n", (void *)notify, get, | |
1793 | ch->partid, ch->number); | |
1794 | } | |
1795 | } | |
1796 | } | |
1797 | ||
1798 | static void | |
1799 | xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch) | |
1800 | { | |
1801 | xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put); | |
1802 | } | |
1803 | ||
1804 | /* | |
1805 | * Clear some of the msg flags in the local message queue. | |
1806 | */ | |
1807 | static inline void | |
1808 | xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1809 | { | |
1810 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1811 | struct xpc_msg *msg; | |
1812 | s64 get; | |
1813 | ||
1814 | get = ch_sn2->w_remote_GP.get; | |
1815 | do { | |
1816 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
1817 | (get % ch->local_nentries) * | |
1818 | ch->msg_size); | |
1819 | msg->flags = 0; | |
1820 | } while (++get < ch_sn2->remote_GP.get); | |
1821 | } | |
1822 | ||
1823 | /* | |
1824 | * Clear some of the msg flags in the remote message queue. | |
1825 | */ | |
1826 | static inline void | |
1827 | xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch) | |
1828 | { | |
1829 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1830 | struct xpc_msg *msg; | |
1831 | s64 put; | |
1832 | ||
1833 | put = ch_sn2->w_remote_GP.put; | |
1834 | do { | |
1835 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + | |
1836 | (put % ch->remote_nentries) * | |
1837 | ch->msg_size); | |
1838 | msg->flags = 0; | |
1839 | } while (++put < ch_sn2->remote_GP.put); | |
1840 | } | |
1841 | ||
1842 | static void | |
7fb5e59d | 1843 | xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number) |
a47d5dac DN |
1844 | { |
1845 | struct xpc_channel *ch = &part->channels[ch_number]; | |
1846 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; | |
1847 | int nmsgs_sent; | |
1848 | ||
1849 | ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number]; | |
1850 | ||
1851 | /* See what, if anything, has changed for each connected channel */ | |
1852 | ||
1853 | xpc_msgqueue_ref(ch); | |
1854 | ||
1855 | if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get && | |
1856 | ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) { | |
1857 | /* nothing changed since GPs were last pulled */ | |
1858 | xpc_msgqueue_deref(ch); | |
1859 | return; | |
1860 | } | |
1861 | ||
1862 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
1863 | xpc_msgqueue_deref(ch); | |
1864 | return; | |
1865 | } | |
1866 | ||
1867 | /* | |
1868 | * First check to see if messages recently sent by us have been | |
1869 | * received by the other side. (The remote GET value will have | |
1870 | * changed since we last looked at it.) | |
1871 | */ | |
1872 | ||
1873 | if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) { | |
1874 | ||
1875 | /* | |
1876 | * We need to notify any senders that want to be notified | |
1877 | * that their sent messages have been received by their | |
1878 | * intended recipients. We need to do this before updating | |
1879 | * w_remote_GP.get so that we don't allocate the same message | |
1880 | * queue entries prematurely (see xpc_allocate_msg()). | |
1881 | */ | |
1882 | if (atomic_read(&ch->n_to_notify) > 0) { | |
1883 | /* | |
1884 | * Notify senders that messages sent have been | |
1885 | * received and delivered by the other side. | |
1886 | */ | |
1887 | xpc_notify_senders_sn2(ch, xpMsgDelivered, | |
1888 | ch_sn2->remote_GP.get); | |
1889 | } | |
1890 | ||
1891 | /* | |
1892 | * Clear msg->flags in previously sent messages, so that | |
1893 | * they're ready for xpc_allocate_msg(). | |
1894 | */ | |
1895 | xpc_clear_local_msgqueue_flags_sn2(ch); | |
1896 | ||
1897 | ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get; | |
1898 | ||
1899 | dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, " | |
1900 | "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid, | |
1901 | ch->number); | |
1902 | ||
1903 | /* | |
1904 | * If anyone was waiting for message queue entries to become | |
1905 | * available, wake them up. | |
1906 | */ | |
1907 | if (atomic_read(&ch->n_on_msg_allocate_wq) > 0) | |
1908 | wake_up(&ch->msg_allocate_wq); | |
1909 | } | |
1910 | ||
1911 | /* | |
1912 | * Now check for newly sent messages by the other side. (The remote | |
1913 | * PUT value will have changed since we last looked at it.) | |
1914 | */ | |
1915 | ||
1916 | if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) { | |
1917 | /* | |
1918 | * Clear msg->flags in previously received messages, so that | |
1919 | * they're ready for xpc_get_deliverable_msg(). | |
1920 | */ | |
1921 | xpc_clear_remote_msgqueue_flags_sn2(ch); | |
1922 | ||
1923 | ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put; | |
1924 | ||
1925 | dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, " | |
1926 | "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid, | |
1927 | ch->number); | |
1928 | ||
1929 | nmsgs_sent = ch_sn2->w_remote_GP.put - ch_sn2->w_local_GP.get; | |
1930 | if (nmsgs_sent > 0) { | |
1931 | dev_dbg(xpc_chan, "msgs waiting to be copied and " | |
1932 | "delivered=%d, partid=%d, channel=%d\n", | |
1933 | nmsgs_sent, ch->partid, ch->number); | |
1934 | ||
1935 | if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) | |
1936 | xpc_activate_kthreads(ch, nmsgs_sent); | |
1937 | } | |
1938 | } | |
1939 | ||
1940 | xpc_msgqueue_deref(ch); | |
1941 | } | |
1942 | ||
e17d416b DN |
1943 | static struct xpc_msg * |
1944 | xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get) | |
1945 | { | |
1946 | struct xpc_partition *part = &xpc_partitions[ch->partid]; | |
a47d5dac | 1947 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
e17d416b DN |
1948 | struct xpc_msg *remote_msg, *msg; |
1949 | u32 msg_index, nmsgs; | |
1950 | u64 msg_offset; | |
1951 | enum xp_retval ret; | |
1952 | ||
a47d5dac | 1953 | if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) { |
e17d416b DN |
1954 | /* we were interrupted by a signal */ |
1955 | return NULL; | |
1956 | } | |
1957 | ||
a47d5dac | 1958 | while (get >= ch_sn2->next_msg_to_pull) { |
e17d416b DN |
1959 | |
1960 | /* pull as many messages as are ready and able to be pulled */ | |
1961 | ||
a47d5dac | 1962 | msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries; |
e17d416b | 1963 | |
a47d5dac DN |
1964 | DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put); |
1965 | nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull; | |
e17d416b DN |
1966 | if (msg_index + nmsgs > ch->remote_nentries) { |
1967 | /* ignore the ones that wrap the msg queue for now */ | |
1968 | nmsgs = ch->remote_nentries - msg_index; | |
1969 | } | |
1970 | ||
1971 | msg_offset = msg_index * ch->msg_size; | |
1972 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + msg_offset); | |
1973 | remote_msg = (struct xpc_msg *)(ch->remote_msgqueue_pa + | |
1974 | msg_offset); | |
1975 | ||
1976 | ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg, | |
1977 | nmsgs * ch->msg_size); | |
1978 | if (ret != xpSuccess) { | |
1979 | ||
1980 | dev_dbg(xpc_chan, "failed to pull %d msgs starting with" | |
1981 | " msg %ld from partition %d, channel=%d, " | |
a47d5dac | 1982 | "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull, |
e17d416b DN |
1983 | ch->partid, ch->number, ret); |
1984 | ||
1985 | XPC_DEACTIVATE_PARTITION(part, ret); | |
1986 | ||
a47d5dac | 1987 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1988 | return NULL; |
1989 | } | |
1990 | ||
a47d5dac | 1991 | ch_sn2->next_msg_to_pull += nmsgs; |
e17d416b DN |
1992 | } |
1993 | ||
a47d5dac | 1994 | mutex_unlock(&ch_sn2->msg_to_pull_mutex); |
e17d416b DN |
1995 | |
1996 | /* return the message we were looking for */ | |
1997 | msg_offset = (get % ch->remote_nentries) * ch->msg_size; | |
1998 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + msg_offset); | |
1999 | ||
2000 | return msg; | |
2001 | } | |
2002 | ||
a47d5dac DN |
2003 | static int |
2004 | xpc_n_of_deliverable_msgs_sn2(struct xpc_channel *ch) | |
2005 | { | |
2006 | return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get; | |
2007 | } | |
2008 | ||
e17d416b DN |
2009 | /* |
2010 | * Get a message to be delivered. | |
2011 | */ | |
2012 | static struct xpc_msg * | |
2013 | xpc_get_deliverable_msg_sn2(struct xpc_channel *ch) | |
2014 | { | |
a47d5dac | 2015 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
e17d416b DN |
2016 | struct xpc_msg *msg = NULL; |
2017 | s64 get; | |
2018 | ||
2019 | do { | |
2020 | if (ch->flags & XPC_C_DISCONNECTING) | |
2021 | break; | |
2022 | ||
a47d5dac | 2023 | get = ch_sn2->w_local_GP.get; |
e17d416b | 2024 | rmb(); /* guarantee that .get loads before .put */ |
a47d5dac | 2025 | if (get == ch_sn2->w_remote_GP.put) |
e17d416b DN |
2026 | break; |
2027 | ||
2028 | /* There are messages waiting to be pulled and delivered. | |
2029 | * We need to try to secure one for ourselves. We'll do this | |
2030 | * by trying to increment w_local_GP.get and hope that no one | |
2031 | * else beats us to it. If they do, we'll we'll simply have | |
2032 | * to try again for the next one. | |
2033 | */ | |
2034 | ||
a47d5dac | 2035 | if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) { |
e17d416b DN |
2036 | /* we got the entry referenced by get */ |
2037 | ||
2038 | dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, " | |
2039 | "partid=%d, channel=%d\n", get + 1, | |
2040 | ch->partid, ch->number); | |
2041 | ||
2042 | /* pull the message from the remote partition */ | |
2043 | ||
2044 | msg = xpc_pull_remote_msg_sn2(ch, get); | |
2045 | ||
2046 | DBUG_ON(msg != NULL && msg->number != get); | |
2047 | DBUG_ON(msg != NULL && (msg->flags & XPC_M_DONE)); | |
2048 | DBUG_ON(msg != NULL && !(msg->flags & XPC_M_READY)); | |
2049 | ||
2050 | break; | |
2051 | } | |
2052 | ||
2053 | } while (1); | |
2054 | ||
2055 | return msg; | |
2056 | } | |
2057 | ||
33ba3c77 DN |
2058 | /* |
2059 | * Now we actually send the messages that are ready to be sent by advancing | |
7fb5e59d DN |
2060 | * the local message queue's Put value and then send a chctl msgrequest to the |
2061 | * recipient partition. | |
33ba3c77 DN |
2062 | */ |
2063 | static void | |
2064 | xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put) | |
2065 | { | |
a47d5dac | 2066 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2067 | struct xpc_msg *msg; |
2068 | s64 put = initial_put + 1; | |
7fb5e59d | 2069 | int send_msgrequest = 0; |
33ba3c77 DN |
2070 | |
2071 | while (1) { | |
2072 | ||
2073 | while (1) { | |
a47d5dac | 2074 | if (put == ch_sn2->w_local_GP.put) |
33ba3c77 DN |
2075 | break; |
2076 | ||
2077 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
2078 | (put % ch->local_nentries) * | |
2079 | ch->msg_size); | |
2080 | ||
2081 | if (!(msg->flags & XPC_M_READY)) | |
2082 | break; | |
2083 | ||
2084 | put++; | |
2085 | } | |
2086 | ||
2087 | if (put == initial_put) { | |
2088 | /* nothing's changed */ | |
2089 | break; | |
2090 | } | |
2091 | ||
a47d5dac | 2092 | if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) != |
33ba3c77 DN |
2093 | initial_put) { |
2094 | /* someone else beat us to it */ | |
a47d5dac | 2095 | DBUG_ON(ch_sn2->local_GP->put < initial_put); |
33ba3c77 DN |
2096 | break; |
2097 | } | |
2098 | ||
2099 | /* we just set the new value of local_GP->put */ | |
2100 | ||
2101 | dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, " | |
2102 | "channel=%d\n", put, ch->partid, ch->number); | |
2103 | ||
7fb5e59d | 2104 | send_msgrequest = 1; |
33ba3c77 DN |
2105 | |
2106 | /* | |
2107 | * We need to ensure that the message referenced by | |
2108 | * local_GP->put is not XPC_M_READY or that local_GP->put | |
2109 | * equals w_local_GP.put, so we'll go have a look. | |
2110 | */ | |
2111 | initial_put = put; | |
2112 | } | |
2113 | ||
7fb5e59d DN |
2114 | if (send_msgrequest) |
2115 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2116 | } |
2117 | ||
2118 | /* | |
2119 | * Allocate an entry for a message from the message queue associated with the | |
2120 | * specified channel. | |
2121 | */ | |
2122 | static enum xp_retval | |
2123 | xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags, | |
2124 | struct xpc_msg **address_of_msg) | |
2125 | { | |
a47d5dac | 2126 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2127 | struct xpc_msg *msg; |
2128 | enum xp_retval ret; | |
2129 | s64 put; | |
2130 | ||
33ba3c77 DN |
2131 | /* |
2132 | * Get the next available message entry from the local message queue. | |
2133 | * If none are available, we'll make sure that we grab the latest | |
2134 | * GP values. | |
2135 | */ | |
2136 | ret = xpTimeout; | |
2137 | ||
2138 | while (1) { | |
2139 | ||
a47d5dac | 2140 | put = ch_sn2->w_local_GP.put; |
33ba3c77 | 2141 | rmb(); /* guarantee that .put loads before .get */ |
a47d5dac | 2142 | if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) { |
33ba3c77 DN |
2143 | |
2144 | /* There are available message entries. We need to try | |
2145 | * to secure one for ourselves. We'll do this by trying | |
2146 | * to increment w_local_GP.put as long as someone else | |
2147 | * doesn't beat us to it. If they do, we'll have to | |
2148 | * try again. | |
2149 | */ | |
a47d5dac DN |
2150 | if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) == |
2151 | put) { | |
33ba3c77 DN |
2152 | /* we got the entry referenced by put */ |
2153 | break; | |
2154 | } | |
2155 | continue; /* try again */ | |
2156 | } | |
2157 | ||
2158 | /* | |
2159 | * There aren't any available msg entries at this time. | |
2160 | * | |
2161 | * In waiting for a message entry to become available, | |
7fb5e59d DN |
2162 | * we set a timeout in case the other side is not sending |
2163 | * completion interrupts. This lets us fake a notify IRQ | |
2164 | * that will cause the notify IRQ handler to fetch the latest | |
2165 | * GP values as if an interrupt was sent by the other side. | |
33ba3c77 DN |
2166 | */ |
2167 | if (ret == xpTimeout) | |
7fb5e59d | 2168 | xpc_send_chctl_local_msgrequest_sn2(ch); |
33ba3c77 | 2169 | |
97bf1aa1 | 2170 | if (flags & XPC_NOWAIT) |
33ba3c77 | 2171 | return xpNoWait; |
33ba3c77 DN |
2172 | |
2173 | ret = xpc_allocate_msg_wait(ch); | |
97bf1aa1 | 2174 | if (ret != xpInterrupted && ret != xpTimeout) |
33ba3c77 | 2175 | return ret; |
33ba3c77 DN |
2176 | } |
2177 | ||
2178 | /* get the message's address and initialize it */ | |
2179 | msg = (struct xpc_msg *)((u64)ch->local_msgqueue + | |
2180 | (put % ch->local_nentries) * ch->msg_size); | |
2181 | ||
2182 | DBUG_ON(msg->flags != 0); | |
2183 | msg->number = put; | |
2184 | ||
2185 | dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, " | |
2186 | "msg_number=%ld, partid=%d, channel=%d\n", put + 1, | |
2187 | (void *)msg, msg->number, ch->partid, ch->number); | |
2188 | ||
2189 | *address_of_msg = msg; | |
33ba3c77 DN |
2190 | return xpSuccess; |
2191 | } | |
2192 | ||
2193 | /* | |
2194 | * Common code that does the actual sending of the message by advancing the | |
7fb5e59d DN |
2195 | * local message queue's Put value and sends a chctl msgrequest to the |
2196 | * partition the message is being sent to. | |
33ba3c77 DN |
2197 | */ |
2198 | static enum xp_retval | |
97bf1aa1 DN |
2199 | xpc_send_msg_sn2(struct xpc_channel *ch, u32 flags, void *payload, |
2200 | u16 payload_size, u8 notify_type, xpc_notify_func func, | |
2201 | void *key) | |
33ba3c77 DN |
2202 | { |
2203 | enum xp_retval ret = xpSuccess; | |
97bf1aa1 | 2204 | struct xpc_msg *msg = msg; |
33ba3c77 | 2205 | struct xpc_notify *notify = notify; |
97bf1aa1 DN |
2206 | s64 msg_number; |
2207 | s64 put; | |
33ba3c77 DN |
2208 | |
2209 | DBUG_ON(notify_type == XPC_N_CALL && func == NULL); | |
97bf1aa1 DN |
2210 | |
2211 | if (XPC_MSG_SIZE(payload_size) > ch->msg_size) | |
2212 | return xpPayloadTooBig; | |
2213 | ||
2214 | xpc_msgqueue_ref(ch); | |
33ba3c77 DN |
2215 | |
2216 | if (ch->flags & XPC_C_DISCONNECTING) { | |
97bf1aa1 DN |
2217 | ret = ch->reason; |
2218 | goto out_1; | |
2219 | } | |
2220 | if (!(ch->flags & XPC_C_CONNECTED)) { | |
2221 | ret = xpNotConnected; | |
2222 | goto out_1; | |
33ba3c77 DN |
2223 | } |
2224 | ||
97bf1aa1 DN |
2225 | ret = xpc_allocate_msg_sn2(ch, flags, &msg); |
2226 | if (ret != xpSuccess) | |
2227 | goto out_1; | |
2228 | ||
2229 | msg_number = msg->number; | |
2230 | ||
33ba3c77 DN |
2231 | if (notify_type != 0) { |
2232 | /* | |
2233 | * Tell the remote side to send an ACK interrupt when the | |
2234 | * message has been delivered. | |
2235 | */ | |
2236 | msg->flags |= XPC_M_INTERRUPT; | |
2237 | ||
2238 | atomic_inc(&ch->n_to_notify); | |
2239 | ||
2240 | notify = &ch->notify_queue[msg_number % ch->local_nentries]; | |
2241 | notify->func = func; | |
2242 | notify->key = key; | |
2243 | notify->type = notify_type; | |
2244 | ||
ea57f80c | 2245 | /* ??? Is a mb() needed here? */ |
33ba3c77 DN |
2246 | |
2247 | if (ch->flags & XPC_C_DISCONNECTING) { | |
2248 | /* | |
2249 | * An error occurred between our last error check and | |
2250 | * this one. We will try to clear the type field from | |
2251 | * the notify entry. If we succeed then | |
2252 | * xpc_disconnect_channel() didn't already process | |
2253 | * the notify entry. | |
2254 | */ | |
2255 | if (cmpxchg(¬ify->type, notify_type, 0) == | |
2256 | notify_type) { | |
2257 | atomic_dec(&ch->n_to_notify); | |
2258 | ret = ch->reason; | |
2259 | } | |
97bf1aa1 | 2260 | goto out_1; |
33ba3c77 DN |
2261 | } |
2262 | } | |
2263 | ||
97bf1aa1 DN |
2264 | memcpy(&msg->payload, payload, payload_size); |
2265 | ||
33ba3c77 DN |
2266 | msg->flags |= XPC_M_READY; |
2267 | ||
2268 | /* | |
2269 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2270 | * load of local_GP->put. |
33ba3c77 DN |
2271 | */ |
2272 | mb(); | |
2273 | ||
2274 | /* see if the message is next in line to be sent, if so send it */ | |
2275 | ||
a47d5dac | 2276 | put = ch->sn.sn2.local_GP->put; |
33ba3c77 DN |
2277 | if (put == msg_number) |
2278 | xpc_send_msgs_sn2(ch, put); | |
2279 | ||
97bf1aa1 | 2280 | out_1: |
33ba3c77 DN |
2281 | xpc_msgqueue_deref(ch); |
2282 | return ret; | |
2283 | } | |
2284 | ||
2285 | /* | |
2286 | * Now we actually acknowledge the messages that have been delivered and ack'd | |
2287 | * by advancing the cached remote message queue's Get value and if requested | |
7fb5e59d | 2288 | * send a chctl msgrequest to the message sender's partition. |
33ba3c77 DN |
2289 | */ |
2290 | static void | |
2291 | xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags) | |
2292 | { | |
a47d5dac | 2293 | struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; |
33ba3c77 DN |
2294 | struct xpc_msg *msg; |
2295 | s64 get = initial_get + 1; | |
7fb5e59d | 2296 | int send_msgrequest = 0; |
33ba3c77 DN |
2297 | |
2298 | while (1) { | |
2299 | ||
2300 | while (1) { | |
a47d5dac | 2301 | if (get == ch_sn2->w_local_GP.get) |
33ba3c77 DN |
2302 | break; |
2303 | ||
2304 | msg = (struct xpc_msg *)((u64)ch->remote_msgqueue + | |
2305 | (get % ch->remote_nentries) * | |
2306 | ch->msg_size); | |
2307 | ||
2308 | if (!(msg->flags & XPC_M_DONE)) | |
2309 | break; | |
2310 | ||
2311 | msg_flags |= msg->flags; | |
2312 | get++; | |
2313 | } | |
2314 | ||
2315 | if (get == initial_get) { | |
2316 | /* nothing's changed */ | |
2317 | break; | |
2318 | } | |
2319 | ||
a47d5dac | 2320 | if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) != |
33ba3c77 DN |
2321 | initial_get) { |
2322 | /* someone else beat us to it */ | |
a47d5dac | 2323 | DBUG_ON(ch_sn2->local_GP->get <= initial_get); |
33ba3c77 DN |
2324 | break; |
2325 | } | |
2326 | ||
2327 | /* we just set the new value of local_GP->get */ | |
2328 | ||
2329 | dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, " | |
2330 | "channel=%d\n", get, ch->partid, ch->number); | |
2331 | ||
7fb5e59d | 2332 | send_msgrequest = (msg_flags & XPC_M_INTERRUPT); |
33ba3c77 DN |
2333 | |
2334 | /* | |
2335 | * We need to ensure that the message referenced by | |
2336 | * local_GP->get is not XPC_M_DONE or that local_GP->get | |
2337 | * equals w_local_GP.get, so we'll go have a look. | |
2338 | */ | |
2339 | initial_get = get; | |
2340 | } | |
2341 | ||
7fb5e59d DN |
2342 | if (send_msgrequest) |
2343 | xpc_send_chctl_msgrequest_sn2(ch); | |
33ba3c77 DN |
2344 | } |
2345 | ||
2346 | static void | |
2347 | xpc_received_msg_sn2(struct xpc_channel *ch, struct xpc_msg *msg) | |
2348 | { | |
2349 | s64 get; | |
2350 | s64 msg_number = msg->number; | |
2351 | ||
2352 | dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n", | |
2353 | (void *)msg, msg_number, ch->partid, ch->number); | |
2354 | ||
2355 | DBUG_ON((((u64)msg - (u64)ch->remote_msgqueue) / ch->msg_size) != | |
2356 | msg_number % ch->remote_nentries); | |
2357 | DBUG_ON(msg->flags & XPC_M_DONE); | |
2358 | ||
2359 | msg->flags |= XPC_M_DONE; | |
2360 | ||
2361 | /* | |
2362 | * The preceding store of msg->flags must occur before the following | |
a47d5dac | 2363 | * load of local_GP->get. |
33ba3c77 DN |
2364 | */ |
2365 | mb(); | |
2366 | ||
2367 | /* | |
2368 | * See if this message is next in line to be acknowledged as having | |
2369 | * been delivered. | |
2370 | */ | |
a47d5dac | 2371 | get = ch->sn.sn2.local_GP->get; |
33ba3c77 DN |
2372 | if (get == msg_number) |
2373 | xpc_acknowledge_msgs_sn2(ch, get, msg->flags); | |
2374 | } | |
2375 | ||
6e41017a | 2376 | int |
94bd2708 DN |
2377 | xpc_init_sn2(void) |
2378 | { | |
6e41017a | 2379 | int ret; |
ee6665e3 | 2380 | size_t buf_size; |
6e41017a | 2381 | |
94bd2708 | 2382 | xpc_rsvd_page_init = xpc_rsvd_page_init_sn2; |
33ba3c77 DN |
2383 | xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; |
2384 | xpc_offline_heartbeat = xpc_offline_heartbeat_sn2; | |
2385 | xpc_online_heartbeat = xpc_online_heartbeat_sn2; | |
2386 | xpc_heartbeat_init = xpc_heartbeat_init_sn2; | |
2387 | xpc_heartbeat_exit = xpc_heartbeat_exit_sn2; | |
2388 | xpc_check_remote_hb = xpc_check_remote_hb_sn2; | |
2389 | ||
a47d5dac DN |
2390 | xpc_request_partition_activation = xpc_request_partition_activation_sn2; |
2391 | xpc_request_partition_reactivation = | |
2392 | xpc_request_partition_reactivation_sn2; | |
2393 | xpc_request_partition_deactivation = | |
2394 | xpc_request_partition_deactivation_sn2; | |
2395 | xpc_cancel_partition_deactivation_request = | |
2396 | xpc_cancel_partition_deactivation_request_sn2; | |
2397 | ||
6e41017a | 2398 | xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2; |
e17d416b DN |
2399 | xpc_setup_infrastructure = xpc_setup_infrastructure_sn2; |
2400 | xpc_teardown_infrastructure = xpc_teardown_infrastructure_sn2; | |
2401 | xpc_make_first_contact = xpc_make_first_contact_sn2; | |
7fb5e59d | 2402 | xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2; |
185c3a1b DN |
2403 | xpc_allocate_msgqueues = xpc_allocate_msgqueues_sn2; |
2404 | xpc_free_msgqueues = xpc_free_msgqueues_sn2; | |
a47d5dac | 2405 | xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2; |
7fb5e59d | 2406 | xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2; |
a47d5dac | 2407 | xpc_n_of_deliverable_msgs = xpc_n_of_deliverable_msgs_sn2; |
e17d416b | 2408 | xpc_get_deliverable_msg = xpc_get_deliverable_msg_sn2; |
33ba3c77 | 2409 | |
a47d5dac | 2410 | xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2; |
33ba3c77 | 2411 | xpc_partition_engaged = xpc_partition_engaged_sn2; |
a47d5dac DN |
2412 | xpc_any_partition_engaged = xpc_any_partition_engaged_sn2; |
2413 | xpc_indicate_partition_disengaged = | |
2414 | xpc_indicate_partition_disengaged_sn2; | |
2415 | xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2; | |
2416 | ||
7fb5e59d DN |
2417 | xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2; |
2418 | xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2; | |
2419 | xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2; | |
2420 | xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2; | |
33ba3c77 | 2421 | |
33ba3c77 DN |
2422 | xpc_send_msg = xpc_send_msg_sn2; |
2423 | xpc_received_msg = xpc_received_msg_sn2; | |
6e41017a | 2424 | |
ee6665e3 DN |
2425 | buf_size = max(XPC_RP_VARS_SIZE, |
2426 | XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2); | |
2427 | xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size, | |
2428 | GFP_KERNEL, | |
2429 | &xpc_remote_copy_buffer_base_sn2); | |
2430 | if (xpc_remote_copy_buffer_sn2 == NULL) { | |
2431 | dev_err(xpc_part, "can't get memory for remote copy buffer\n"); | |
2432 | return -ENOMEM; | |
2433 | } | |
2434 | ||
c39838ce | 2435 | /* open up protections for IPI and [potentially] amo operations */ |
6e41017a | 2436 | xpc_allow_IPI_ops_sn2(); |
c39838ce | 2437 | xpc_allow_amo_ops_shub_wars_1_1_sn2(); |
6e41017a DN |
2438 | |
2439 | /* | |
2440 | * This is safe to do before the xpc_hb_checker thread has started | |
2441 | * because the handler releases a wait queue. If an interrupt is | |
2442 | * received before the thread is waiting, it will not go to sleep, | |
2443 | * but rather immediately process the interrupt. | |
2444 | */ | |
2445 | ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0, | |
2446 | "xpc hb", NULL); | |
2447 | if (ret != 0) { | |
2448 | dev_err(xpc_part, "can't register ACTIVATE IRQ handler, " | |
2449 | "errno=%d\n", -ret); | |
2450 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2451 | kfree(xpc_remote_copy_buffer_base_sn2); |
6e41017a DN |
2452 | } |
2453 | return ret; | |
94bd2708 DN |
2454 | } |
2455 | ||
2456 | void | |
2457 | xpc_exit_sn2(void) | |
2458 | { | |
6e41017a DN |
2459 | free_irq(SGI_XPC_ACTIVATE, NULL); |
2460 | xpc_disallow_IPI_ops_sn2(); | |
ee6665e3 | 2461 | kfree(xpc_remote_copy_buffer_base_sn2); |
94bd2708 | 2462 | } |