]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - ubuntu/vbox/r0drv/linux/mp-r0drv-linux.c
UBUNTU: ubuntu: vbox -- update to 5.1.6-dfsg-1
[mirror_ubuntu-zesty-kernel.git] / ubuntu / vbox / r0drv / linux / mp-r0drv-linux.c
1 /* $Id: mp-r0drv-linux.c $ */
2 /** @file
3 * IPRT - Multiprocessor, Ring-0 Driver, Linux.
4 */
5
6 /*
7 * Copyright (C) 2008-2016 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28 /*********************************************************************************************************************************
29 * Header Files *
30 *********************************************************************************************************************************/
31 #include "the-linux-kernel.h"
32 #include "internal/iprt.h"
33
34 #include <iprt/mp.h>
35 #include <iprt/cpuset.h>
36 #include <iprt/err.h>
37 #include <iprt/asm.h>
38 #include <iprt/thread.h>
39 #include "r0drv/mp-r0drv.h"
40
41
42 RTDECL(RTCPUID) RTMpCpuId(void)
43 {
44 return smp_processor_id();
45 }
46 RT_EXPORT_SYMBOL(RTMpCpuId);
47
48
49 RTDECL(int) RTMpCurSetIndex(void)
50 {
51 return smp_processor_id();
52 }
53 RT_EXPORT_SYMBOL(RTMpCurSetIndex);
54
55
56 RTDECL(int) RTMpCurSetIndexAndId(PRTCPUID pidCpu)
57 {
58 return *pidCpu = smp_processor_id();
59 }
60 RT_EXPORT_SYMBOL(RTMpCurSetIndexAndId);
61
62
63 RTDECL(int) RTMpCpuIdToSetIndex(RTCPUID idCpu)
64 {
65 return idCpu < RTCPUSET_MAX_CPUS && idCpu < NR_CPUS ? (int)idCpu : -1;
66 }
67 RT_EXPORT_SYMBOL(RTMpCpuIdToSetIndex);
68
69
70 RTDECL(RTCPUID) RTMpCpuIdFromSetIndex(int iCpu)
71 {
72 return iCpu < NR_CPUS ? (RTCPUID)iCpu : NIL_RTCPUID;
73 }
74 RT_EXPORT_SYMBOL(RTMpCpuIdFromSetIndex);
75
76
77 RTDECL(RTCPUID) RTMpGetMaxCpuId(void)
78 {
79 return NR_CPUS - 1; //???
80 }
81 RT_EXPORT_SYMBOL(RTMpGetMaxCpuId);
82
83
84 RTDECL(bool) RTMpIsCpuPossible(RTCPUID idCpu)
85 {
86 #if defined(CONFIG_SMP)
87 if (RT_UNLIKELY(idCpu >= NR_CPUS))
88 return false;
89
90 # if defined(cpu_possible)
91 return cpu_possible(idCpu);
92 # else /* < 2.5.29 */
93 return idCpu < (RTCPUID)smp_num_cpus;
94 # endif
95 #else
96 return idCpu == RTMpCpuId();
97 #endif
98 }
99 RT_EXPORT_SYMBOL(RTMpIsCpuPossible);
100
101
102 RTDECL(PRTCPUSET) RTMpGetSet(PRTCPUSET pSet)
103 {
104 RTCPUID idCpu;
105
106 RTCpuSetEmpty(pSet);
107 idCpu = RTMpGetMaxCpuId();
108 do
109 {
110 if (RTMpIsCpuPossible(idCpu))
111 RTCpuSetAdd(pSet, idCpu);
112 } while (idCpu-- > 0);
113 return pSet;
114 }
115 RT_EXPORT_SYMBOL(RTMpGetSet);
116
117
118 RTDECL(RTCPUID) RTMpGetCount(void)
119 {
120 #ifdef CONFIG_SMP
121 # if defined(CONFIG_HOTPLUG_CPU) /* introduced & uses cpu_present */
122 return num_present_cpus();
123 # elif defined(num_possible_cpus)
124 return num_possible_cpus();
125 # elif LINUX_VERSION_CODE < KERNEL_VERSION(2, 5, 0)
126 return smp_num_cpus;
127 # else
128 RTCPUSET Set;
129 RTMpGetSet(&Set);
130 return RTCpuSetCount(&Set);
131 # endif
132 #else
133 return 1;
134 #endif
135 }
136 RT_EXPORT_SYMBOL(RTMpGetCount);
137
138
139 RTDECL(bool) RTMpIsCpuOnline(RTCPUID idCpu)
140 {
141 #ifdef CONFIG_SMP
142 if (RT_UNLIKELY(idCpu >= NR_CPUS))
143 return false;
144 # ifdef cpu_online
145 return cpu_online(idCpu);
146 # else /* 2.4: */
147 return cpu_online_map & RT_BIT_64(idCpu);
148 # endif
149 #else
150 return idCpu == RTMpCpuId();
151 #endif
152 }
153 RT_EXPORT_SYMBOL(RTMpIsCpuOnline);
154
155
156 RTDECL(PRTCPUSET) RTMpGetOnlineSet(PRTCPUSET pSet)
157 {
158 #ifdef CONFIG_SMP
159 RTCPUID idCpu;
160
161 RTCpuSetEmpty(pSet);
162 idCpu = RTMpGetMaxCpuId();
163 do
164 {
165 if (RTMpIsCpuOnline(idCpu))
166 RTCpuSetAdd(pSet, idCpu);
167 } while (idCpu-- > 0);
168 #else
169 RTCpuSetEmpty(pSet);
170 RTCpuSetAdd(pSet, RTMpCpuId());
171 #endif
172 return pSet;
173 }
174 RT_EXPORT_SYMBOL(RTMpGetOnlineSet);
175
176
177 RTDECL(RTCPUID) RTMpGetOnlineCount(void)
178 {
179 #ifdef CONFIG_SMP
180 # if defined(num_online_cpus)
181 return num_online_cpus();
182 # else
183 RTCPUSET Set;
184 RTMpGetOnlineSet(&Set);
185 return RTCpuSetCount(&Set);
186 # endif
187 #else
188 return 1;
189 #endif
190 }
191 RT_EXPORT_SYMBOL(RTMpGetOnlineCount);
192
193
194 RTDECL(bool) RTMpIsCpuWorkPending(void)
195 {
196 /** @todo (not used on non-Windows platforms yet). */
197 return false;
198 }
199 RT_EXPORT_SYMBOL(RTMpIsCpuWorkPending);
200
201
202 /**
203 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER.
204 *
205 * @param pvInfo Pointer to the RTMPARGS package.
206 */
207 static void rtmpLinuxWrapper(void *pvInfo)
208 {
209 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
210 ASMAtomicIncU32(&pArgs->cHits);
211 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
212 }
213
214
215 /**
216 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER, does hit
217 * increment after calling the worker.
218 *
219 * @param pvInfo Pointer to the RTMPARGS package.
220 */
221 static void rtmpLinuxWrapperPostInc(void *pvInfo)
222 {
223 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
224 pArgs->pfnWorker(RTMpCpuId(), pArgs->pvUser1, pArgs->pvUser2);
225 ASMAtomicIncU32(&pArgs->cHits);
226 }
227
228
229 /**
230 * Wrapper between the native linux all-cpu callbacks and PFNRTWORKER.
231 *
232 * @param pvInfo Pointer to the RTMPARGS package.
233 */
234 static void rtmpLinuxAllWrapper(void *pvInfo)
235 {
236 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
237 PRTCPUSET pWorkerSet = pArgs->pWorkerSet;
238 RTCPUID idCpu = RTMpCpuId();
239 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
240
241 if (RTCpuSetIsMember(pWorkerSet, idCpu))
242 {
243 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
244 RTCpuSetDel(pWorkerSet, idCpu);
245 }
246 }
247
248
249 RTDECL(int) RTMpOnAll(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
250 {
251 IPRT_LINUX_SAVE_EFL_AC();
252 int rc;
253 RTMPARGS Args;
254 RTCPUSET OnlineSet;
255 RTCPUID idCpu;
256 uint32_t cLoops;
257
258 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
259
260 Args.pfnWorker = pfnWorker;
261 Args.pvUser1 = pvUser1;
262 Args.pvUser2 = pvUser2;
263 Args.idCpu = NIL_RTCPUID;
264 Args.cHits = 0;
265
266 RTThreadPreemptDisable(&PreemptState);
267 RTMpGetOnlineSet(&OnlineSet);
268 Args.pWorkerSet = &OnlineSet;
269 idCpu = RTMpCpuId();
270
271 if (RTCpuSetCount(&OnlineSet) > 1)
272 {
273 /* Fire the function on all other CPUs without waiting for completion. */
274 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
275 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* wait */);
276 #else
277 rc = smp_call_function(rtmpLinuxAllWrapper, &Args, 0 /* retry */, 0 /* wait */);
278 #endif
279 Assert(!rc); NOREF(rc);
280 }
281
282 /* Fire the function on this CPU. */
283 Args.pfnWorker(idCpu, Args.pvUser1, Args.pvUser2);
284 RTCpuSetDel(Args.pWorkerSet, idCpu);
285
286 /* Wait for all of them finish. */
287 cLoops = 64000;
288 while (!RTCpuSetIsEmpty(Args.pWorkerSet))
289 {
290 /* Periodically check if any CPU in the wait set has gone offline, if so update the wait set. */
291 if (!cLoops--)
292 {
293 RTCPUSET OnlineSetNow;
294 RTMpGetOnlineSet(&OnlineSetNow);
295 RTCpuSetAnd(Args.pWorkerSet, &OnlineSetNow);
296
297 cLoops = 64000;
298 }
299
300 ASMNopPause();
301 }
302
303 RTThreadPreemptRestore(&PreemptState);
304 IPRT_LINUX_RESTORE_EFL_AC();
305 return VINF_SUCCESS;
306 }
307 RT_EXPORT_SYMBOL(RTMpOnAll);
308
309
310 RTDECL(int) RTMpOnOthers(PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
311 {
312 IPRT_LINUX_SAVE_EFL_AC();
313 int rc;
314 RTMPARGS Args;
315
316 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
317 Args.pfnWorker = pfnWorker;
318 Args.pvUser1 = pvUser1;
319 Args.pvUser2 = pvUser2;
320 Args.idCpu = NIL_RTCPUID;
321 Args.cHits = 0;
322
323 RTThreadPreemptDisable(&PreemptState);
324 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
325 rc = smp_call_function(rtmpLinuxWrapper, &Args, 1 /* wait */);
326 #else /* older kernels */
327 rc = smp_call_function(rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
328 #endif /* older kernels */
329 RTThreadPreemptRestore(&PreemptState);
330
331 Assert(rc == 0); NOREF(rc);
332 IPRT_LINUX_RESTORE_EFL_AC();
333 return VINF_SUCCESS;
334 }
335 RT_EXPORT_SYMBOL(RTMpOnOthers);
336
337
338 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 27)
339 /**
340 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
341 * employed by RTMpOnPair on older kernels that lacks smp_call_function_many.
342 *
343 * @param pvInfo Pointer to the RTMPARGS package.
344 */
345 static void rtMpLinuxOnPairWrapper(void *pvInfo)
346 {
347 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
348 RTCPUID idCpu = RTMpCpuId();
349
350 if ( idCpu == pArgs->idCpu
351 || idCpu == pArgs->idCpu2)
352 {
353 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
354 ASMAtomicIncU32(&pArgs->cHits);
355 }
356 }
357 #endif
358
359
360 RTDECL(int) RTMpOnPair(RTCPUID idCpu1, RTCPUID idCpu2, uint32_t fFlags, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
361 {
362 IPRT_LINUX_SAVE_EFL_AC();
363 int rc;
364 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
365
366 AssertReturn(idCpu1 != idCpu2, VERR_INVALID_PARAMETER);
367 AssertReturn(!(fFlags & RTMPON_F_VALID_MASK), VERR_INVALID_FLAGS);
368
369 /*
370 * Check that both CPUs are online before doing the broadcast call.
371 */
372 RTThreadPreemptDisable(&PreemptState);
373 if ( RTMpIsCpuOnline(idCpu1)
374 && RTMpIsCpuOnline(idCpu2))
375 {
376 /*
377 * Use the smp_call_function variant taking a cpu mask where available,
378 * falling back on broadcast with filter. Slight snag if one of the
379 * CPUs is the one we're running on, we must do the call and the post
380 * call wait ourselves.
381 */
382 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
383 cpumask_t DstCpuMask;
384 #endif
385 RTCPUID idCpuSelf = RTMpCpuId();
386 bool const fCallSelf = idCpuSelf == idCpu1 || idCpuSelf == idCpu2;
387 RTMPARGS Args;
388 Args.pfnWorker = pfnWorker;
389 Args.pvUser1 = pvUser1;
390 Args.pvUser2 = pvUser2;
391 Args.idCpu = idCpu1;
392 Args.idCpu2 = idCpu2;
393 Args.cHits = 0;
394
395 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
396 cpumask_clear(&DstCpuMask);
397 cpumask_set_cpu(idCpu1, &DstCpuMask);
398 cpumask_set_cpu(idCpu2, &DstCpuMask);
399 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
400 cpus_clear(DstCpuMask);
401 cpu_set(idCpu1, DstCpuMask);
402 cpu_set(idCpu2, DstCpuMask);
403 #endif
404
405 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 29)
406 smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
407 rc = 0;
408 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 28)
409 rc = smp_call_function_many(&DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
410 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
411 rc = smp_call_function_mask(DstCpuMask, rtmpLinuxWrapperPostInc, &Args, !fCallSelf /* wait */);
412 #else /* older kernels */
413 rc = smp_call_function(rtMpLinuxOnPairWrapper, &Args, 0 /* retry */, !fCallSelf /* wait */);
414 #endif /* older kernels */
415 Assert(rc == 0);
416
417 /* Call ourselves if necessary and wait for the other party to be done. */
418 if (fCallSelf)
419 {
420 uint32_t cLoops = 0;
421 rtmpLinuxWrapper(&Args);
422 while (ASMAtomicReadU32(&Args.cHits) < 2)
423 {
424 if ((cLoops & 0x1ff) == 0 && !RTMpIsCpuOnline(idCpuSelf == idCpu1 ? idCpu2 : idCpu1))
425 break;
426 cLoops++;
427 ASMNopPause();
428 }
429 }
430
431 Assert(Args.cHits <= 2);
432 if (Args.cHits == 2)
433 rc = VINF_SUCCESS;
434 else if (Args.cHits == 1)
435 rc = VERR_NOT_ALL_CPUS_SHOWED;
436 else if (Args.cHits == 0)
437 rc = VERR_CPU_OFFLINE;
438 else
439 rc = VERR_CPU_IPE_1;
440 }
441 /*
442 * A CPU must be present to be considered just offline.
443 */
444 else if ( RTMpIsCpuPresent(idCpu1)
445 && RTMpIsCpuPresent(idCpu2))
446 rc = VERR_CPU_OFFLINE;
447 else
448 rc = VERR_CPU_NOT_FOUND;
449 RTThreadPreemptRestore(&PreemptState);;
450 IPRT_LINUX_RESTORE_EFL_AC();
451 return rc;
452 }
453 RT_EXPORT_SYMBOL(RTMpOnPair);
454
455
456 RTDECL(bool) RTMpOnPairIsConcurrentExecSupported(void)
457 {
458 return true;
459 }
460 RT_EXPORT_SYMBOL(RTMpOnPairIsConcurrentExecSupported);
461
462
463 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 19)
464 /**
465 * Wrapper between the native linux per-cpu callbacks and PFNRTWORKER
466 * employed by RTMpOnSpecific on older kernels that lacks smp_call_function_single.
467 *
468 * @param pvInfo Pointer to the RTMPARGS package.
469 */
470 static void rtmpOnSpecificLinuxWrapper(void *pvInfo)
471 {
472 PRTMPARGS pArgs = (PRTMPARGS)pvInfo;
473 RTCPUID idCpu = RTMpCpuId();
474
475 if (idCpu == pArgs->idCpu)
476 {
477 pArgs->pfnWorker(idCpu, pArgs->pvUser1, pArgs->pvUser2);
478 ASMAtomicIncU32(&pArgs->cHits);
479 }
480 }
481 #endif
482
483
484 RTDECL(int) RTMpOnSpecific(RTCPUID idCpu, PFNRTMPWORKER pfnWorker, void *pvUser1, void *pvUser2)
485 {
486 IPRT_LINUX_SAVE_EFL_AC();
487 int rc;
488 RTMPARGS Args;
489
490 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
491 Args.pfnWorker = pfnWorker;
492 Args.pvUser1 = pvUser1;
493 Args.pvUser2 = pvUser2;
494 Args.idCpu = idCpu;
495 Args.cHits = 0;
496
497 if (!RTMpIsCpuPossible(idCpu))
498 return VERR_CPU_NOT_FOUND;
499
500 RTThreadPreemptDisable(&PreemptState);
501 if (idCpu != RTMpCpuId())
502 {
503 if (RTMpIsCpuOnline(idCpu))
504 {
505 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
506 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 1 /* wait */);
507 #elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
508 rc = smp_call_function_single(idCpu, rtmpLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
509 #else /* older kernels */
510 rc = smp_call_function(rtmpOnSpecificLinuxWrapper, &Args, 0 /* retry */, 1 /* wait */);
511 #endif /* older kernels */
512 Assert(rc == 0);
513 rc = Args.cHits ? VINF_SUCCESS : VERR_CPU_OFFLINE;
514 }
515 else
516 rc = VERR_CPU_OFFLINE;
517 }
518 else
519 {
520 rtmpLinuxWrapper(&Args);
521 rc = VINF_SUCCESS;
522 }
523 RTThreadPreemptRestore(&PreemptState);;
524
525 NOREF(rc);
526 IPRT_LINUX_RESTORE_EFL_AC();
527 return rc;
528 }
529 RT_EXPORT_SYMBOL(RTMpOnSpecific);
530
531
532 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
533 /**
534 * Dummy callback used by RTMpPokeCpu.
535 *
536 * @param pvInfo Ignored.
537 */
538 static void rtmpLinuxPokeCpuCallback(void *pvInfo)
539 {
540 NOREF(pvInfo);
541 }
542 #endif
543
544
545 RTDECL(int) RTMpPokeCpu(RTCPUID idCpu)
546 {
547 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
548 int rc;
549 IPRT_LINUX_SAVE_EFL_AC();
550
551 if (!RTMpIsCpuPossible(idCpu))
552 return VERR_CPU_NOT_FOUND;
553 if (!RTMpIsCpuOnline(idCpu))
554 return VERR_CPU_OFFLINE;
555
556 # if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 27)
557 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* wait */);
558 # elif LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 19)
559 rc = smp_call_function_single(idCpu, rtmpLinuxPokeCpuCallback, NULL, 0 /* retry */, 0 /* wait */);
560 # else /* older kernels */
561 # error oops
562 # endif /* older kernels */
563 NOREF(rc);
564 Assert(rc == 0);
565 IPRT_LINUX_RESTORE_EFL_AC();
566 return VINF_SUCCESS;
567
568 #else /* older kernels */
569 /* no unicast here? */
570 return VERR_NOT_SUPPORTED;
571 #endif /* older kernels */
572 }
573 RT_EXPORT_SYMBOL(RTMpPokeCpu);
574
575
576 RTDECL(bool) RTMpOnAllIsConcurrentSafe(void)
577 {
578 return true;
579 }
580 RT_EXPORT_SYMBOL(RTMpOnAllIsConcurrentSafe);
581