]>
Commit | Line | Data |
---|---|---|
22e4ebb9 MD |
1 | /* |
2 | * Copyright (C) 2010-2017 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
3 | * | |
4 | * membarrier system call | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License as published by | |
8 | * the Free Software Foundation; either version 2 of the License, or | |
9 | * (at your option) any later version. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | */ | |
16 | ||
17 | #include <linux/syscalls.h> | |
18 | #include <linux/membarrier.h> | |
19 | #include <linux/tick.h> | |
20 | #include <linux/cpumask.h> | |
21 | ||
22 | #include "sched.h" /* for cpu_rq(). */ | |
23 | ||
24 | /* | |
25 | * Bitmask made from a "or" of all commands within enum membarrier_cmd, | |
26 | * except MEMBARRIER_CMD_QUERY. | |
27 | */ | |
28 | #define MEMBARRIER_CMD_BITMASK \ | |
29 | (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED) | |
30 | ||
31 | static void ipi_mb(void *info) | |
32 | { | |
33 | smp_mb(); /* IPIs should be serializing but paranoid. */ | |
34 | } | |
35 | ||
36 | static void membarrier_private_expedited(void) | |
37 | { | |
38 | int cpu; | |
39 | bool fallback = false; | |
40 | cpumask_var_t tmpmask; | |
41 | ||
42 | if (num_online_cpus() == 1) | |
43 | return; | |
44 | ||
45 | /* | |
46 | * Matches memory barriers around rq->curr modification in | |
47 | * scheduler. | |
48 | */ | |
49 | smp_mb(); /* system call entry is not a mb. */ | |
50 | ||
51 | /* | |
52 | * Expedited membarrier commands guarantee that they won't | |
53 | * block, hence the GFP_NOWAIT allocation flag and fallback | |
54 | * implementation. | |
55 | */ | |
56 | if (!zalloc_cpumask_var(&tmpmask, GFP_NOWAIT)) { | |
57 | /* Fallback for OOM. */ | |
58 | fallback = true; | |
59 | } | |
60 | ||
61 | cpus_read_lock(); | |
62 | for_each_online_cpu(cpu) { | |
63 | struct task_struct *p; | |
64 | ||
65 | /* | |
66 | * Skipping the current CPU is OK even through we can be | |
67 | * migrated at any point. The current CPU, at the point | |
68 | * where we read raw_smp_processor_id(), is ensured to | |
69 | * be in program order with respect to the caller | |
70 | * thread. Therefore, we can skip this CPU from the | |
71 | * iteration. | |
72 | */ | |
73 | if (cpu == raw_smp_processor_id()) | |
74 | continue; | |
75 | rcu_read_lock(); | |
76 | p = task_rcu_dereference(&cpu_rq(cpu)->curr); | |
77 | if (p && p->mm == current->mm) { | |
78 | if (!fallback) | |
79 | __cpumask_set_cpu(cpu, tmpmask); | |
80 | else | |
81 | smp_call_function_single(cpu, ipi_mb, NULL, 1); | |
82 | } | |
83 | rcu_read_unlock(); | |
84 | } | |
85 | if (!fallback) { | |
86 | smp_call_function_many(tmpmask, ipi_mb, NULL, 1); | |
87 | free_cpumask_var(tmpmask); | |
88 | } | |
89 | cpus_read_unlock(); | |
90 | ||
91 | /* | |
92 | * Memory barrier on the caller thread _after_ we finished | |
93 | * waiting for the last IPI. Matches memory barriers around | |
94 | * rq->curr modification in scheduler. | |
95 | */ | |
96 | smp_mb(); /* exit from system call is not a mb */ | |
97 | } | |
98 | ||
99 | /** | |
100 | * sys_membarrier - issue memory barriers on a set of threads | |
101 | * @cmd: Takes command values defined in enum membarrier_cmd. | |
102 | * @flags: Currently needs to be 0. For future extensions. | |
103 | * | |
104 | * If this system call is not implemented, -ENOSYS is returned. If the | |
105 | * command specified does not exist, not available on the running | |
106 | * kernel, or if the command argument is invalid, this system call | |
107 | * returns -EINVAL. For a given command, with flags argument set to 0, | |
108 | * this system call is guaranteed to always return the same value until | |
109 | * reboot. | |
110 | * | |
111 | * All memory accesses performed in program order from each targeted thread | |
112 | * is guaranteed to be ordered with respect to sys_membarrier(). If we use | |
113 | * the semantic "barrier()" to represent a compiler barrier forcing memory | |
114 | * accesses to be performed in program order across the barrier, and | |
115 | * smp_mb() to represent explicit memory barriers forcing full memory | |
116 | * ordering across the barrier, we have the following ordering table for | |
117 | * each pair of barrier(), sys_membarrier() and smp_mb(): | |
118 | * | |
119 | * The pair ordering is detailed as (O: ordered, X: not ordered): | |
120 | * | |
121 | * barrier() smp_mb() sys_membarrier() | |
122 | * barrier() X X O | |
123 | * smp_mb() X O O | |
124 | * sys_membarrier() O O O | |
125 | */ | |
126 | SYSCALL_DEFINE2(membarrier, int, cmd, int, flags) | |
127 | { | |
128 | if (unlikely(flags)) | |
129 | return -EINVAL; | |
130 | switch (cmd) { | |
131 | case MEMBARRIER_CMD_QUERY: | |
132 | { | |
133 | int cmd_mask = MEMBARRIER_CMD_BITMASK; | |
134 | ||
135 | if (tick_nohz_full_enabled()) | |
136 | cmd_mask &= ~MEMBARRIER_CMD_SHARED; | |
137 | return cmd_mask; | |
138 | } | |
139 | case MEMBARRIER_CMD_SHARED: | |
140 | /* MEMBARRIER_CMD_SHARED is not compatible with nohz_full. */ | |
141 | if (tick_nohz_full_enabled()) | |
142 | return -EINVAL; | |
143 | if (num_online_cpus() > 1) | |
144 | synchronize_sched(); | |
145 | return 0; | |
146 | case MEMBARRIER_CMD_PRIVATE_EXPEDITED: | |
147 | membarrier_private_expedited(); | |
148 | return 0; | |
149 | default: | |
150 | return -EINVAL; | |
151 | } | |
152 | } |