]>
Commit | Line | Data |
---|---|---|
1e4df1fa FG |
1 | From 120f27d6c4ff44d31052fc74438efa64b361980a Mon Sep 17 00:00:00 2001 |
2 | From: Tejun Heo <tj@kernel.org> | |
3 | Date: Tue, 12 Jul 2016 17:03:15 +0100 | |
4 | Subject: [PATCH 1/2] percpu: fix synchronization between | |
5 | chunk->map_extend_work and chunk destruction | |
6 | ||
7 | Atomic allocations can trigger async map extensions which is serviced | |
8 | by chunk->map_extend_work. pcpu_balance_work which is responsible for | |
9 | destroying idle chunks wasn't synchronizing properly against | |
10 | chunk->map_extend_work and may end up freeing the chunk while the work | |
11 | item is still in flight. | |
12 | ||
13 | This patch fixes the bug by rolling async map extension operations | |
14 | into pcpu_balance_work. | |
15 | ||
16 | Signed-off-by: Tejun Heo <tj@kernel.org> | |
17 | Reported-and-tested-by: Alexei Starovoitov <alexei.starovoitov@gmail.com> | |
18 | Reported-by: Vlastimil Babka <vbabka@suse.cz> | |
19 | Reported-by: Sasha Levin <sasha.levin@oracle.com> | |
20 | Cc: stable@vger.kernel.org # v3.18+ | |
21 | Fixes: 9c824b6a172c ("percpu: make sure chunk->map array has available space") | |
22 | (cherry picked from commit 4f996e234dad488e5d9ba0858bc1bae12eff82c3) | |
23 | CVE-2016-4794 | |
24 | BugLink: https://bugs.launchpad.net/bugs/1581871 | |
25 | Signed-off-by: Luis Henriques <luis.henriques@canonical.com> | |
26 | Acked-by: Christopher Arges <chris.j.arges@canonical.com> | |
27 | Signed-off-by: Kamal Mostafa <kamal@canonical.com> | |
28 | --- | |
29 | mm/percpu.c | 57 ++++++++++++++++++++++++++++++++++++--------------------- | |
30 | 1 file changed, 36 insertions(+), 21 deletions(-) | |
31 | ||
32 | diff --git a/mm/percpu.c b/mm/percpu.c | |
33 | index 8a943b9..58b0149 100644 | |
34 | --- a/mm/percpu.c | |
35 | +++ b/mm/percpu.c | |
36 | @@ -110,7 +110,7 @@ struct pcpu_chunk { | |
37 | int map_used; /* # of map entries used before the sentry */ | |
38 | int map_alloc; /* # of map entries allocated */ | |
39 | int *map; /* allocation map */ | |
40 | - struct work_struct map_extend_work;/* async ->map[] extension */ | |
41 | + struct list_head map_extend_list;/* on pcpu_map_extend_chunks */ | |
42 | ||
43 | void *data; /* chunk data */ | |
44 | int first_free; /* no free below this */ | |
45 | @@ -164,6 +164,9 @@ static DEFINE_MUTEX(pcpu_alloc_mutex); /* chunk create/destroy, [de]pop */ | |
46 | ||
47 | static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */ | |
48 | ||
49 | +/* chunks which need their map areas extended, protected by pcpu_lock */ | |
50 | +static LIST_HEAD(pcpu_map_extend_chunks); | |
51 | + | |
52 | /* | |
53 | * The number of empty populated pages, protected by pcpu_lock. The | |
54 | * reserved chunk doesn't contribute to the count. | |
55 | @@ -397,13 +400,19 @@ static int pcpu_need_to_extend(struct pcpu_chunk *chunk, bool is_atomic) | |
56 | { | |
57 | int margin, new_alloc; | |
58 | ||
59 | + lockdep_assert_held(&pcpu_lock); | |
60 | + | |
61 | if (is_atomic) { | |
62 | margin = 3; | |
63 | ||
64 | if (chunk->map_alloc < | |
65 | - chunk->map_used + PCPU_ATOMIC_MAP_MARGIN_LOW && | |
66 | - pcpu_async_enabled) | |
67 | - schedule_work(&chunk->map_extend_work); | |
68 | + chunk->map_used + PCPU_ATOMIC_MAP_MARGIN_LOW) { | |
69 | + if (list_empty(&chunk->map_extend_list)) { | |
70 | + list_add_tail(&chunk->map_extend_list, | |
71 | + &pcpu_map_extend_chunks); | |
72 | + pcpu_schedule_balance_work(); | |
73 | + } | |
74 | + } | |
75 | } else { | |
76 | margin = PCPU_ATOMIC_MAP_MARGIN_HIGH; | |
77 | } | |
78 | @@ -469,20 +478,6 @@ out_unlock: | |
79 | return 0; | |
80 | } | |
81 | ||
82 | -static void pcpu_map_extend_workfn(struct work_struct *work) | |
83 | -{ | |
84 | - struct pcpu_chunk *chunk = container_of(work, struct pcpu_chunk, | |
85 | - map_extend_work); | |
86 | - int new_alloc; | |
87 | - | |
88 | - spin_lock_irq(&pcpu_lock); | |
89 | - new_alloc = pcpu_need_to_extend(chunk, false); | |
90 | - spin_unlock_irq(&pcpu_lock); | |
91 | - | |
92 | - if (new_alloc) | |
93 | - pcpu_extend_area_map(chunk, new_alloc); | |
94 | -} | |
95 | - | |
96 | /** | |
97 | * pcpu_fit_in_area - try to fit the requested allocation in a candidate area | |
98 | * @chunk: chunk the candidate area belongs to | |
99 | @@ -742,7 +737,7 @@ static struct pcpu_chunk *pcpu_alloc_chunk(void) | |
100 | chunk->map_used = 1; | |
101 | ||
102 | INIT_LIST_HEAD(&chunk->list); | |
103 | - INIT_WORK(&chunk->map_extend_work, pcpu_map_extend_workfn); | |
104 | + INIT_LIST_HEAD(&chunk->map_extend_list); | |
105 | chunk->free_size = pcpu_unit_size; | |
106 | chunk->contig_hint = pcpu_unit_size; | |
107 | ||
108 | @@ -1131,6 +1126,7 @@ static void pcpu_balance_workfn(struct work_struct *work) | |
109 | if (chunk == list_first_entry(free_head, struct pcpu_chunk, list)) | |
110 | continue; | |
111 | ||
112 | + list_del_init(&chunk->map_extend_list); | |
113 | list_move(&chunk->list, &to_free); | |
114 | } | |
115 | ||
116 | @@ -1148,6 +1144,25 @@ static void pcpu_balance_workfn(struct work_struct *work) | |
117 | pcpu_destroy_chunk(chunk); | |
118 | } | |
119 | ||
120 | + /* service chunks which requested async area map extension */ | |
121 | + do { | |
122 | + int new_alloc = 0; | |
123 | + | |
124 | + spin_lock_irq(&pcpu_lock); | |
125 | + | |
126 | + chunk = list_first_entry_or_null(&pcpu_map_extend_chunks, | |
127 | + struct pcpu_chunk, map_extend_list); | |
128 | + if (chunk) { | |
129 | + list_del_init(&chunk->map_extend_list); | |
130 | + new_alloc = pcpu_need_to_extend(chunk, false); | |
131 | + } | |
132 | + | |
133 | + spin_unlock_irq(&pcpu_lock); | |
134 | + | |
135 | + if (new_alloc) | |
136 | + pcpu_extend_area_map(chunk, new_alloc); | |
137 | + } while (chunk); | |
138 | + | |
139 | /* | |
140 | * Ensure there are certain number of free populated pages for | |
141 | * atomic allocs. Fill up from the most packed so that atomic | |
142 | @@ -1646,7 +1661,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |
143 | */ | |
144 | schunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0); | |
145 | INIT_LIST_HEAD(&schunk->list); | |
146 | - INIT_WORK(&schunk->map_extend_work, pcpu_map_extend_workfn); | |
147 | + INIT_LIST_HEAD(&schunk->map_extend_list); | |
148 | schunk->base_addr = base_addr; | |
149 | schunk->map = smap; | |
150 | schunk->map_alloc = ARRAY_SIZE(smap); | |
151 | @@ -1675,7 +1690,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, | |
152 | if (dyn_size) { | |
153 | dchunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0); | |
154 | INIT_LIST_HEAD(&dchunk->list); | |
155 | - INIT_WORK(&dchunk->map_extend_work, pcpu_map_extend_workfn); | |
156 | + INIT_LIST_HEAD(&dchunk->map_extend_list); | |
157 | dchunk->base_addr = base_addr; | |
158 | dchunk->map = dmap; | |
159 | dchunk->map_alloc = ARRAY_SIZE(dmap); | |
160 | -- | |
161 | 2.1.4 | |
162 |