]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - kernel/padata.c
arm64: mm: Invalidate both kernel and user ASIDs when performing TLBI
[mirror_ubuntu-artful-kernel.git] / kernel / padata.c
CommitLineData
16295bec
SK
1/*
2 * padata.c - generic interface to process data streams in parallel
3 *
107f8bda
SK
4 * See Documentation/padata.txt for an api documentation.
5 *
16295bec
SK
6 * Copyright (C) 2008, 2009 secunet Security Networks AG
7 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
8 *
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
21 */
22
9984de1a 23#include <linux/export.h>
16295bec
SK
24#include <linux/cpumask.h>
25#include <linux/err.h>
26#include <linux/cpu.h>
27#include <linux/padata.h>
28#include <linux/mutex.h>
29#include <linux/sched.h>
5a0e3ad6 30#include <linux/slab.h>
5e017dc3 31#include <linux/sysfs.h>
16295bec 32#include <linux/rcupdate.h>
30e92153 33#include <linux/module.h>
16295bec 34
97e3d94a 35#define MAX_OBJ_NUM 1000
16295bec
SK
36
37static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
38{
39 int cpu, target_cpu;
40
e15bacbe 41 target_cpu = cpumask_first(pd->cpumask.pcpu);
16295bec 42 for (cpu = 0; cpu < cpu_index; cpu++)
e15bacbe 43 target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);
16295bec
SK
44
45 return target_cpu;
46}
47
2dc9b5db 48static int padata_cpu_hash(struct parallel_data *pd)
16295bec 49{
0b6b098e 50 unsigned int seq_nr;
16295bec 51 int cpu_index;
16295bec
SK
52
53 /*
54 * Hash the sequence numbers to the cpus by taking
55 * seq_nr mod. number of cpus in use.
56 */
2dc9b5db 57
0b6b098e
MK
58 seq_nr = atomic_inc_return(&pd->seq_nr);
59 cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
16295bec
SK
60
61 return padata_index_to_cpu(pd, cpu_index);
62}
63
e15bacbe 64static void padata_parallel_worker(struct work_struct *parallel_work)
16295bec 65{
e15bacbe 66 struct padata_parallel_queue *pqueue;
16295bec
SK
67 LIST_HEAD(local_list);
68
69 local_bh_disable();
e15bacbe
DK
70 pqueue = container_of(parallel_work,
71 struct padata_parallel_queue, work);
16295bec 72
e15bacbe
DK
73 spin_lock(&pqueue->parallel.lock);
74 list_replace_init(&pqueue->parallel.list, &local_list);
75 spin_unlock(&pqueue->parallel.lock);
16295bec
SK
76
77 while (!list_empty(&local_list)) {
78 struct padata_priv *padata;
79
80 padata = list_entry(local_list.next,
81 struct padata_priv, list);
82
83 list_del_init(&padata->list);
84
85 padata->parallel(padata);
86 }
87
88 local_bh_enable();
89}
90
0198ffd1 91/**
16295bec
SK
92 * padata_do_parallel - padata parallelization function
93 *
94 * @pinst: padata instance
95 * @padata: object to be parallelized
96 * @cb_cpu: cpu the serialization callback function will run on,
e15bacbe 97 * must be in the serial cpumask of padata(i.e. cpumask.cbcpu).
16295bec
SK
98 *
99 * The parallelization callback function will run with BHs off.
100 * Note: Every object which is parallelized by padata_do_parallel
101 * must be seen by padata_do_serial.
102 */
103int padata_do_parallel(struct padata_instance *pinst,
104 struct padata_priv *padata, int cb_cpu)
105{
106 int target_cpu, err;
e15bacbe 107 struct padata_parallel_queue *queue;
16295bec
SK
108 struct parallel_data *pd;
109
110 rcu_read_lock_bh();
111
c0e656b7 112 pd = rcu_dereference_bh(pinst->pd);
16295bec 113
83f619f3 114 err = -EINVAL;
7424713b 115 if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
16295bec
SK
116 goto out;
117
e15bacbe 118 if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu))
16295bec
SK
119 goto out;
120
121 err = -EBUSY;
122 if ((pinst->flags & PADATA_RESET))
123 goto out;
124
125 if (atomic_read(&pd->refcnt) >= MAX_OBJ_NUM)
126 goto out;
127
83f619f3 128 err = 0;
16295bec
SK
129 atomic_inc(&pd->refcnt);
130 padata->pd = pd;
131 padata->cb_cpu = cb_cpu;
132
2dc9b5db 133 target_cpu = padata_cpu_hash(pd);
e15bacbe 134 queue = per_cpu_ptr(pd->pqueue, target_cpu);
16295bec
SK
135
136 spin_lock(&queue->parallel.lock);
137 list_add_tail(&padata->list, &queue->parallel.list);
138 spin_unlock(&queue->parallel.lock);
139
e15bacbe 140 queue_work_on(target_cpu, pinst->wq, &queue->work);
16295bec
SK
141
142out:
143 rcu_read_unlock_bh();
144
145 return err;
146}
147EXPORT_SYMBOL(padata_do_parallel);
148
0198ffd1
SK
149/*
150 * padata_get_next - Get the next object that needs serialization.
151 *
152 * Return values are:
153 *
154 * A pointer to the control struct of the next object that needs
155 * serialization, if present in one of the percpu reorder queues.
156 *
0198ffd1
SK
157 * -EINPROGRESS, if the next object that needs serialization will
158 * be parallel processed by another cpu and is not yet present in
159 * the cpu's reorder queue.
160 *
161 * -ENODATA, if this cpu has to do the parallel processing for
162 * the next object.
163 */
16295bec
SK
164static struct padata_priv *padata_get_next(struct parallel_data *pd)
165{
5f1a8c1b 166 int cpu, num_cpus;
2dc9b5db 167 unsigned int next_nr, next_index;
f0fcf200 168 struct padata_parallel_queue *next_queue;
16295bec
SK
169 struct padata_priv *padata;
170 struct padata_list *reorder;
171
e15bacbe 172 num_cpus = cpumask_weight(pd->cpumask.pcpu);
16295bec 173
5f1a8c1b
SK
174 /*
175 * Calculate the percpu reorder queue and the sequence
176 * number of the next object.
177 */
178 next_nr = pd->processed;
179 next_index = next_nr % num_cpus;
180 cpu = padata_index_to_cpu(pd, next_index);
e15bacbe 181 next_queue = per_cpu_ptr(pd->pqueue, cpu);
5f1a8c1b 182
16295bec
SK
183 reorder = &next_queue->reorder;
184
de5540d0 185 spin_lock(&reorder->lock);
16295bec
SK
186 if (!list_empty(&reorder->list)) {
187 padata = list_entry(reorder->list.next,
188 struct padata_priv, list);
189
16295bec
SK
190 list_del_init(&padata->list);
191 atomic_dec(&pd->reorder_objects);
16295bec 192
5f1a8c1b 193 pd->processed++;
16295bec 194
de5540d0 195 spin_unlock(&reorder->lock);
16295bec
SK
196 goto out;
197 }
de5540d0 198 spin_unlock(&reorder->lock);
16295bec 199
f0fcf200 200 if (__this_cpu_read(pd->pqueue->cpu_index) == next_queue->cpu_index) {
16295bec
SK
201 padata = ERR_PTR(-ENODATA);
202 goto out;
203 }
204
205 padata = ERR_PTR(-EINPROGRESS);
206out:
207 return padata;
208}
209
210static void padata_reorder(struct parallel_data *pd)
211{
3047817b 212 int cb_cpu;
16295bec 213 struct padata_priv *padata;
e15bacbe 214 struct padata_serial_queue *squeue;
16295bec
SK
215 struct padata_instance *pinst = pd->pinst;
216
0198ffd1
SK
217 /*
218 * We need to ensure that only one cpu can work on dequeueing of
219 * the reorder queue the time. Calculating in which percpu reorder
220 * queue the next object will arrive takes some time. A spinlock
221 * would be highly contended. Also it is not clear in which order
222 * the objects arrive to the reorder queues. So a cpu could wait to
223 * get the lock just to notice that there is nothing to do at the
224 * moment. Therefore we use a trylock and let the holder of the lock
225 * care for all the objects enqueued during the holdtime of the lock.
226 */
16295bec 227 if (!spin_trylock_bh(&pd->lock))
d46a5ac7 228 return;
16295bec
SK
229
230 while (1) {
231 padata = padata_get_next(pd);
232
0198ffd1 233 /*
69b34844
JD
234 * If the next object that needs serialization is parallel
235 * processed by another cpu and is still on it's way to the
236 * cpu's reorder queue, nothing to do for now.
0198ffd1 237 */
69b34844 238 if (PTR_ERR(padata) == -EINPROGRESS)
16295bec
SK
239 break;
240
0198ffd1
SK
241 /*
242 * This cpu has to do the parallel processing of the next
243 * object. It's waiting in the cpu's parallelization queue,
25985edc 244 * so exit immediately.
0198ffd1 245 */
16295bec 246 if (PTR_ERR(padata) == -ENODATA) {
d46a5ac7 247 del_timer(&pd->timer);
16295bec 248 spin_unlock_bh(&pd->lock);
d46a5ac7 249 return;
16295bec
SK
250 }
251
3047817b
SK
252 cb_cpu = padata->cb_cpu;
253 squeue = per_cpu_ptr(pd->squeue, cb_cpu);
16295bec 254
e15bacbe
DK
255 spin_lock(&squeue->serial.lock);
256 list_add_tail(&padata->list, &squeue->serial.list);
257 spin_unlock(&squeue->serial.lock);
16295bec 258
3047817b 259 queue_work_on(cb_cpu, pinst->wq, &squeue->work);
16295bec
SK
260 }
261
262 spin_unlock_bh(&pd->lock);
263
0198ffd1
SK
264 /*
265 * The next object that needs serialization might have arrived to
266 * the reorder queues in the meantime, we will be called again
25985edc 267 * from the timer function if no one else cares for it.
0198ffd1 268 */
d46a5ac7
SK
269 if (atomic_read(&pd->reorder_objects)
270 && !(pinst->flags & PADATA_RESET))
271 mod_timer(&pd->timer, jiffies + HZ);
272 else
273 del_timer(&pd->timer);
16295bec 274
16295bec
SK
275 return;
276}
277
d46a5ac7
SK
278static void padata_reorder_timer(unsigned long arg)
279{
280 struct parallel_data *pd = (struct parallel_data *)arg;
281
282 padata_reorder(pd);
283}
284
e15bacbe 285static void padata_serial_worker(struct work_struct *serial_work)
16295bec 286{
e15bacbe 287 struct padata_serial_queue *squeue;
16295bec
SK
288 struct parallel_data *pd;
289 LIST_HEAD(local_list);
290
291 local_bh_disable();
e15bacbe
DK
292 squeue = container_of(serial_work, struct padata_serial_queue, work);
293 pd = squeue->pd;
16295bec 294
e15bacbe
DK
295 spin_lock(&squeue->serial.lock);
296 list_replace_init(&squeue->serial.list, &local_list);
297 spin_unlock(&squeue->serial.lock);
16295bec
SK
298
299 while (!list_empty(&local_list)) {
300 struct padata_priv *padata;
301
302 padata = list_entry(local_list.next,
303 struct padata_priv, list);
304
305 list_del_init(&padata->list);
306
307 padata->serial(padata);
308 atomic_dec(&pd->refcnt);
309 }
310 local_bh_enable();
311}
312
0198ffd1 313/**
16295bec
SK
314 * padata_do_serial - padata serialization function
315 *
316 * @padata: object to be serialized.
317 *
318 * padata_do_serial must be called for every parallelized object.
319 * The serialization callback function will run with BHs off.
320 */
321void padata_do_serial(struct padata_priv *padata)
322{
323 int cpu;
e15bacbe 324 struct padata_parallel_queue *pqueue;
16295bec
SK
325 struct parallel_data *pd;
326
327 pd = padata->pd;
328
329 cpu = get_cpu();
e15bacbe 330 pqueue = per_cpu_ptr(pd->pqueue, cpu);
16295bec 331
e15bacbe 332 spin_lock(&pqueue->reorder.lock);
16295bec 333 atomic_inc(&pd->reorder_objects);
e15bacbe
DK
334 list_add_tail(&padata->list, &pqueue->reorder.list);
335 spin_unlock(&pqueue->reorder.lock);
16295bec
SK
336
337 put_cpu();
338
339 padata_reorder(pd);
340}
341EXPORT_SYMBOL(padata_do_serial);
342
e15bacbe
DK
343static int padata_setup_cpumasks(struct parallel_data *pd,
344 const struct cpumask *pcpumask,
345 const struct cpumask *cbcpumask)
16295bec 346{
e15bacbe
DK
347 if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
348 return -ENOMEM;
16295bec 349
13614e0f 350 cpumask_and(pd->cpumask.pcpu, pcpumask, cpu_online_mask);
e15bacbe 351 if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL)) {
07a77929 352 free_cpumask_var(pd->cpumask.pcpu);
e15bacbe
DK
353 return -ENOMEM;
354 }
16295bec 355
13614e0f 356 cpumask_and(pd->cpumask.cbcpu, cbcpumask, cpu_online_mask);
e15bacbe
DK
357 return 0;
358}
16295bec 359
e15bacbe
DK
360static void __padata_list_init(struct padata_list *pd_list)
361{
362 INIT_LIST_HEAD(&pd_list->list);
363 spin_lock_init(&pd_list->lock);
364}
16295bec 365
e15bacbe
DK
366/* Initialize all percpu queues used by serial workers */
367static void padata_init_squeues(struct parallel_data *pd)
368{
369 int cpu;
370 struct padata_serial_queue *squeue;
7b389b2c 371
e15bacbe
DK
372 for_each_cpu(cpu, pd->cpumask.cbcpu) {
373 squeue = per_cpu_ptr(pd->squeue, cpu);
374 squeue->pd = pd;
375 __padata_list_init(&squeue->serial);
376 INIT_WORK(&squeue->work, padata_serial_worker);
377 }
378}
16295bec 379
e15bacbe
DK
380/* Initialize all percpu queues used by parallel workers */
381static void padata_init_pqueues(struct parallel_data *pd)
382{
2dc9b5db 383 int cpu_index, cpu;
e15bacbe 384 struct padata_parallel_queue *pqueue;
16295bec 385
e15bacbe
DK
386 cpu_index = 0;
387 for_each_cpu(cpu, pd->cpumask.pcpu) {
388 pqueue = per_cpu_ptr(pd->pqueue, cpu);
389 pqueue->pd = pd;
390 pqueue->cpu_index = cpu_index;
7b389b2c 391 cpu_index++;
16295bec 392
e15bacbe
DK
393 __padata_list_init(&pqueue->reorder);
394 __padata_list_init(&pqueue->parallel);
395 INIT_WORK(&pqueue->work, padata_parallel_worker);
396 atomic_set(&pqueue->num_obj, 0);
16295bec 397 }
e15bacbe 398}
16295bec 399
e15bacbe
DK
400/* Allocate and initialize the internal cpumask dependend resources. */
401static struct parallel_data *padata_alloc_pd(struct padata_instance *pinst,
402 const struct cpumask *pcpumask,
403 const struct cpumask *cbcpumask)
404{
405 struct parallel_data *pd;
16295bec 406
e15bacbe
DK
407 pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);
408 if (!pd)
409 goto err;
16295bec 410
e15bacbe
DK
411 pd->pqueue = alloc_percpu(struct padata_parallel_queue);
412 if (!pd->pqueue)
413 goto err_free_pd;
414
415 pd->squeue = alloc_percpu(struct padata_serial_queue);
416 if (!pd->squeue)
417 goto err_free_pqueue;
418 if (padata_setup_cpumasks(pd, pcpumask, cbcpumask) < 0)
419 goto err_free_squeue;
16295bec 420
e15bacbe
DK
421 padata_init_pqueues(pd);
422 padata_init_squeues(pd);
d46a5ac7 423 setup_timer(&pd->timer, padata_reorder_timer, (unsigned long)pd);
0b6b098e 424 atomic_set(&pd->seq_nr, -1);
16295bec
SK
425 atomic_set(&pd->reorder_objects, 0);
426 atomic_set(&pd->refcnt, 0);
427 pd->pinst = pinst;
428 spin_lock_init(&pd->lock);
429
430 return pd;
431
e15bacbe
DK
432err_free_squeue:
433 free_percpu(pd->squeue);
434err_free_pqueue:
435 free_percpu(pd->pqueue);
16295bec
SK
436err_free_pd:
437 kfree(pd);
438err:
439 return NULL;
440}
441
442static void padata_free_pd(struct parallel_data *pd)
443{
e15bacbe
DK
444 free_cpumask_var(pd->cpumask.pcpu);
445 free_cpumask_var(pd->cpumask.cbcpu);
446 free_percpu(pd->pqueue);
447 free_percpu(pd->squeue);
16295bec
SK
448 kfree(pd);
449}
450
0198ffd1 451/* Flush all objects out of the padata queues. */
2b73b07a
SK
452static void padata_flush_queues(struct parallel_data *pd)
453{
454 int cpu;
e15bacbe
DK
455 struct padata_parallel_queue *pqueue;
456 struct padata_serial_queue *squeue;
2b73b07a 457
e15bacbe
DK
458 for_each_cpu(cpu, pd->cpumask.pcpu) {
459 pqueue = per_cpu_ptr(pd->pqueue, cpu);
460 flush_work(&pqueue->work);
2b73b07a
SK
461 }
462
463 del_timer_sync(&pd->timer);
464
465 if (atomic_read(&pd->reorder_objects))
466 padata_reorder(pd);
467
e15bacbe
DK
468 for_each_cpu(cpu, pd->cpumask.cbcpu) {
469 squeue = per_cpu_ptr(pd->squeue, cpu);
470 flush_work(&squeue->work);
2b73b07a
SK
471 }
472
473 BUG_ON(atomic_read(&pd->refcnt) != 0);
474}
475
4c879170
SK
476static void __padata_start(struct padata_instance *pinst)
477{
478 pinst->flags |= PADATA_INIT;
479}
480
ee836555
SK
481static void __padata_stop(struct padata_instance *pinst)
482{
483 if (!(pinst->flags & PADATA_INIT))
484 return;
485
486 pinst->flags &= ~PADATA_INIT;
487
488 synchronize_rcu();
489
490 get_online_cpus();
491 padata_flush_queues(pinst->pd);
492 put_online_cpus();
493}
494
25985edc 495/* Replace the internal control structure with a new one. */
16295bec
SK
496static void padata_replace(struct padata_instance *pinst,
497 struct parallel_data *pd_new)
498{
499 struct parallel_data *pd_old = pinst->pd;
e15bacbe 500 int notification_mask = 0;
16295bec
SK
501
502 pinst->flags |= PADATA_RESET;
503
504 rcu_assign_pointer(pinst->pd, pd_new);
505
506 synchronize_rcu();
507
e15bacbe
DK
508 if (!cpumask_equal(pd_old->cpumask.pcpu, pd_new->cpumask.pcpu))
509 notification_mask |= PADATA_CPU_PARALLEL;
510 if (!cpumask_equal(pd_old->cpumask.cbcpu, pd_new->cpumask.cbcpu))
511 notification_mask |= PADATA_CPU_SERIAL;
512
2b73b07a 513 padata_flush_queues(pd_old);
16295bec
SK
514 padata_free_pd(pd_old);
515
e15bacbe
DK
516 if (notification_mask)
517 blocking_notifier_call_chain(&pinst->cpumask_change_notifier,
c635696c
SK
518 notification_mask,
519 &pd_new->cpumask);
16295bec
SK
520
521 pinst->flags &= ~PADATA_RESET;
522}
523
0198ffd1 524/**
e15bacbe
DK
525 * padata_register_cpumask_notifier - Registers a notifier that will be called
526 * if either pcpu or cbcpu or both cpumasks change.
16295bec 527 *
e15bacbe
DK
528 * @pinst: A poineter to padata instance
529 * @nblock: A pointer to notifier block.
16295bec 530 */
e15bacbe
DK
531int padata_register_cpumask_notifier(struct padata_instance *pinst,
532 struct notifier_block *nblock)
16295bec 533{
e15bacbe
DK
534 return blocking_notifier_chain_register(&pinst->cpumask_change_notifier,
535 nblock);
536}
537EXPORT_SYMBOL(padata_register_cpumask_notifier);
538
539/**
540 * padata_unregister_cpumask_notifier - Unregisters cpumask notifier
541 * registered earlier using padata_register_cpumask_notifier
542 *
543 * @pinst: A pointer to data instance.
544 * @nlock: A pointer to notifier block.
545 */
546int padata_unregister_cpumask_notifier(struct padata_instance *pinst,
547 struct notifier_block *nblock)
548{
549 return blocking_notifier_chain_unregister(
550 &pinst->cpumask_change_notifier,
551 nblock);
552}
553EXPORT_SYMBOL(padata_unregister_cpumask_notifier);
554
555
33e54450
SK
556/* If cpumask contains no active cpu, we mark the instance as invalid. */
557static bool padata_validate_cpumask(struct padata_instance *pinst,
558 const struct cpumask *cpumask)
559{
13614e0f 560 if (!cpumask_intersects(cpumask, cpu_online_mask)) {
33e54450
SK
561 pinst->flags |= PADATA_INVALID;
562 return false;
563 }
564
565 pinst->flags &= ~PADATA_INVALID;
566 return true;
567}
568
65ff577e
SK
569static int __padata_set_cpumasks(struct padata_instance *pinst,
570 cpumask_var_t pcpumask,
571 cpumask_var_t cbcpumask)
572{
573 int valid;
16295bec 574 struct parallel_data *pd;
65ff577e
SK
575
576 valid = padata_validate_cpumask(pinst, pcpumask);
577 if (!valid) {
578 __padata_stop(pinst);
579 goto out_replace;
580 }
581
582 valid = padata_validate_cpumask(pinst, cbcpumask);
583 if (!valid)
584 __padata_stop(pinst);
585
586out_replace:
587 pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);
588 if (!pd)
589 return -ENOMEM;
590
591 cpumask_copy(pinst->cpumask.pcpu, pcpumask);
592 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
593
594 padata_replace(pinst, pd);
595
596 if (valid)
597 __padata_start(pinst);
598
599 return 0;
600}
601
e15bacbe
DK
602/**
603 * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value
604 * equivalent to @cpumask.
16295bec
SK
605 *
606 * @pinst: padata instance
e15bacbe
DK
607 * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
608 * to parallel and serial cpumasks respectively.
16295bec
SK
609 * @cpumask: the cpumask to use
610 */
e15bacbe
DK
611int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
612 cpumask_var_t cpumask)
613{
614 struct cpumask *serial_mask, *parallel_mask;
65ff577e
SK
615 int err = -EINVAL;
616
617 mutex_lock(&pinst->lock);
6751fb3c
SK
618 get_online_cpus();
619
e15bacbe
DK
620 switch (cpumask_type) {
621 case PADATA_CPU_PARALLEL:
622 serial_mask = pinst->cpumask.cbcpu;
623 parallel_mask = cpumask;
624 break;
625 case PADATA_CPU_SERIAL:
626 parallel_mask = pinst->cpumask.pcpu;
627 serial_mask = cpumask;
628 break;
629 default:
65ff577e 630 goto out;
16295bec
SK
631 }
632
65ff577e 633 err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask);
16295bec
SK
634
635out:
6751fb3c 636 put_online_cpus();
16295bec
SK
637 mutex_unlock(&pinst->lock);
638
639 return err;
640}
641EXPORT_SYMBOL(padata_set_cpumask);
642
19d795b6
AB
643/**
644 * padata_start - start the parallel processing
645 *
646 * @pinst: padata instance to start
647 */
648int padata_start(struct padata_instance *pinst)
649{
650 int err = 0;
651
652 mutex_lock(&pinst->lock);
653
654 if (pinst->flags & PADATA_INVALID)
655 err = -EINVAL;
656
657 __padata_start(pinst);
658
659 mutex_unlock(&pinst->lock);
660
661 return err;
662}
663EXPORT_SYMBOL(padata_start);
664
665/**
666 * padata_stop - stop the parallel processing
667 *
668 * @pinst: padata instance to stop
669 */
670void padata_stop(struct padata_instance *pinst)
671{
672 mutex_lock(&pinst->lock);
673 __padata_stop(pinst);
674 mutex_unlock(&pinst->lock);
675}
676EXPORT_SYMBOL(padata_stop);
677
678#ifdef CONFIG_HOTPLUG_CPU
679
16295bec
SK
680static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
681{
682 struct parallel_data *pd;
683
13614e0f 684 if (cpumask_test_cpu(cpu, cpu_online_mask)) {
e15bacbe
DK
685 pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
686 pinst->cpumask.cbcpu);
16295bec
SK
687 if (!pd)
688 return -ENOMEM;
689
690 padata_replace(pinst, pd);
33e54450 691
e15bacbe
DK
692 if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
693 padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
33e54450 694 __padata_start(pinst);
16295bec
SK
695 }
696
697 return 0;
698}
699
16295bec
SK
700static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
701{
33e54450 702 struct parallel_data *pd = NULL;
16295bec
SK
703
704 if (cpumask_test_cpu(cpu, cpu_online_mask)) {
33e54450 705
e15bacbe 706 if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
b89661df 707 !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
33e54450 708 __padata_stop(pinst);
33e54450 709
e15bacbe
DK
710 pd = padata_alloc_pd(pinst, pinst->cpumask.pcpu,
711 pinst->cpumask.cbcpu);
16295bec
SK
712 if (!pd)
713 return -ENOMEM;
714
715 padata_replace(pinst, pd);
96120905
SK
716
717 cpumask_clear_cpu(cpu, pd->cpumask.cbcpu);
718 cpumask_clear_cpu(cpu, pd->cpumask.pcpu);
16295bec
SK
719 }
720
721 return 0;
722}
723
e15bacbe 724 /**
25985edc 725 * padata_remove_cpu - remove a cpu from the one or both(serial and parallel)
e15bacbe 726 * padata cpumasks.
16295bec
SK
727 *
728 * @pinst: padata instance
729 * @cpu: cpu to remove
e15bacbe
DK
730 * @mask: bitmask specifying from which cpumask @cpu should be removed
731 * The @mask may be any combination of the following flags:
732 * PADATA_CPU_SERIAL - serial cpumask
733 * PADATA_CPU_PARALLEL - parallel cpumask
16295bec 734 */
e15bacbe 735int padata_remove_cpu(struct padata_instance *pinst, int cpu, int mask)
16295bec
SK
736{
737 int err;
738
e15bacbe
DK
739 if (!(mask & (PADATA_CPU_SERIAL | PADATA_CPU_PARALLEL)))
740 return -EINVAL;
741
16295bec
SK
742 mutex_lock(&pinst->lock);
743
6751fb3c 744 get_online_cpus();
e15bacbe
DK
745 if (mask & PADATA_CPU_SERIAL)
746 cpumask_clear_cpu(cpu, pinst->cpumask.cbcpu);
747 if (mask & PADATA_CPU_PARALLEL)
748 cpumask_clear_cpu(cpu, pinst->cpumask.pcpu);
749
16295bec 750 err = __padata_remove_cpu(pinst, cpu);
6751fb3c 751 put_online_cpus();
16295bec
SK
752
753 mutex_unlock(&pinst->lock);
754
755 return err;
756}
757EXPORT_SYMBOL(padata_remove_cpu);
758
e15bacbe
DK
759static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
760{
761 return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
762 cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);
763}
764
30e92153 765static int padata_cpu_online(unsigned int cpu, struct hlist_node *node)
16295bec 766{
16295bec 767 struct padata_instance *pinst;
30e92153 768 int ret;
16295bec 769
30e92153
SAS
770 pinst = hlist_entry_safe(node, struct padata_instance, node);
771 if (!pinst_has_cpu(pinst, cpu))
772 return 0;
16295bec 773
30e92153
SAS
774 mutex_lock(&pinst->lock);
775 ret = __padata_add_cpu(pinst, cpu);
776 mutex_unlock(&pinst->lock);
777 return ret;
778}
16295bec 779
30e92153
SAS
780static int padata_cpu_prep_down(unsigned int cpu, struct hlist_node *node)
781{
782 struct padata_instance *pinst;
783 int ret;
784
785 pinst = hlist_entry_safe(node, struct padata_instance, node);
786 if (!pinst_has_cpu(pinst, cpu))
787 return 0;
16295bec 788
30e92153
SAS
789 mutex_lock(&pinst->lock);
790 ret = __padata_remove_cpu(pinst, cpu);
791 mutex_unlock(&pinst->lock);
792 return ret;
16295bec 793}
30e92153
SAS
794
795static enum cpuhp_state hp_online;
e2cb2f1c 796#endif
16295bec 797
5e017dc3
DK
798static void __padata_free(struct padata_instance *pinst)
799{
800#ifdef CONFIG_HOTPLUG_CPU
30e92153 801 cpuhp_state_remove_instance_nocalls(hp_online, &pinst->node);
5e017dc3
DK
802#endif
803
804 padata_stop(pinst);
805 padata_free_pd(pinst->pd);
806 free_cpumask_var(pinst->cpumask.pcpu);
807 free_cpumask_var(pinst->cpumask.cbcpu);
808 kfree(pinst);
809}
810
811#define kobj2pinst(_kobj) \
812 container_of(_kobj, struct padata_instance, kobj)
813#define attr2pentry(_attr) \
814 container_of(_attr, struct padata_sysfs_entry, attr)
815
816static void padata_sysfs_release(struct kobject *kobj)
817{
818 struct padata_instance *pinst = kobj2pinst(kobj);
819 __padata_free(pinst);
820}
821
822struct padata_sysfs_entry {
823 struct attribute attr;
824 ssize_t (*show)(struct padata_instance *, struct attribute *, char *);
825 ssize_t (*store)(struct padata_instance *, struct attribute *,
826 const char *, size_t);
827};
828
829static ssize_t show_cpumask(struct padata_instance *pinst,
830 struct attribute *attr, char *buf)
831{
832 struct cpumask *cpumask;
833 ssize_t len;
834
835 mutex_lock(&pinst->lock);
836 if (!strcmp(attr->name, "serial_cpumask"))
837 cpumask = pinst->cpumask.cbcpu;
838 else
839 cpumask = pinst->cpumask.pcpu;
840
4497da6f
TH
841 len = snprintf(buf, PAGE_SIZE, "%*pb\n",
842 nr_cpu_ids, cpumask_bits(cpumask));
5e017dc3 843 mutex_unlock(&pinst->lock);
4497da6f 844 return len < PAGE_SIZE ? len : -EINVAL;
5e017dc3
DK
845}
846
847static ssize_t store_cpumask(struct padata_instance *pinst,
848 struct attribute *attr,
849 const char *buf, size_t count)
850{
851 cpumask_var_t new_cpumask;
852 ssize_t ret;
853 int mask_type;
854
855 if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL))
856 return -ENOMEM;
857
858 ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask),
859 nr_cpumask_bits);
860 if (ret < 0)
861 goto out;
862
863 mask_type = !strcmp(attr->name, "serial_cpumask") ?
864 PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL;
865 ret = padata_set_cpumask(pinst, mask_type, new_cpumask);
866 if (!ret)
867 ret = count;
868
869out:
870 free_cpumask_var(new_cpumask);
871 return ret;
872}
873
874#define PADATA_ATTR_RW(_name, _show_name, _store_name) \
875 static struct padata_sysfs_entry _name##_attr = \
876 __ATTR(_name, 0644, _show_name, _store_name)
877#define PADATA_ATTR_RO(_name, _show_name) \
878 static struct padata_sysfs_entry _name##_attr = \
879 __ATTR(_name, 0400, _show_name, NULL)
880
881PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask);
882PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask);
883
884/*
885 * Padata sysfs provides the following objects:
886 * serial_cpumask [RW] - cpumask for serial workers
887 * parallel_cpumask [RW] - cpumask for parallel workers
888 */
889static struct attribute *padata_default_attrs[] = {
890 &serial_cpumask_attr.attr,
891 &parallel_cpumask_attr.attr,
892 NULL,
893};
894
895static ssize_t padata_sysfs_show(struct kobject *kobj,
896 struct attribute *attr, char *buf)
897{
898 struct padata_instance *pinst;
899 struct padata_sysfs_entry *pentry;
900 ssize_t ret = -EIO;
901
902 pinst = kobj2pinst(kobj);
903 pentry = attr2pentry(attr);
904 if (pentry->show)
905 ret = pentry->show(pinst, attr, buf);
906
907 return ret;
908}
909
910static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr,
911 const char *buf, size_t count)
912{
913 struct padata_instance *pinst;
914 struct padata_sysfs_entry *pentry;
915 ssize_t ret = -EIO;
916
917 pinst = kobj2pinst(kobj);
918 pentry = attr2pentry(attr);
919 if (pentry->show)
920 ret = pentry->store(pinst, attr, buf, count);
921
922 return ret;
923}
924
925static const struct sysfs_ops padata_sysfs_ops = {
926 .show = padata_sysfs_show,
927 .store = padata_sysfs_store,
928};
929
930static struct kobj_type padata_attr_type = {
931 .sysfs_ops = &padata_sysfs_ops,
932 .default_attrs = padata_default_attrs,
933 .release = padata_sysfs_release,
934};
935
e15bacbe 936/**
e6cc1170
SK
937 * padata_alloc - allocate and initialize a padata instance and specify
938 * cpumasks for serial and parallel workers.
16295bec 939 *
16295bec 940 * @wq: workqueue to use for the allocated padata instance
e15bacbe
DK
941 * @pcpumask: cpumask that will be used for padata parallelization
942 * @cbcpumask: cpumask that will be used for padata serialization
c5a81c8f
SAS
943 *
944 * Must be called from a cpus_read_lock() protected region
16295bec 945 */
9596695e
TG
946static struct padata_instance *padata_alloc(struct workqueue_struct *wq,
947 const struct cpumask *pcpumask,
948 const struct cpumask *cbcpumask)
16295bec 949{
16295bec 950 struct padata_instance *pinst;
33e54450 951 struct parallel_data *pd = NULL;
16295bec
SK
952
953 pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);
954 if (!pinst)
955 goto err;
956
e15bacbe 957 if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))
16295bec 958 goto err_free_inst;
e15bacbe
DK
959 if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {
960 free_cpumask_var(pinst->cpumask.pcpu);
16295bec 961 goto err_free_inst;
33e54450 962 }
e15bacbe
DK
963 if (!padata_validate_cpumask(pinst, pcpumask) ||
964 !padata_validate_cpumask(pinst, cbcpumask))
965 goto err_free_masks;
16295bec 966
e15bacbe
DK
967 pd = padata_alloc_pd(pinst, pcpumask, cbcpumask);
968 if (!pd)
969 goto err_free_masks;
74781387 970
16295bec
SK
971 rcu_assign_pointer(pinst->pd, pd);
972
973 pinst->wq = wq;
974
e15bacbe
DK
975 cpumask_copy(pinst->cpumask.pcpu, pcpumask);
976 cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
16295bec
SK
977
978 pinst->flags = 0;
979
e15bacbe 980 BLOCKING_INIT_NOTIFIER_HEAD(&pinst->cpumask_change_notifier);
5e017dc3 981 kobject_init(&pinst->kobj, &padata_attr_type);
16295bec
SK
982 mutex_init(&pinst->lock);
983
b8b4a416 984#ifdef CONFIG_HOTPLUG_CPU
c5a81c8f 985 cpuhp_state_add_instance_nocalls_cpuslocked(hp_online, &pinst->node);
b8b4a416 986#endif
16295bec
SK
987 return pinst;
988
e15bacbe
DK
989err_free_masks:
990 free_cpumask_var(pinst->cpumask.pcpu);
991 free_cpumask_var(pinst->cpumask.cbcpu);
16295bec
SK
992err_free_inst:
993 kfree(pinst);
994err:
995 return NULL;
996}
16295bec 997
9596695e
TG
998/**
999 * padata_alloc_possible - Allocate and initialize padata instance.
1000 * Use the cpu_possible_mask for serial and
1001 * parallel workers.
1002 *
1003 * @wq: workqueue to use for the allocated padata instance
c5a81c8f
SAS
1004 *
1005 * Must be called from a cpus_read_lock() protected region
9596695e
TG
1006 */
1007struct padata_instance *padata_alloc_possible(struct workqueue_struct *wq)
1008{
c5a81c8f 1009 lockdep_assert_cpus_held();
9596695e
TG
1010 return padata_alloc(wq, cpu_possible_mask, cpu_possible_mask);
1011}
1012EXPORT_SYMBOL(padata_alloc_possible);
1013
0198ffd1 1014/**
16295bec
SK
1015 * padata_free - free a padata instance
1016 *
0198ffd1 1017 * @padata_inst: padata instance to free
16295bec
SK
1018 */
1019void padata_free(struct padata_instance *pinst)
1020{
5e017dc3 1021 kobject_put(&pinst->kobj);
16295bec
SK
1022}
1023EXPORT_SYMBOL(padata_free);
30e92153
SAS
1024
1025#ifdef CONFIG_HOTPLUG_CPU
1026
1027static __init int padata_driver_init(void)
1028{
1029 int ret;
1030
1031 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online",
1032 padata_cpu_online,
1033 padata_cpu_prep_down);
1034 if (ret < 0)
1035 return ret;
1036 hp_online = ret;
1037 return 0;
1038}
1039module_init(padata_driver_init);
1040
1041static __exit void padata_driver_exit(void)
1042{
1043 cpuhp_remove_multi_state(hp_online);
1044}
1045module_exit(padata_driver_exit);
1046#endif