2 * padata.c - generic interface to process data streams in parallel
4 * See Documentation/padata.txt for an api documentation.
6 * Copyright (C) 2008, 2009 secunet Security Networks AG
7 * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
13 * This program is distributed in the hope it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
18 * You should have received a copy of the GNU General Public License along with
19 * this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
23 #include <linux/export.h>
24 #include <linux/cpumask.h>
25 #include <linux/err.h>
26 #include <linux/cpu.h>
27 #include <linux/padata.h>
28 #include <linux/mutex.h>
29 #include <linux/sched.h>
30 #include <linux/slab.h>
31 #include <linux/sysfs.h>
32 #include <linux/rcupdate.h>
33 #include <linux/module.h>
35 #define MAX_OBJ_NUM 1000
37 static int padata_index_to_cpu(struct parallel_data
*pd
, int cpu_index
)
41 target_cpu
= cpumask_first(pd
->cpumask
.pcpu
);
42 for (cpu
= 0; cpu
< cpu_index
; cpu
++)
43 target_cpu
= cpumask_next(target_cpu
, pd
->cpumask
.pcpu
);
48 static int padata_cpu_hash(struct parallel_data
*pd
)
54 * Hash the sequence numbers to the cpus by taking
55 * seq_nr mod. number of cpus in use.
58 seq_nr
= atomic_inc_return(&pd
->seq_nr
);
59 cpu_index
= seq_nr
% cpumask_weight(pd
->cpumask
.pcpu
);
61 return padata_index_to_cpu(pd
, cpu_index
);
64 static void padata_parallel_worker(struct work_struct
*parallel_work
)
66 struct padata_parallel_queue
*pqueue
;
67 LIST_HEAD(local_list
);
70 pqueue
= container_of(parallel_work
,
71 struct padata_parallel_queue
, work
);
73 spin_lock(&pqueue
->parallel
.lock
);
74 list_replace_init(&pqueue
->parallel
.list
, &local_list
);
75 spin_unlock(&pqueue
->parallel
.lock
);
77 while (!list_empty(&local_list
)) {
78 struct padata_priv
*padata
;
80 padata
= list_entry(local_list
.next
,
81 struct padata_priv
, list
);
83 list_del_init(&padata
->list
);
85 padata
->parallel(padata
);
92 * padata_do_parallel - padata parallelization function
94 * @pinst: padata instance
95 * @padata: object to be parallelized
96 * @cb_cpu: cpu the serialization callback function will run on,
97 * must be in the serial cpumask of padata(i.e. cpumask.cbcpu).
99 * The parallelization callback function will run with BHs off.
100 * Note: Every object which is parallelized by padata_do_parallel
101 * must be seen by padata_do_serial.
103 int padata_do_parallel(struct padata_instance
*pinst
,
104 struct padata_priv
*padata
, int cb_cpu
)
107 struct padata_parallel_queue
*queue
;
108 struct parallel_data
*pd
;
112 pd
= rcu_dereference_bh(pinst
->pd
);
115 if (!(pinst
->flags
& PADATA_INIT
) || pinst
->flags
& PADATA_INVALID
)
118 if (!cpumask_test_cpu(cb_cpu
, pd
->cpumask
.cbcpu
))
122 if ((pinst
->flags
& PADATA_RESET
))
125 if (atomic_read(&pd
->refcnt
) >= MAX_OBJ_NUM
)
129 atomic_inc(&pd
->refcnt
);
131 padata
->cb_cpu
= cb_cpu
;
133 target_cpu
= padata_cpu_hash(pd
);
134 padata
->cpu
= target_cpu
;
135 queue
= per_cpu_ptr(pd
->pqueue
, target_cpu
);
137 spin_lock(&queue
->parallel
.lock
);
138 list_add_tail(&padata
->list
, &queue
->parallel
.list
);
139 spin_unlock(&queue
->parallel
.lock
);
141 queue_work_on(target_cpu
, pinst
->wq
, &queue
->work
);
144 rcu_read_unlock_bh();
148 EXPORT_SYMBOL(padata_do_parallel
);
151 * padata_get_next - Get the next object that needs serialization.
155 * A pointer to the control struct of the next object that needs
156 * serialization, if present in one of the percpu reorder queues.
158 * -EINPROGRESS, if the next object that needs serialization will
159 * be parallel processed by another cpu and is not yet present in
160 * the cpu's reorder queue.
162 * -ENODATA, if this cpu has to do the parallel processing for
165 static struct padata_priv
*padata_get_next(struct parallel_data
*pd
)
168 unsigned int next_nr
, next_index
;
169 struct padata_parallel_queue
*next_queue
;
170 struct padata_priv
*padata
;
171 struct padata_list
*reorder
;
173 num_cpus
= cpumask_weight(pd
->cpumask
.pcpu
);
176 * Calculate the percpu reorder queue and the sequence
177 * number of the next object.
179 next_nr
= pd
->processed
;
180 next_index
= next_nr
% num_cpus
;
181 cpu
= padata_index_to_cpu(pd
, next_index
);
182 next_queue
= per_cpu_ptr(pd
->pqueue
, cpu
);
184 reorder
= &next_queue
->reorder
;
186 spin_lock(&reorder
->lock
);
187 if (!list_empty(&reorder
->list
)) {
188 padata
= list_entry(reorder
->list
.next
,
189 struct padata_priv
, list
);
191 list_del_init(&padata
->list
);
192 atomic_dec(&pd
->reorder_objects
);
196 spin_unlock(&reorder
->lock
);
199 spin_unlock(&reorder
->lock
);
201 if (__this_cpu_read(pd
->pqueue
->cpu_index
) == next_queue
->cpu_index
) {
202 padata
= ERR_PTR(-ENODATA
);
206 padata
= ERR_PTR(-EINPROGRESS
);
211 static void padata_reorder(struct parallel_data
*pd
)
214 struct padata_priv
*padata
;
215 struct padata_serial_queue
*squeue
;
216 struct padata_instance
*pinst
= pd
->pinst
;
219 * We need to ensure that only one cpu can work on dequeueing of
220 * the reorder queue the time. Calculating in which percpu reorder
221 * queue the next object will arrive takes some time. A spinlock
222 * would be highly contended. Also it is not clear in which order
223 * the objects arrive to the reorder queues. So a cpu could wait to
224 * get the lock just to notice that there is nothing to do at the
225 * moment. Therefore we use a trylock and let the holder of the lock
226 * care for all the objects enqueued during the holdtime of the lock.
228 if (!spin_trylock_bh(&pd
->lock
))
232 padata
= padata_get_next(pd
);
235 * If the next object that needs serialization is parallel
236 * processed by another cpu and is still on it's way to the
237 * cpu's reorder queue, nothing to do for now.
239 if (PTR_ERR(padata
) == -EINPROGRESS
)
243 * This cpu has to do the parallel processing of the next
244 * object. It's waiting in the cpu's parallelization queue,
245 * so exit immediately.
247 if (PTR_ERR(padata
) == -ENODATA
) {
248 del_timer(&pd
->timer
);
249 spin_unlock_bh(&pd
->lock
);
253 cb_cpu
= padata
->cb_cpu
;
254 squeue
= per_cpu_ptr(pd
->squeue
, cb_cpu
);
256 spin_lock(&squeue
->serial
.lock
);
257 list_add_tail(&padata
->list
, &squeue
->serial
.list
);
258 spin_unlock(&squeue
->serial
.lock
);
260 queue_work_on(cb_cpu
, pinst
->wq
, &squeue
->work
);
263 spin_unlock_bh(&pd
->lock
);
266 * The next object that needs serialization might have arrived to
267 * the reorder queues in the meantime, we will be called again
268 * from the timer function if no one else cares for it.
270 * Ensure reorder_objects is read after pd->lock is dropped so we see
271 * an increment from another task in padata_do_serial. Pairs with
272 * smp_mb__after_atomic in padata_do_serial.
275 if (atomic_read(&pd
->reorder_objects
)
276 && !(pinst
->flags
& PADATA_RESET
))
277 mod_timer(&pd
->timer
, jiffies
+ HZ
);
279 del_timer(&pd
->timer
);
284 static void invoke_padata_reorder(struct work_struct
*work
)
286 struct padata_parallel_queue
*pqueue
;
287 struct parallel_data
*pd
;
290 pqueue
= container_of(work
, struct padata_parallel_queue
, reorder_work
);
296 static void padata_reorder_timer(struct timer_list
*t
)
298 struct parallel_data
*pd
= from_timer(pd
, t
, timer
);
304 /* We don't lock pd here to not interfere with parallel processing
305 * padata_reorder() calls on other CPUs. We just need any CPU out of
306 * the cpumask.pcpu set. It would be nice if it's the right one but
307 * it doesn't matter if we're off to the next one by using an outdated
308 * pd->processed value.
310 weight
= cpumask_weight(pd
->cpumask
.pcpu
);
311 target_cpu
= padata_index_to_cpu(pd
, pd
->processed
% weight
);
313 /* ensure to call the reorder callback on the correct CPU */
314 if (cpu
!= target_cpu
) {
315 struct padata_parallel_queue
*pqueue
;
316 struct padata_instance
*pinst
;
318 /* The timer function is serialized wrt itself -- no locking
322 pqueue
= per_cpu_ptr(pd
->pqueue
, target_cpu
);
323 queue_work_on(target_cpu
, pinst
->wq
, &pqueue
->reorder_work
);
331 static void padata_serial_worker(struct work_struct
*serial_work
)
333 struct padata_serial_queue
*squeue
;
334 struct parallel_data
*pd
;
335 LIST_HEAD(local_list
);
338 squeue
= container_of(serial_work
, struct padata_serial_queue
, work
);
341 spin_lock(&squeue
->serial
.lock
);
342 list_replace_init(&squeue
->serial
.list
, &local_list
);
343 spin_unlock(&squeue
->serial
.lock
);
345 while (!list_empty(&local_list
)) {
346 struct padata_priv
*padata
;
348 padata
= list_entry(local_list
.next
,
349 struct padata_priv
, list
);
351 list_del_init(&padata
->list
);
353 padata
->serial(padata
);
354 atomic_dec(&pd
->refcnt
);
360 * padata_do_serial - padata serialization function
362 * @padata: object to be serialized.
364 * padata_do_serial must be called for every parallelized object.
365 * The serialization callback function will run with BHs off.
367 void padata_do_serial(struct padata_priv
*padata
)
370 struct padata_parallel_queue
*pqueue
;
371 struct parallel_data
*pd
;
372 int reorder_via_wq
= 0;
378 /* We need to run on the same CPU padata_do_parallel(.., padata, ..)
379 * was called on -- or, at least, enqueue the padata object into the
380 * correct per-cpu queue.
382 if (cpu
!= padata
->cpu
) {
387 pqueue
= per_cpu_ptr(pd
->pqueue
, cpu
);
389 spin_lock(&pqueue
->reorder
.lock
);
390 atomic_inc(&pd
->reorder_objects
);
391 list_add_tail(&padata
->list
, &pqueue
->reorder
.list
);
392 spin_unlock(&pqueue
->reorder
.lock
);
395 * Ensure the atomic_inc of reorder_objects above is ordered correctly
396 * with the trylock of pd->lock in padata_reorder. Pairs with smp_mb
399 smp_mb__after_atomic();
403 /* If we're running on the wrong CPU, call padata_reorder() via a
407 queue_work_on(cpu
, pd
->pinst
->wq
, &pqueue
->reorder_work
);
411 EXPORT_SYMBOL(padata_do_serial
);
413 static int padata_setup_cpumasks(struct parallel_data
*pd
,
414 const struct cpumask
*pcpumask
,
415 const struct cpumask
*cbcpumask
)
417 if (!alloc_cpumask_var(&pd
->cpumask
.pcpu
, GFP_KERNEL
))
420 cpumask_and(pd
->cpumask
.pcpu
, pcpumask
, cpu_online_mask
);
421 if (!alloc_cpumask_var(&pd
->cpumask
.cbcpu
, GFP_KERNEL
)) {
422 free_cpumask_var(pd
->cpumask
.pcpu
);
426 cpumask_and(pd
->cpumask
.cbcpu
, cbcpumask
, cpu_online_mask
);
430 static void __padata_list_init(struct padata_list
*pd_list
)
432 INIT_LIST_HEAD(&pd_list
->list
);
433 spin_lock_init(&pd_list
->lock
);
436 /* Initialize all percpu queues used by serial workers */
437 static void padata_init_squeues(struct parallel_data
*pd
)
440 struct padata_serial_queue
*squeue
;
442 for_each_cpu(cpu
, pd
->cpumask
.cbcpu
) {
443 squeue
= per_cpu_ptr(pd
->squeue
, cpu
);
445 __padata_list_init(&squeue
->serial
);
446 INIT_WORK(&squeue
->work
, padata_serial_worker
);
450 /* Initialize all percpu queues used by parallel workers */
451 static void padata_init_pqueues(struct parallel_data
*pd
)
454 struct padata_parallel_queue
*pqueue
;
457 for_each_possible_cpu(cpu
) {
458 pqueue
= per_cpu_ptr(pd
->pqueue
, cpu
);
460 if (!cpumask_test_cpu(cpu
, pd
->cpumask
.pcpu
)) {
461 pqueue
->cpu_index
= -1;
466 pqueue
->cpu_index
= cpu_index
;
469 __padata_list_init(&pqueue
->reorder
);
470 __padata_list_init(&pqueue
->parallel
);
471 INIT_WORK(&pqueue
->work
, padata_parallel_worker
);
472 INIT_WORK(&pqueue
->reorder_work
, invoke_padata_reorder
);
473 atomic_set(&pqueue
->num_obj
, 0);
477 /* Allocate and initialize the internal cpumask dependend resources. */
478 static struct parallel_data
*padata_alloc_pd(struct padata_instance
*pinst
,
479 const struct cpumask
*pcpumask
,
480 const struct cpumask
*cbcpumask
)
482 struct parallel_data
*pd
;
484 pd
= kzalloc(sizeof(struct parallel_data
), GFP_KERNEL
);
488 pd
->pqueue
= alloc_percpu(struct padata_parallel_queue
);
492 pd
->squeue
= alloc_percpu(struct padata_serial_queue
);
494 goto err_free_pqueue
;
495 if (padata_setup_cpumasks(pd
, pcpumask
, cbcpumask
) < 0)
496 goto err_free_squeue
;
498 padata_init_pqueues(pd
);
499 padata_init_squeues(pd
);
500 timer_setup(&pd
->timer
, padata_reorder_timer
, 0);
501 atomic_set(&pd
->seq_nr
, -1);
502 atomic_set(&pd
->reorder_objects
, 0);
503 atomic_set(&pd
->refcnt
, 0);
505 spin_lock_init(&pd
->lock
);
510 free_percpu(pd
->squeue
);
512 free_percpu(pd
->pqueue
);
519 static void padata_free_pd(struct parallel_data
*pd
)
521 free_cpumask_var(pd
->cpumask
.pcpu
);
522 free_cpumask_var(pd
->cpumask
.cbcpu
);
523 free_percpu(pd
->pqueue
);
524 free_percpu(pd
->squeue
);
528 /* Flush all objects out of the padata queues. */
529 static void padata_flush_queues(struct parallel_data
*pd
)
532 struct padata_parallel_queue
*pqueue
;
533 struct padata_serial_queue
*squeue
;
535 for_each_cpu(cpu
, pd
->cpumask
.pcpu
) {
536 pqueue
= per_cpu_ptr(pd
->pqueue
, cpu
);
537 flush_work(&pqueue
->work
);
540 del_timer_sync(&pd
->timer
);
542 if (atomic_read(&pd
->reorder_objects
))
545 for_each_cpu(cpu
, pd
->cpumask
.cbcpu
) {
546 squeue
= per_cpu_ptr(pd
->squeue
, cpu
);
547 flush_work(&squeue
->work
);
550 BUG_ON(atomic_read(&pd
->refcnt
) != 0);
553 static void __padata_start(struct padata_instance
*pinst
)
555 pinst
->flags
|= PADATA_INIT
;
558 static void __padata_stop(struct padata_instance
*pinst
)
560 if (!(pinst
->flags
& PADATA_INIT
))
563 pinst
->flags
&= ~PADATA_INIT
;
568 padata_flush_queues(pinst
->pd
);
572 /* Replace the internal control structure with a new one. */
573 static void padata_replace(struct padata_instance
*pinst
,
574 struct parallel_data
*pd_new
)
576 struct parallel_data
*pd_old
= pinst
->pd
;
577 int notification_mask
= 0;
579 pinst
->flags
|= PADATA_RESET
;
581 rcu_assign_pointer(pinst
->pd
, pd_new
);
585 if (!cpumask_equal(pd_old
->cpumask
.pcpu
, pd_new
->cpumask
.pcpu
))
586 notification_mask
|= PADATA_CPU_PARALLEL
;
587 if (!cpumask_equal(pd_old
->cpumask
.cbcpu
, pd_new
->cpumask
.cbcpu
))
588 notification_mask
|= PADATA_CPU_SERIAL
;
590 padata_flush_queues(pd_old
);
591 padata_free_pd(pd_old
);
593 if (notification_mask
)
594 blocking_notifier_call_chain(&pinst
->cpumask_change_notifier
,
598 pinst
->flags
&= ~PADATA_RESET
;
602 * padata_register_cpumask_notifier - Registers a notifier that will be called
603 * if either pcpu or cbcpu or both cpumasks change.
605 * @pinst: A poineter to padata instance
606 * @nblock: A pointer to notifier block.
608 int padata_register_cpumask_notifier(struct padata_instance
*pinst
,
609 struct notifier_block
*nblock
)
611 return blocking_notifier_chain_register(&pinst
->cpumask_change_notifier
,
614 EXPORT_SYMBOL(padata_register_cpumask_notifier
);
617 * padata_unregister_cpumask_notifier - Unregisters cpumask notifier
618 * registered earlier using padata_register_cpumask_notifier
620 * @pinst: A pointer to data instance.
621 * @nlock: A pointer to notifier block.
623 int padata_unregister_cpumask_notifier(struct padata_instance
*pinst
,
624 struct notifier_block
*nblock
)
626 return blocking_notifier_chain_unregister(
627 &pinst
->cpumask_change_notifier
,
630 EXPORT_SYMBOL(padata_unregister_cpumask_notifier
);
633 /* If cpumask contains no active cpu, we mark the instance as invalid. */
634 static bool padata_validate_cpumask(struct padata_instance
*pinst
,
635 const struct cpumask
*cpumask
)
637 if (!cpumask_intersects(cpumask
, cpu_online_mask
)) {
638 pinst
->flags
|= PADATA_INVALID
;
642 pinst
->flags
&= ~PADATA_INVALID
;
646 static int __padata_set_cpumasks(struct padata_instance
*pinst
,
647 cpumask_var_t pcpumask
,
648 cpumask_var_t cbcpumask
)
651 struct parallel_data
*pd
;
653 valid
= padata_validate_cpumask(pinst
, pcpumask
);
655 __padata_stop(pinst
);
659 valid
= padata_validate_cpumask(pinst
, cbcpumask
);
661 __padata_stop(pinst
);
664 pd
= padata_alloc_pd(pinst
, pcpumask
, cbcpumask
);
668 cpumask_copy(pinst
->cpumask
.pcpu
, pcpumask
);
669 cpumask_copy(pinst
->cpumask
.cbcpu
, cbcpumask
);
671 padata_replace(pinst
, pd
);
674 __padata_start(pinst
);
680 * padata_set_cpumask: Sets specified by @cpumask_type cpumask to the value
681 * equivalent to @cpumask.
683 * @pinst: padata instance
684 * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
685 * to parallel and serial cpumasks respectively.
686 * @cpumask: the cpumask to use
688 int padata_set_cpumask(struct padata_instance
*pinst
, int cpumask_type
,
689 cpumask_var_t cpumask
)
691 struct cpumask
*serial_mask
, *parallel_mask
;
694 mutex_lock(&pinst
->lock
);
697 switch (cpumask_type
) {
698 case PADATA_CPU_PARALLEL
:
699 serial_mask
= pinst
->cpumask
.cbcpu
;
700 parallel_mask
= cpumask
;
702 case PADATA_CPU_SERIAL
:
703 parallel_mask
= pinst
->cpumask
.pcpu
;
704 serial_mask
= cpumask
;
710 err
= __padata_set_cpumasks(pinst
, parallel_mask
, serial_mask
);
714 mutex_unlock(&pinst
->lock
);
718 EXPORT_SYMBOL(padata_set_cpumask
);
721 * padata_start - start the parallel processing
723 * @pinst: padata instance to start
725 int padata_start(struct padata_instance
*pinst
)
729 mutex_lock(&pinst
->lock
);
731 if (pinst
->flags
& PADATA_INVALID
)
734 __padata_start(pinst
);
736 mutex_unlock(&pinst
->lock
);
740 EXPORT_SYMBOL(padata_start
);
743 * padata_stop - stop the parallel processing
745 * @pinst: padata instance to stop
747 void padata_stop(struct padata_instance
*pinst
)
749 mutex_lock(&pinst
->lock
);
750 __padata_stop(pinst
);
751 mutex_unlock(&pinst
->lock
);
753 EXPORT_SYMBOL(padata_stop
);
755 #ifdef CONFIG_HOTPLUG_CPU
757 static int __padata_add_cpu(struct padata_instance
*pinst
, int cpu
)
759 struct parallel_data
*pd
;
761 if (cpumask_test_cpu(cpu
, cpu_online_mask
)) {
762 pd
= padata_alloc_pd(pinst
, pinst
->cpumask
.pcpu
,
763 pinst
->cpumask
.cbcpu
);
767 padata_replace(pinst
, pd
);
769 if (padata_validate_cpumask(pinst
, pinst
->cpumask
.pcpu
) &&
770 padata_validate_cpumask(pinst
, pinst
->cpumask
.cbcpu
))
771 __padata_start(pinst
);
777 static int __padata_remove_cpu(struct padata_instance
*pinst
, int cpu
)
779 struct parallel_data
*pd
= NULL
;
781 if (cpumask_test_cpu(cpu
, cpu_online_mask
)) {
783 if (!padata_validate_cpumask(pinst
, pinst
->cpumask
.pcpu
) ||
784 !padata_validate_cpumask(pinst
, pinst
->cpumask
.cbcpu
))
785 __padata_stop(pinst
);
787 pd
= padata_alloc_pd(pinst
, pinst
->cpumask
.pcpu
,
788 pinst
->cpumask
.cbcpu
);
792 padata_replace(pinst
, pd
);
794 cpumask_clear_cpu(cpu
, pd
->cpumask
.cbcpu
);
795 cpumask_clear_cpu(cpu
, pd
->cpumask
.pcpu
);
802 * padata_remove_cpu - remove a cpu from the one or both(serial and parallel)
805 * @pinst: padata instance
806 * @cpu: cpu to remove
807 * @mask: bitmask specifying from which cpumask @cpu should be removed
808 * The @mask may be any combination of the following flags:
809 * PADATA_CPU_SERIAL - serial cpumask
810 * PADATA_CPU_PARALLEL - parallel cpumask
812 int padata_remove_cpu(struct padata_instance
*pinst
, int cpu
, int mask
)
816 if (!(mask
& (PADATA_CPU_SERIAL
| PADATA_CPU_PARALLEL
)))
819 mutex_lock(&pinst
->lock
);
822 if (mask
& PADATA_CPU_SERIAL
)
823 cpumask_clear_cpu(cpu
, pinst
->cpumask
.cbcpu
);
824 if (mask
& PADATA_CPU_PARALLEL
)
825 cpumask_clear_cpu(cpu
, pinst
->cpumask
.pcpu
);
827 err
= __padata_remove_cpu(pinst
, cpu
);
830 mutex_unlock(&pinst
->lock
);
834 EXPORT_SYMBOL(padata_remove_cpu
);
836 static inline int pinst_has_cpu(struct padata_instance
*pinst
, int cpu
)
838 return cpumask_test_cpu(cpu
, pinst
->cpumask
.pcpu
) ||
839 cpumask_test_cpu(cpu
, pinst
->cpumask
.cbcpu
);
842 static int padata_cpu_online(unsigned int cpu
, struct hlist_node
*node
)
844 struct padata_instance
*pinst
;
847 pinst
= hlist_entry_safe(node
, struct padata_instance
, node
);
848 if (!pinst_has_cpu(pinst
, cpu
))
851 mutex_lock(&pinst
->lock
);
852 ret
= __padata_add_cpu(pinst
, cpu
);
853 mutex_unlock(&pinst
->lock
);
857 static int padata_cpu_prep_down(unsigned int cpu
, struct hlist_node
*node
)
859 struct padata_instance
*pinst
;
862 pinst
= hlist_entry_safe(node
, struct padata_instance
, node
);
863 if (!pinst_has_cpu(pinst
, cpu
))
866 mutex_lock(&pinst
->lock
);
867 ret
= __padata_remove_cpu(pinst
, cpu
);
868 mutex_unlock(&pinst
->lock
);
872 static enum cpuhp_state hp_online
;
875 static void __padata_free(struct padata_instance
*pinst
)
877 #ifdef CONFIG_HOTPLUG_CPU
878 cpuhp_state_remove_instance_nocalls(hp_online
, &pinst
->node
);
882 padata_free_pd(pinst
->pd
);
883 free_cpumask_var(pinst
->cpumask
.pcpu
);
884 free_cpumask_var(pinst
->cpumask
.cbcpu
);
888 #define kobj2pinst(_kobj) \
889 container_of(_kobj, struct padata_instance, kobj)
890 #define attr2pentry(_attr) \
891 container_of(_attr, struct padata_sysfs_entry, attr)
893 static void padata_sysfs_release(struct kobject
*kobj
)
895 struct padata_instance
*pinst
= kobj2pinst(kobj
);
896 __padata_free(pinst
);
899 struct padata_sysfs_entry
{
900 struct attribute attr
;
901 ssize_t (*show
)(struct padata_instance
*, struct attribute
*, char *);
902 ssize_t (*store
)(struct padata_instance
*, struct attribute
*,
903 const char *, size_t);
906 static ssize_t
show_cpumask(struct padata_instance
*pinst
,
907 struct attribute
*attr
, char *buf
)
909 struct cpumask
*cpumask
;
912 mutex_lock(&pinst
->lock
);
913 if (!strcmp(attr
->name
, "serial_cpumask"))
914 cpumask
= pinst
->cpumask
.cbcpu
;
916 cpumask
= pinst
->cpumask
.pcpu
;
918 len
= snprintf(buf
, PAGE_SIZE
, "%*pb\n",
919 nr_cpu_ids
, cpumask_bits(cpumask
));
920 mutex_unlock(&pinst
->lock
);
921 return len
< PAGE_SIZE
? len
: -EINVAL
;
924 static ssize_t
store_cpumask(struct padata_instance
*pinst
,
925 struct attribute
*attr
,
926 const char *buf
, size_t count
)
928 cpumask_var_t new_cpumask
;
932 if (!alloc_cpumask_var(&new_cpumask
, GFP_KERNEL
))
935 ret
= bitmap_parse(buf
, count
, cpumask_bits(new_cpumask
),
940 mask_type
= !strcmp(attr
->name
, "serial_cpumask") ?
941 PADATA_CPU_SERIAL
: PADATA_CPU_PARALLEL
;
942 ret
= padata_set_cpumask(pinst
, mask_type
, new_cpumask
);
947 free_cpumask_var(new_cpumask
);
951 #define PADATA_ATTR_RW(_name, _show_name, _store_name) \
952 static struct padata_sysfs_entry _name##_attr = \
953 __ATTR(_name, 0644, _show_name, _store_name)
954 #define PADATA_ATTR_RO(_name, _show_name) \
955 static struct padata_sysfs_entry _name##_attr = \
956 __ATTR(_name, 0400, _show_name, NULL)
958 PADATA_ATTR_RW(serial_cpumask
, show_cpumask
, store_cpumask
);
959 PADATA_ATTR_RW(parallel_cpumask
, show_cpumask
, store_cpumask
);
962 * Padata sysfs provides the following objects:
963 * serial_cpumask [RW] - cpumask for serial workers
964 * parallel_cpumask [RW] - cpumask for parallel workers
966 static struct attribute
*padata_default_attrs
[] = {
967 &serial_cpumask_attr
.attr
,
968 ¶llel_cpumask_attr
.attr
,
972 static ssize_t
padata_sysfs_show(struct kobject
*kobj
,
973 struct attribute
*attr
, char *buf
)
975 struct padata_instance
*pinst
;
976 struct padata_sysfs_entry
*pentry
;
979 pinst
= kobj2pinst(kobj
);
980 pentry
= attr2pentry(attr
);
982 ret
= pentry
->show(pinst
, attr
, buf
);
987 static ssize_t
padata_sysfs_store(struct kobject
*kobj
, struct attribute
*attr
,
988 const char *buf
, size_t count
)
990 struct padata_instance
*pinst
;
991 struct padata_sysfs_entry
*pentry
;
994 pinst
= kobj2pinst(kobj
);
995 pentry
= attr2pentry(attr
);
997 ret
= pentry
->store(pinst
, attr
, buf
, count
);
1002 static const struct sysfs_ops padata_sysfs_ops
= {
1003 .show
= padata_sysfs_show
,
1004 .store
= padata_sysfs_store
,
1007 static struct kobj_type padata_attr_type
= {
1008 .sysfs_ops
= &padata_sysfs_ops
,
1009 .default_attrs
= padata_default_attrs
,
1010 .release
= padata_sysfs_release
,
1014 * padata_alloc - allocate and initialize a padata instance and specify
1015 * cpumasks for serial and parallel workers.
1017 * @wq: workqueue to use for the allocated padata instance
1018 * @pcpumask: cpumask that will be used for padata parallelization
1019 * @cbcpumask: cpumask that will be used for padata serialization
1021 * Must be called from a cpus_read_lock() protected region
1023 static struct padata_instance
*padata_alloc(struct workqueue_struct
*wq
,
1024 const struct cpumask
*pcpumask
,
1025 const struct cpumask
*cbcpumask
)
1027 struct padata_instance
*pinst
;
1028 struct parallel_data
*pd
= NULL
;
1030 pinst
= kzalloc(sizeof(struct padata_instance
), GFP_KERNEL
);
1034 if (!alloc_cpumask_var(&pinst
->cpumask
.pcpu
, GFP_KERNEL
))
1036 if (!alloc_cpumask_var(&pinst
->cpumask
.cbcpu
, GFP_KERNEL
)) {
1037 free_cpumask_var(pinst
->cpumask
.pcpu
);
1040 if (!padata_validate_cpumask(pinst
, pcpumask
) ||
1041 !padata_validate_cpumask(pinst
, cbcpumask
))
1042 goto err_free_masks
;
1044 pd
= padata_alloc_pd(pinst
, pcpumask
, cbcpumask
);
1046 goto err_free_masks
;
1048 rcu_assign_pointer(pinst
->pd
, pd
);
1052 cpumask_copy(pinst
->cpumask
.pcpu
, pcpumask
);
1053 cpumask_copy(pinst
->cpumask
.cbcpu
, cbcpumask
);
1057 BLOCKING_INIT_NOTIFIER_HEAD(&pinst
->cpumask_change_notifier
);
1058 kobject_init(&pinst
->kobj
, &padata_attr_type
);
1059 mutex_init(&pinst
->lock
);
1061 #ifdef CONFIG_HOTPLUG_CPU
1062 cpuhp_state_add_instance_nocalls_cpuslocked(hp_online
, &pinst
->node
);
1067 free_cpumask_var(pinst
->cpumask
.pcpu
);
1068 free_cpumask_var(pinst
->cpumask
.cbcpu
);
1076 * padata_alloc_possible - Allocate and initialize padata instance.
1077 * Use the cpu_possible_mask for serial and
1080 * @wq: workqueue to use for the allocated padata instance
1082 * Must be called from a cpus_read_lock() protected region
1084 struct padata_instance
*padata_alloc_possible(struct workqueue_struct
*wq
)
1086 lockdep_assert_cpus_held();
1087 return padata_alloc(wq
, cpu_possible_mask
, cpu_possible_mask
);
1089 EXPORT_SYMBOL(padata_alloc_possible
);
1092 * padata_free - free a padata instance
1094 * @padata_inst: padata instance to free
1096 void padata_free(struct padata_instance
*pinst
)
1098 kobject_put(&pinst
->kobj
);
1100 EXPORT_SYMBOL(padata_free
);
1102 #ifdef CONFIG_HOTPLUG_CPU
1104 static __init
int padata_driver_init(void)
1108 ret
= cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN
, "padata:online",
1110 padata_cpu_prep_down
);
1116 module_init(padata_driver_init
);
1118 static __exit
void padata_driver_exit(void)
1120 cpuhp_remove_multi_state(hp_online
);
1122 module_exit(padata_driver_exit
);