mm/shmem.c: mark expected switch fall-through

[mirror_ubuntu-bionic-kernel.git] / kernel / padata.c

diff --git a/kernel/padata.c b/kernel/padata.c
index 868f947166d70bf528fb3feefb18577538455076..f262c9a4e70ab76d5ee5e748542845cb99e8eaff 100644 (file)
--- a/kernel/padata.c
+++ b/kernel/padata.c
@@ -131,6 +131,7 @@ int padata_do_parallel(struct padata_instance *pinst,
        padata->cb_cpu = cb_cpu;
 
        target_cpu = padata_cpu_hash(pd);
+       padata->cpu = target_cpu;
        queue = per_cpu_ptr(pd->pqueue, target_cpu);
 
        spin_lock(&queue->parallel.lock);
@@ -275,11 +276,51 @@ static void padata_reorder(struct parallel_data *pd)
        return;
 }
 
+static void invoke_padata_reorder(struct work_struct *work)
+{
+       struct padata_parallel_queue *pqueue;
+       struct parallel_data *pd;
+
+       local_bh_disable();
+       pqueue = container_of(work, struct padata_parallel_queue, reorder_work);
+       pd = pqueue->pd;
+       padata_reorder(pd);
+       local_bh_enable();
+}
+
 static void padata_reorder_timer(unsigned long arg)
 {
        struct parallel_data *pd = (struct parallel_data *)arg;
+       unsigned int weight;
+       int target_cpu, cpu;
 
-       padata_reorder(pd);
+       cpu = get_cpu();
+
+       /* We don't lock pd here to not interfere with parallel processing
+        * padata_reorder() calls on other CPUs. We just need any CPU out of
+        * the cpumask.pcpu set. It would be nice if it's the right one but
+        * it doesn't matter if we're off to the next one by using an outdated
+        * pd->processed value.
+        */
+       weight = cpumask_weight(pd->cpumask.pcpu);
+       target_cpu = padata_index_to_cpu(pd, pd->processed % weight);
+
+       /* ensure to call the reorder callback on the correct CPU */
+       if (cpu != target_cpu) {
+               struct padata_parallel_queue *pqueue;
+               struct padata_instance *pinst;
+
+               /* The timer function is serialized wrt itself -- no locking
+                * needed.
+                */
+               pinst = pd->pinst;
+               pqueue = per_cpu_ptr(pd->pqueue, target_cpu);
+               queue_work_on(target_cpu, pinst->wq, &pqueue->reorder_work);
+       } else {
+               padata_reorder(pd);
+       }
+
+       put_cpu();
 }
 
 static void padata_serial_worker(struct work_struct *serial_work)
@@ -323,10 +364,21 @@ void padata_do_serial(struct padata_priv *padata)
        int cpu;
        struct padata_parallel_queue *pqueue;
        struct parallel_data *pd;
+       int reorder_via_wq = 0;
 
        pd = padata->pd;
 
        cpu = get_cpu();
+
+       /* We need to run on the same CPU padata_do_parallel(.., padata, ..)
+        * was called on -- or, at least, enqueue the padata object into the
+        * correct per-cpu queue.
+        */
+       if (cpu != padata->cpu) {
+               reorder_via_wq = 1;
+               cpu = padata->cpu;
+       }
+
        pqueue = per_cpu_ptr(pd->pqueue, cpu);
 
        spin_lock(&pqueue->reorder.lock);
@@ -336,7 +388,13 @@ void padata_do_serial(struct padata_priv *padata)
 
        put_cpu();
 
-       padata_reorder(pd);
+       /* If we're running on the wrong CPU, call padata_reorder() via a
+        * kernel worker.
+        */
+       if (reorder_via_wq)
+               queue_work_on(cpu, pd->pinst->wq, &pqueue->reorder_work);
+       else
+               padata_reorder(pd);
 }
 EXPORT_SYMBOL(padata_do_serial);
 
@@ -384,8 +442,14 @@ static void padata_init_pqueues(struct parallel_data *pd)
        struct padata_parallel_queue *pqueue;
 
        cpu_index = 0;
-       for_each_cpu(cpu, pd->cpumask.pcpu) {
+       for_each_possible_cpu(cpu) {
                pqueue = per_cpu_ptr(pd->pqueue, cpu);
+
+               if (!cpumask_test_cpu(cpu, pd->cpumask.pcpu)) {
+                       pqueue->cpu_index = -1;
+                       continue;
+               }
+
                pqueue->pd = pd;
                pqueue->cpu_index = cpu_index;
                cpu_index++;
@@ -393,6 +457,7 @@ static void padata_init_pqueues(struct parallel_data *pd)
                __padata_list_init(&pqueue->reorder);
                __padata_list_init(&pqueue->parallel);
                INIT_WORK(&pqueue->work, padata_parallel_worker);
+               INIT_WORK(&pqueue->reorder_work, invoke_padata_reorder);
                atomic_set(&pqueue->num_obj, 0);
        }
 }