sched: fix the task_group hierarchy for UID grouping

[mirror_ubuntu-kernels.git] / kernel / sched.c

diff --git a/kernel/sched.c b/kernel/sched.c
index 1b7399dfa361ee64b183b9985a4ba4abdad51142..e03b45ccf789772c946562582cc4876dd7dc1da1 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -274,6 +274,14 @@ struct task_group {
 };
 
 #ifdef CONFIG_USER_SCHED
+
+/*
+ * Root task group.
+ *     Every UID task group (including init_task_group aka UID-0) will
+ *     be a child to this group.
+ */
+struct task_group root_task_group;
+
 #ifdef CONFIG_FAIR_GROUP_SCHED
 /* Default task group's sched entity on each cpu */
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
@@ -285,6 +293,8 @@ static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
 static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
 #endif
+#else
+#define root_task_group init_task_group
 #endif
 
 /* task_group_lock serializes add/remove of task groups and also changes to
@@ -7438,10 +7448,11 @@ static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq)
 }
 
 #ifdef CONFIG_FAIR_GROUP_SCHED
-static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
-               struct cfs_rq *cfs_rq, struct sched_entity *se,
-               int cpu, int add)
+static void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
+                               struct sched_entity *se, int cpu, int add,
+                               struct sched_entity *parent)
 {
+       struct rq *rq = cpu_rq(cpu);
        tg->cfs_rq[cpu] = cfs_rq;
        init_cfs_rq(cfs_rq, rq);
        cfs_rq->tg = tg;
@@ -7453,19 +7464,25 @@ static void init_tg_cfs_entry(struct rq *rq, struct task_group *tg,
        if (!se)
                return;
 
-       se->cfs_rq = &rq->cfs;
+       if (!parent)
+               se->cfs_rq = &rq->cfs;
+       else
+               se->cfs_rq = parent->my_q;
+
        se->my_q = cfs_rq;
        se->load.weight = tg->shares;
        se->load.inv_weight = div64_64(1ULL<<32, se->load.weight);
-       se->parent = NULL;
+       se->parent = parent;
 }
 #endif
 
 #ifdef CONFIG_RT_GROUP_SCHED
-static void init_tg_rt_entry(struct rq *rq, struct task_group *tg,
-               struct rt_rq *rt_rq, struct sched_rt_entity *rt_se,
-               int cpu, int add)
+static void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
+               struct sched_rt_entity *rt_se, int cpu, int add,
+               struct sched_rt_entity *parent)
 {
+       struct rq *rq = cpu_rq(cpu);
+
        tg->rt_rq[cpu] = rt_rq;
        init_rt_rq(rt_rq, rq);
        rt_rq->tg = tg;
@@ -7478,9 +7495,14 @@ static void init_tg_rt_entry(struct rq *rq, struct task_group *tg,
        if (!rt_se)
                return;
 
+       if (!parent)
+               rt_se->rt_rq = &rq->rt;
+       else
+               rt_se->rt_rq = parent->my_q;
+
        rt_se->rt_rq = &rq->rt;
        rt_se->my_q = rt_rq;
-       rt_se->parent = NULL;
+       rt_se->parent = parent;
        INIT_LIST_HEAD(&rt_se->run_list);
 }
 #endif
@@ -7495,6 +7517,9 @@ void __init sched_init(void)
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
        alloc_size += 2 * nr_cpu_ids * sizeof(void **);
+#endif
+#ifdef CONFIG_USER_SCHED
+       alloc_size *= 2;
 #endif
        /*
         * As sched_init() is called before page_alloc is setup,
@@ -7509,12 +7534,29 @@ void __init sched_init(void)
 
                init_task_group.cfs_rq = (struct cfs_rq **)ptr;
                ptr += nr_cpu_ids * sizeof(void **);
+
+#ifdef CONFIG_USER_SCHED
+               root_task_group.se = (struct sched_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               root_task_group.cfs_rq = (struct cfs_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+#endif
 #endif
 #ifdef CONFIG_RT_GROUP_SCHED
                init_task_group.rt_se = (struct sched_rt_entity **)ptr;
                ptr += nr_cpu_ids * sizeof(void **);
 
                init_task_group.rt_rq = (struct rt_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+#ifdef CONFIG_USER_SCHED
+               root_task_group.rt_se = (struct sched_rt_entity **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+
+               root_task_group.rt_rq = (struct rt_rq **)ptr;
+               ptr += nr_cpu_ids * sizeof(void **);
+#endif
 #endif
        }
 
@@ -7528,6 +7570,10 @@ void __init sched_init(void)
 #ifdef CONFIG_RT_GROUP_SCHED
        init_rt_bandwidth(&init_task_group.rt_bandwidth,
                        global_rt_period(), global_rt_runtime());
+#ifdef CONFIG_USER_SCHED
+       init_rt_bandwidth(&root_task_group.rt_bandwidth,
+                       global_rt_period(), RUNTIME_INF);
+#endif
 #endif
 
 #ifdef CONFIG_GROUP_SCHED
@@ -7568,8 +7614,10 @@ void __init sched_init(void)
                 * We achieve this by letting init_task_group's tasks sit
                 * directly in rq->cfs (i.e init_task_group->se[] = NULL).
                 */
-               init_tg_cfs_entry(rq, &init_task_group, &rq->cfs, NULL, i, 1);
+               init_tg_cfs_entry(&init_task_group, &rq->cfs, NULL, i, 1, NULL);
 #elif defined CONFIG_USER_SCHED
+               root_task_group.shares = NICE_0_LOAD;
+               init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, 0, NULL);
                /*
                 * In case of task-groups formed thr' the user id of tasks,
                 * init_task_group represents tasks belonging to root user.
@@ -7581,9 +7629,10 @@ void __init sched_init(void)
                 * (init_cfs_rq) and having one entity represent this group of
                 * tasks in rq->cfs (i.e init_task_group->se[] != NULL).
                 */
-               init_tg_cfs_entry(rq, &init_task_group,
+               init_tg_cfs_entry(&init_task_group,
                                &per_cpu(init_cfs_rq, i),
-                               &per_cpu(init_sched_entity, i), i, 1);
+                               &per_cpu(init_sched_entity, i), i, 1,
+                               root_task_group.se[i]);
 
 #endif
 #endif /* CONFIG_FAIR_GROUP_SCHED */
@@ -7592,11 +7641,13 @@ void __init sched_init(void)
 #ifdef CONFIG_RT_GROUP_SCHED
                INIT_LIST_HEAD(&rq->leaf_rt_rq_list);
 #ifdef CONFIG_CGROUP_SCHED
-               init_tg_rt_entry(rq, &init_task_group, &rq->rt, NULL, i, 1);
+               init_tg_rt_entry(&init_task_group, &rq->rt, NULL, i, 1, NULL);
 #elif defined CONFIG_USER_SCHED
-               init_tg_rt_entry(rq, &init_task_group,
+               init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, 0, NULL);
+               init_tg_rt_entry(&init_task_group,
                                &per_cpu(init_rt_rq, i),
-                               &per_cpu(init_sched_rt_entity, i), i, 1);
+                               &per_cpu(init_sched_rt_entity, i), i, 1,
+                               root_task_group.rt_se[i]);
 #endif
 #endif
 
@@ -7798,10 +7849,11 @@ static void free_fair_sched_group(struct task_group *tg)
        kfree(tg->se);
 }
 
-static int alloc_fair_sched_group(struct task_group *tg)
+static
+int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
        struct cfs_rq *cfs_rq;
-       struct sched_entity *se;
+       struct sched_entity *se, *parent_se;
        struct rq *rq;
        int i;
 
@@ -7827,7 +7879,8 @@ static int alloc_fair_sched_group(struct task_group *tg)
                if (!se)
                        goto err;
 
-               init_tg_cfs_entry(rq, tg, cfs_rq, se, i, 0);
+               parent_se = parent ? parent->se[i] : NULL;
+               init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
        }
 
        return 1;
@@ -7851,7 +7904,8 @@ static inline void free_fair_sched_group(struct task_group *tg)
 {
 }
 
-static inline int alloc_fair_sched_group(struct task_group *tg)
+static inline
+int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
 {
        return 1;
 }
@@ -7883,10 +7937,11 @@ static void free_rt_sched_group(struct task_group *tg)
        kfree(tg->rt_se);
 }
 
-static int alloc_rt_sched_group(struct task_group *tg)
+static
+int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
        struct rt_rq *rt_rq;
-       struct sched_rt_entity *rt_se;
+       struct sched_rt_entity *rt_se, *parent_se;
        struct rq *rq;
        int i;
 
@@ -7913,7 +7968,8 @@ static int alloc_rt_sched_group(struct task_group *tg)
                if (!rt_se)
                        goto err;
 
-               init_tg_rt_entry(rq, tg, rt_rq, rt_se, i, 0);
+               parent_se = parent ? parent->rt_se[i] : NULL;
+               init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
        }
 
        return 1;
@@ -7937,7 +7993,8 @@ static inline void free_rt_sched_group(struct task_group *tg)
 {
 }
 
-static inline int alloc_rt_sched_group(struct task_group *tg)
+static inline
+int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
 {
        return 1;
 }
@@ -7960,7 +8017,7 @@ static void free_sched_group(struct task_group *tg)
 }
 
 /* allocate runqueue etc for a new task group */
-struct task_group *sched_create_group(void)
+struct task_group *sched_create_group(struct task_group *parent)
 {
        struct task_group *tg;
        unsigned long flags;
@@ -7970,10 +8027,10 @@ struct task_group *sched_create_group(void)
        if (!tg)
                return ERR_PTR(-ENOMEM);
 
-       if (!alloc_fair_sched_group(tg))
+       if (!alloc_fair_sched_group(tg, parent))
                goto err;
 
-       if (!alloc_rt_sched_group(tg))
+       if (!alloc_rt_sched_group(tg, parent))
                goto err;
 
        spin_lock_irqsave(&task_group_lock, flags);
@@ -8084,6 +8141,12 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
        int i;
        unsigned long flags;
 
+       /*
+        * We can't change the weight of the root cgroup.
+        */
+       if (!tg->se[0])
+               return -EINVAL;
+
        /*
         * A weight of 0 or 1 can cause arithmetics problems.
         * (The default weight is 1024 - so there's no practical
@@ -8327,7 +8390,7 @@ static inline struct task_group *cgroup_tg(struct cgroup *cgrp)
 static struct cgroup_subsys_state *
 cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
-       struct task_group *tg;
+       struct task_group *tg, *parent;
 
        if (!cgrp->parent) {
                /* This is early initialization for the top cgroup */
@@ -8335,11 +8398,8 @@ cpu_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cgrp)
                return &init_task_group.css;
        }
 
-       /* we support only 1-level deep hierarchical scheduler atm */
-       if (cgrp->parent->parent)
-               return ERR_PTR(-EINVAL);
-
-       tg = sched_create_group();
+       parent = cgroup_tg(cgrp->parent);
+       tg = sched_create_group(parent);
        if (IS_ERR(tg))
                return ERR_PTR(-ENOMEM);