]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - mm/allocpercpu.c
2 * linux/mm/allocpercpu.c
4 * Separated from slab.c August 11, 2006 Christoph Lameter
7 #include <linux/module.h>
9 #ifndef cache_line_size
10 #define cache_line_size() L1_CACHE_BYTES
14 * percpu_depopulate - depopulate per-cpu data for given cpu
15 * @__pdata: per-cpu data to depopulate
16 * @cpu: depopulate per-cpu data for this cpu
18 * Depopulating per-cpu data for a cpu going offline would be a typical
19 * use case. You need to register a cpu hotplug handler for that purpose.
21 static void percpu_depopulate(void *__pdata
, int cpu
)
23 struct percpu_data
*pdata
= __percpu_disguise(__pdata
);
25 kfree(pdata
->ptrs
[cpu
]);
26 pdata
->ptrs
[cpu
] = NULL
;
30 * percpu_depopulate_mask - depopulate per-cpu data for some cpu's
31 * @__pdata: per-cpu data to depopulate
32 * @mask: depopulate per-cpu data for cpu's selected through mask bits
34 static void __percpu_depopulate_mask(void *__pdata
, cpumask_t
*mask
)
37 for_each_cpu_mask_nr(cpu
, *mask
)
38 percpu_depopulate(__pdata
, cpu
);
41 #define percpu_depopulate_mask(__pdata, mask) \
42 __percpu_depopulate_mask((__pdata), &(mask))
45 * percpu_populate - populate per-cpu data for given cpu
46 * @__pdata: per-cpu data to populate further
47 * @size: size of per-cpu object
48 * @gfp: may sleep or not etc.
49 * @cpu: populate per-data for this cpu
51 * Populating per-cpu data for a cpu coming online would be a typical
52 * use case. You need to register a cpu hotplug handler for that purpose.
53 * Per-cpu object is populated with zeroed buffer.
55 static void *percpu_populate(void *__pdata
, size_t size
, gfp_t gfp
, int cpu
)
57 struct percpu_data
*pdata
= __percpu_disguise(__pdata
);
58 int node
= cpu_to_node(cpu
);
61 * We should make sure each CPU gets private memory.
63 size
= roundup(size
, cache_line_size());
65 BUG_ON(pdata
->ptrs
[cpu
]);
66 if (node_online(node
))
67 pdata
->ptrs
[cpu
] = kmalloc_node(size
, gfp
|__GFP_ZERO
, node
);
69 pdata
->ptrs
[cpu
] = kzalloc(size
, gfp
);
70 return pdata
->ptrs
[cpu
];
74 * percpu_populate_mask - populate per-cpu data for more cpu's
75 * @__pdata: per-cpu data to populate further
76 * @size: size of per-cpu object
77 * @gfp: may sleep or not etc.
78 * @mask: populate per-cpu data for cpu's selected through mask bits
80 * Per-cpu objects are populated with zeroed buffers.
82 static int __percpu_populate_mask(void *__pdata
, size_t size
, gfp_t gfp
,
88 cpus_clear(populated
);
89 for_each_cpu_mask_nr(cpu
, *mask
)
90 if (unlikely(!percpu_populate(__pdata
, size
, gfp
, cpu
))) {
91 __percpu_depopulate_mask(__pdata
, &populated
);
94 cpu_set(cpu
, populated
);
98 #define percpu_populate_mask(__pdata, size, gfp, mask) \
99 __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
102 * alloc_percpu - initial setup of per-cpu data
103 * @size: size of per-cpu object
106 * Allocate dynamic percpu area. Percpu objects are populated with
109 void *__alloc_percpu(size_t size
, size_t align
)
112 * We allocate whole cache lines to avoid false sharing
114 size_t sz
= roundup(nr_cpu_ids
* sizeof(void *), cache_line_size());
115 void *pdata
= kzalloc(sz
, GFP_KERNEL
);
116 void *__pdata
= __percpu_disguise(pdata
);
119 * Can't easily make larger alignment work with kmalloc. WARN
120 * on it. Larger alignment should only be used for module
121 * percpu sections on SMP for which this path isn't used.
123 WARN_ON_ONCE(align
> SMP_CACHE_BYTES
);
125 if (unlikely(!pdata
))
127 if (likely(!__percpu_populate_mask(__pdata
, size
, GFP_KERNEL
,
133 EXPORT_SYMBOL_GPL(__alloc_percpu
);
136 * free_percpu - final cleanup of per-cpu data
137 * @__pdata: object to clean up
139 * We simply clean up any per-cpu object left. No need for the client to
140 * track and specify through a bis mask which per-cpu objects are to free.
142 void free_percpu(void *__pdata
)
144 if (unlikely(!__pdata
))
146 __percpu_depopulate_mask(__pdata
, &cpu_possible_map
);
147 kfree(__percpu_disguise(__pdata
));
149 EXPORT_SYMBOL_GPL(free_percpu
);