]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * linux/fs/file.c | |
3 | * | |
4 | * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes | |
5 | * | |
6 | * Manage the dynamic fd arrays in the process files_struct. | |
7 | */ | |
8 | ||
9 | #include <linux/fs.h> | |
10 | #include <linux/mm.h> | |
11 | #include <linux/time.h> | |
12 | #include <linux/slab.h> | |
13 | #include <linux/vmalloc.h> | |
14 | #include <linux/file.h> | |
15 | #include <linux/bitops.h> | |
16 | #include <linux/interrupt.h> | |
17 | #include <linux/spinlock.h> | |
18 | #include <linux/rcupdate.h> | |
19 | #include <linux/workqueue.h> | |
20 | ||
21 | struct fdtable_defer { | |
22 | spinlock_t lock; | |
23 | struct work_struct wq; | |
24 | struct fdtable *next; | |
25 | }; | |
26 | ||
27 | /* | |
28 | * We use this list to defer free fdtables that have vmalloced | |
29 | * sets/arrays. By keeping a per-cpu list, we avoid having to embed | |
30 | * the work_struct in fdtable itself which avoids a 64 byte (i386) increase in | |
31 | * this per-task structure. | |
32 | */ | |
33 | static DEFINE_PER_CPU(struct fdtable_defer, fdtable_defer_list); | |
34 | ||
35 | ||
36 | /* | |
37 | * Allocate an fd array, using kmalloc or vmalloc. | |
38 | * Note: the array isn't cleared at allocation time. | |
39 | */ | |
40 | struct file ** alloc_fd_array(int num) | |
41 | { | |
42 | struct file **new_fds; | |
43 | int size = num * sizeof(struct file *); | |
44 | ||
45 | if (size <= PAGE_SIZE) | |
46 | new_fds = (struct file **) kmalloc(size, GFP_KERNEL); | |
47 | else | |
48 | new_fds = (struct file **) vmalloc(size); | |
49 | return new_fds; | |
50 | } | |
51 | ||
52 | void free_fd_array(struct file **array, int num) | |
53 | { | |
54 | int size = num * sizeof(struct file *); | |
55 | ||
56 | if (!array) { | |
57 | printk (KERN_ERR "free_fd_array: array = 0 (num = %d)\n", num); | |
58 | return; | |
59 | } | |
60 | ||
61 | if (num <= NR_OPEN_DEFAULT) /* Don't free the embedded fd array! */ | |
62 | return; | |
63 | else if (size <= PAGE_SIZE) | |
64 | kfree(array); | |
65 | else | |
66 | vfree(array); | |
67 | } | |
68 | ||
69 | static void __free_fdtable(struct fdtable *fdt) | |
70 | { | |
71 | free_fdset(fdt->open_fds, fdt->max_fds); | |
72 | free_fdset(fdt->close_on_exec, fdt->max_fds); | |
73 | free_fd_array(fdt->fd, fdt->max_fds); | |
74 | kfree(fdt); | |
75 | } | |
76 | ||
77 | static void free_fdtable_work(struct work_struct *work) | |
78 | { | |
79 | struct fdtable_defer *f = | |
80 | container_of(work, struct fdtable_defer, wq); | |
81 | struct fdtable *fdt; | |
82 | ||
83 | spin_lock_bh(&f->lock); | |
84 | fdt = f->next; | |
85 | f->next = NULL; | |
86 | spin_unlock_bh(&f->lock); | |
87 | while(fdt) { | |
88 | struct fdtable *next = fdt->next; | |
89 | __free_fdtable(fdt); | |
90 | fdt = next; | |
91 | } | |
92 | } | |
93 | ||
94 | void free_fdtable_rcu(struct rcu_head *rcu) | |
95 | { | |
96 | struct fdtable *fdt = container_of(rcu, struct fdtable, rcu); | |
97 | int fdset_size, fdarray_size; | |
98 | struct fdtable_defer *fddef; | |
99 | ||
100 | BUG_ON(!fdt); | |
101 | fdset_size = fdt->max_fds / 8; | |
102 | fdarray_size = fdt->max_fds * sizeof(struct file *); | |
103 | ||
104 | if (fdt->max_fds <= NR_OPEN_DEFAULT) { | |
105 | /* | |
106 | * This fdtable is embedded in the files structure and that | |
107 | * structure itself is getting destroyed. | |
108 | */ | |
109 | kmem_cache_free(files_cachep, | |
110 | container_of(fdt, struct files_struct, fdtab)); | |
111 | return; | |
112 | } | |
113 | if (fdset_size <= PAGE_SIZE && fdarray_size <= PAGE_SIZE) { | |
114 | kfree(fdt->open_fds); | |
115 | kfree(fdt->close_on_exec); | |
116 | kfree(fdt->fd); | |
117 | kfree(fdt); | |
118 | } else { | |
119 | fddef = &get_cpu_var(fdtable_defer_list); | |
120 | spin_lock(&fddef->lock); | |
121 | fdt->next = fddef->next; | |
122 | fddef->next = fdt; | |
123 | /* vmallocs are handled from the workqueue context */ | |
124 | schedule_work(&fddef->wq); | |
125 | spin_unlock(&fddef->lock); | |
126 | put_cpu_var(fdtable_defer_list); | |
127 | } | |
128 | } | |
129 | ||
130 | /* | |
131 | * Expand the fdset in the files_struct. Called with the files spinlock | |
132 | * held for write. | |
133 | */ | |
134 | static void copy_fdtable(struct fdtable *nfdt, struct fdtable *fdt) | |
135 | { | |
136 | int i; | |
137 | int count; | |
138 | ||
139 | BUG_ON(nfdt->max_fds < fdt->max_fds); | |
140 | /* Copy the existing tables and install the new pointers */ | |
141 | ||
142 | i = fdt->max_fds / (sizeof(unsigned long) * 8); | |
143 | count = (nfdt->max_fds - fdt->max_fds) / 8; | |
144 | ||
145 | /* | |
146 | * Don't copy the entire array if the current fdset is | |
147 | * not yet initialised. | |
148 | */ | |
149 | if (i) { | |
150 | memcpy (nfdt->open_fds, fdt->open_fds, | |
151 | fdt->max_fds/8); | |
152 | memcpy (nfdt->close_on_exec, fdt->close_on_exec, | |
153 | fdt->max_fds/8); | |
154 | memset (&nfdt->open_fds->fds_bits[i], 0, count); | |
155 | memset (&nfdt->close_on_exec->fds_bits[i], 0, count); | |
156 | } | |
157 | ||
158 | /* Don't copy/clear the array if we are creating a new | |
159 | fd array for fork() */ | |
160 | if (fdt->max_fds) { | |
161 | memcpy(nfdt->fd, fdt->fd, | |
162 | fdt->max_fds * sizeof(struct file *)); | |
163 | /* clear the remainder of the array */ | |
164 | memset(&nfdt->fd[fdt->max_fds], 0, | |
165 | (nfdt->max_fds - fdt->max_fds) * | |
166 | sizeof(struct file *)); | |
167 | } | |
168 | } | |
169 | ||
170 | /* | |
171 | * Allocate an fdset array, using kmalloc or vmalloc. | |
172 | * Note: the array isn't cleared at allocation time. | |
173 | */ | |
174 | fd_set * alloc_fdset(int num) | |
175 | { | |
176 | fd_set *new_fdset; | |
177 | int size = num / 8; | |
178 | ||
179 | if (size <= PAGE_SIZE) | |
180 | new_fdset = (fd_set *) kmalloc(size, GFP_KERNEL); | |
181 | else | |
182 | new_fdset = (fd_set *) vmalloc(size); | |
183 | return new_fdset; | |
184 | } | |
185 | ||
186 | void free_fdset(fd_set *array, int num) | |
187 | { | |
188 | if (num <= NR_OPEN_DEFAULT) /* Don't free an embedded fdset */ | |
189 | return; | |
190 | else if (num <= 8 * PAGE_SIZE) | |
191 | kfree(array); | |
192 | else | |
193 | vfree(array); | |
194 | } | |
195 | ||
196 | static struct fdtable *alloc_fdtable(int nr) | |
197 | { | |
198 | struct fdtable *fdt = NULL; | |
199 | int nfds = 0; | |
200 | fd_set *new_openset = NULL, *new_execset = NULL; | |
201 | struct file **new_fds; | |
202 | ||
203 | fdt = kzalloc(sizeof(*fdt), GFP_KERNEL); | |
204 | if (!fdt) | |
205 | goto out; | |
206 | ||
207 | nfds = NR_OPEN_DEFAULT; | |
208 | /* | |
209 | * Expand to the max in easy steps, and keep expanding it until | |
210 | * we have enough for the requested fd array size. | |
211 | */ | |
212 | do { | |
213 | #if NR_OPEN_DEFAULT < 256 | |
214 | if (nfds < 256) | |
215 | nfds = 256; | |
216 | else | |
217 | #endif | |
218 | if (nfds < (PAGE_SIZE / sizeof(struct file *))) | |
219 | nfds = PAGE_SIZE / sizeof(struct file *); | |
220 | else { | |
221 | nfds = nfds * 2; | |
222 | if (nfds > NR_OPEN) | |
223 | nfds = NR_OPEN; | |
224 | } | |
225 | } while (nfds <= nr); | |
226 | ||
227 | new_openset = alloc_fdset(nfds); | |
228 | new_execset = alloc_fdset(nfds); | |
229 | if (!new_openset || !new_execset) | |
230 | goto out; | |
231 | fdt->open_fds = new_openset; | |
232 | fdt->close_on_exec = new_execset; | |
233 | ||
234 | new_fds = alloc_fd_array(nfds); | |
235 | if (!new_fds) | |
236 | goto out; | |
237 | fdt->fd = new_fds; | |
238 | fdt->max_fds = nfds; | |
239 | return fdt; | |
240 | out: | |
241 | free_fdset(new_openset, nfds); | |
242 | free_fdset(new_execset, nfds); | |
243 | kfree(fdt); | |
244 | return NULL; | |
245 | } | |
246 | ||
247 | /* | |
248 | * Expand the file descriptor table. | |
249 | * This function will allocate a new fdtable and both fd array and fdset, of | |
250 | * the given size. | |
251 | * Return <0 error code on error; 1 on successful completion. | |
252 | * The files->file_lock should be held on entry, and will be held on exit. | |
253 | */ | |
254 | static int expand_fdtable(struct files_struct *files, int nr) | |
255 | __releases(files->file_lock) | |
256 | __acquires(files->file_lock) | |
257 | { | |
258 | struct fdtable *new_fdt, *cur_fdt; | |
259 | ||
260 | spin_unlock(&files->file_lock); | |
261 | new_fdt = alloc_fdtable(nr); | |
262 | spin_lock(&files->file_lock); | |
263 | if (!new_fdt) | |
264 | return -ENOMEM; | |
265 | /* | |
266 | * Check again since another task may have expanded the fd table while | |
267 | * we dropped the lock | |
268 | */ | |
269 | cur_fdt = files_fdtable(files); | |
270 | if (nr >= cur_fdt->max_fds) { | |
271 | /* Continue as planned */ | |
272 | copy_fdtable(new_fdt, cur_fdt); | |
273 | rcu_assign_pointer(files->fdt, new_fdt); | |
274 | if (cur_fdt->max_fds > NR_OPEN_DEFAULT) | |
275 | call_rcu(&cur_fdt->rcu, free_fdtable_rcu); | |
276 | } else { | |
277 | /* Somebody else expanded, so undo our attempt */ | |
278 | __free_fdtable(new_fdt); | |
279 | } | |
280 | return 1; | |
281 | } | |
282 | ||
283 | /* | |
284 | * Expand files. | |
285 | * This function will expand the file structures, if the requested size exceeds | |
286 | * the current capacity and there is room for expansion. | |
287 | * Return <0 error code on error; 0 when nothing done; 1 when files were | |
288 | * expanded and execution may have blocked. | |
289 | * The files->file_lock should be held on entry, and will be held on exit. | |
290 | */ | |
291 | int expand_files(struct files_struct *files, int nr) | |
292 | { | |
293 | struct fdtable *fdt; | |
294 | ||
295 | fdt = files_fdtable(files); | |
296 | /* Do we need to expand? */ | |
297 | if (nr < fdt->max_fds) | |
298 | return 0; | |
299 | /* Can we expand? */ | |
300 | if (nr >= NR_OPEN) | |
301 | return -EMFILE; | |
302 | ||
303 | /* All good, so we try */ | |
304 | return expand_fdtable(files, nr); | |
305 | } | |
306 | ||
307 | static void __devinit fdtable_defer_list_init(int cpu) | |
308 | { | |
309 | struct fdtable_defer *fddef = &per_cpu(fdtable_defer_list, cpu); | |
310 | spin_lock_init(&fddef->lock); | |
311 | INIT_WORK(&fddef->wq, free_fdtable_work); | |
312 | fddef->next = NULL; | |
313 | } | |
314 | ||
315 | void __init files_defer_init(void) | |
316 | { | |
317 | int i; | |
318 | for_each_possible_cpu(i) | |
319 | fdtable_defer_list_init(i); | |
320 | } |