]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * linux/arch/i386/kernel/i387.c | |
3 | * | |
4 | * Copyright (C) 1994 Linus Torvalds | |
5 | * | |
6 | * Pentium III FXSR, SSE support | |
7 | * General FPU state handling cleanups | |
8 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
9 | */ | |
10 | ||
11 | #include <linux/config.h> | |
12 | #include <linux/sched.h> | |
129f6946 | 13 | #include <linux/module.h> |
1da177e4 LT |
14 | #include <asm/processor.h> |
15 | #include <asm/i387.h> | |
16 | #include <asm/math_emu.h> | |
17 | #include <asm/sigcontext.h> | |
18 | #include <asm/user.h> | |
19 | #include <asm/ptrace.h> | |
20 | #include <asm/uaccess.h> | |
21 | ||
22 | #ifdef CONFIG_MATH_EMULATION | |
23 | #define HAVE_HWFP (boot_cpu_data.hard_math) | |
24 | #else | |
25 | #define HAVE_HWFP 1 | |
26 | #endif | |
27 | ||
28 | static unsigned long mxcsr_feature_mask = 0xffffffff; | |
29 | ||
30 | void mxcsr_feature_mask_init(void) | |
31 | { | |
32 | unsigned long mask = 0; | |
33 | clts(); | |
34 | if (cpu_has_fxsr) { | |
35 | memset(¤t->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
36 | asm volatile("fxsave %0" : : "m" (current->thread.i387.fxsave)); | |
37 | mask = current->thread.i387.fxsave.mxcsr_mask; | |
38 | if (mask == 0) mask = 0x0000ffbf; | |
39 | } | |
40 | mxcsr_feature_mask &= mask; | |
41 | stts(); | |
42 | } | |
43 | ||
44 | /* | |
45 | * The _current_ task is using the FPU for the first time | |
46 | * so initialize it and set the mxcsr to its default | |
47 | * value at reset if we support XMM instructions and then | |
48 | * remeber the current task has used the FPU. | |
49 | */ | |
50 | void init_fpu(struct task_struct *tsk) | |
51 | { | |
52 | if (cpu_has_fxsr) { | |
53 | memset(&tsk->thread.i387.fxsave, 0, sizeof(struct i387_fxsave_struct)); | |
54 | tsk->thread.i387.fxsave.cwd = 0x37f; | |
55 | if (cpu_has_xmm) | |
56 | tsk->thread.i387.fxsave.mxcsr = 0x1f80; | |
57 | } else { | |
58 | memset(&tsk->thread.i387.fsave, 0, sizeof(struct i387_fsave_struct)); | |
59 | tsk->thread.i387.fsave.cwd = 0xffff037fu; | |
60 | tsk->thread.i387.fsave.swd = 0xffff0000u; | |
61 | tsk->thread.i387.fsave.twd = 0xffffffffu; | |
62 | tsk->thread.i387.fsave.fos = 0xffff0000u; | |
63 | } | |
64 | /* only the device not available exception or ptrace can call init_fpu */ | |
65 | set_stopped_child_used_math(tsk); | |
66 | } | |
67 | ||
68 | /* | |
69 | * FPU lazy state save handling. | |
70 | */ | |
71 | ||
72 | void kernel_fpu_begin(void) | |
73 | { | |
74 | struct thread_info *thread = current_thread_info(); | |
75 | ||
76 | preempt_disable(); | |
77 | if (thread->status & TS_USEDFPU) { | |
78 | __save_init_fpu(thread->task); | |
79 | return; | |
80 | } | |
81 | clts(); | |
82 | } | |
129f6946 | 83 | EXPORT_SYMBOL_GPL(kernel_fpu_begin); |
1da177e4 | 84 | |
1da177e4 LT |
85 | /* |
86 | * FPU tag word conversions. | |
87 | */ | |
88 | ||
89 | static inline unsigned short twd_i387_to_fxsr( unsigned short twd ) | |
90 | { | |
91 | unsigned int tmp; /* to avoid 16 bit prefixes in the code */ | |
92 | ||
93 | /* Transform each pair of bits into 01 (valid) or 00 (empty) */ | |
94 | tmp = ~twd; | |
95 | tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */ | |
96 | /* and move the valid bits to the lower byte. */ | |
97 | tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */ | |
98 | tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */ | |
99 | tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */ | |
100 | return tmp; | |
101 | } | |
102 | ||
103 | static inline unsigned long twd_fxsr_to_i387( struct i387_fxsave_struct *fxsave ) | |
104 | { | |
105 | struct _fpxreg *st = NULL; | |
106 | unsigned long tos = (fxsave->swd >> 11) & 7; | |
107 | unsigned long twd = (unsigned long) fxsave->twd; | |
108 | unsigned long tag; | |
109 | unsigned long ret = 0xffff0000u; | |
110 | int i; | |
111 | ||
112 | #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16); | |
113 | ||
114 | for ( i = 0 ; i < 8 ; i++ ) { | |
115 | if ( twd & 0x1 ) { | |
116 | st = FPREG_ADDR( fxsave, (i - tos) & 7 ); | |
117 | ||
118 | switch ( st->exponent & 0x7fff ) { | |
119 | case 0x7fff: | |
120 | tag = 2; /* Special */ | |
121 | break; | |
122 | case 0x0000: | |
123 | if ( !st->significand[0] && | |
124 | !st->significand[1] && | |
125 | !st->significand[2] && | |
126 | !st->significand[3] ) { | |
127 | tag = 1; /* Zero */ | |
128 | } else { | |
129 | tag = 2; /* Special */ | |
130 | } | |
131 | break; | |
132 | default: | |
133 | if ( st->significand[3] & 0x8000 ) { | |
134 | tag = 0; /* Valid */ | |
135 | } else { | |
136 | tag = 2; /* Special */ | |
137 | } | |
138 | break; | |
139 | } | |
140 | } else { | |
141 | tag = 3; /* Empty */ | |
142 | } | |
143 | ret |= (tag << (2 * i)); | |
144 | twd = twd >> 1; | |
145 | } | |
146 | return ret; | |
147 | } | |
148 | ||
149 | /* | |
150 | * FPU state interaction. | |
151 | */ | |
152 | ||
153 | unsigned short get_fpu_cwd( struct task_struct *tsk ) | |
154 | { | |
155 | if ( cpu_has_fxsr ) { | |
156 | return tsk->thread.i387.fxsave.cwd; | |
157 | } else { | |
158 | return (unsigned short)tsk->thread.i387.fsave.cwd; | |
159 | } | |
160 | } | |
161 | ||
162 | unsigned short get_fpu_swd( struct task_struct *tsk ) | |
163 | { | |
164 | if ( cpu_has_fxsr ) { | |
165 | return tsk->thread.i387.fxsave.swd; | |
166 | } else { | |
167 | return (unsigned short)tsk->thread.i387.fsave.swd; | |
168 | } | |
169 | } | |
170 | ||
171 | #if 0 | |
172 | unsigned short get_fpu_twd( struct task_struct *tsk ) | |
173 | { | |
174 | if ( cpu_has_fxsr ) { | |
175 | return tsk->thread.i387.fxsave.twd; | |
176 | } else { | |
177 | return (unsigned short)tsk->thread.i387.fsave.twd; | |
178 | } | |
179 | } | |
180 | #endif /* 0 */ | |
181 | ||
182 | unsigned short get_fpu_mxcsr( struct task_struct *tsk ) | |
183 | { | |
184 | if ( cpu_has_xmm ) { | |
185 | return tsk->thread.i387.fxsave.mxcsr; | |
186 | } else { | |
187 | return 0x1f80; | |
188 | } | |
189 | } | |
190 | ||
191 | #if 0 | |
192 | ||
193 | void set_fpu_cwd( struct task_struct *tsk, unsigned short cwd ) | |
194 | { | |
195 | if ( cpu_has_fxsr ) { | |
196 | tsk->thread.i387.fxsave.cwd = cwd; | |
197 | } else { | |
198 | tsk->thread.i387.fsave.cwd = ((long)cwd | 0xffff0000u); | |
199 | } | |
200 | } | |
201 | ||
202 | void set_fpu_swd( struct task_struct *tsk, unsigned short swd ) | |
203 | { | |
204 | if ( cpu_has_fxsr ) { | |
205 | tsk->thread.i387.fxsave.swd = swd; | |
206 | } else { | |
207 | tsk->thread.i387.fsave.swd = ((long)swd | 0xffff0000u); | |
208 | } | |
209 | } | |
210 | ||
211 | void set_fpu_twd( struct task_struct *tsk, unsigned short twd ) | |
212 | { | |
213 | if ( cpu_has_fxsr ) { | |
214 | tsk->thread.i387.fxsave.twd = twd_i387_to_fxsr(twd); | |
215 | } else { | |
216 | tsk->thread.i387.fsave.twd = ((long)twd | 0xffff0000u); | |
217 | } | |
218 | } | |
219 | ||
220 | #endif /* 0 */ | |
221 | ||
222 | /* | |
223 | * FXSR floating point environment conversions. | |
224 | */ | |
225 | ||
226 | static int convert_fxsr_to_user( struct _fpstate __user *buf, | |
227 | struct i387_fxsave_struct *fxsave ) | |
228 | { | |
229 | unsigned long env[7]; | |
230 | struct _fpreg __user *to; | |
231 | struct _fpxreg *from; | |
232 | int i; | |
233 | ||
234 | env[0] = (unsigned long)fxsave->cwd | 0xffff0000ul; | |
235 | env[1] = (unsigned long)fxsave->swd | 0xffff0000ul; | |
236 | env[2] = twd_fxsr_to_i387(fxsave); | |
237 | env[3] = fxsave->fip; | |
238 | env[4] = fxsave->fcs | ((unsigned long)fxsave->fop << 16); | |
239 | env[5] = fxsave->foo; | |
240 | env[6] = fxsave->fos; | |
241 | ||
242 | if ( __copy_to_user( buf, env, 7 * sizeof(unsigned long) ) ) | |
243 | return 1; | |
244 | ||
245 | to = &buf->_st[0]; | |
246 | from = (struct _fpxreg *) &fxsave->st_space[0]; | |
247 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
248 | unsigned long __user *t = (unsigned long __user *)to; | |
249 | unsigned long *f = (unsigned long *)from; | |
250 | ||
251 | if (__put_user(*f, t) || | |
252 | __put_user(*(f + 1), t + 1) || | |
253 | __put_user(from->exponent, &to->exponent)) | |
254 | return 1; | |
255 | } | |
256 | return 0; | |
257 | } | |
258 | ||
259 | static int convert_fxsr_from_user( struct i387_fxsave_struct *fxsave, | |
260 | struct _fpstate __user *buf ) | |
261 | { | |
262 | unsigned long env[7]; | |
263 | struct _fpxreg *to; | |
264 | struct _fpreg __user *from; | |
265 | int i; | |
266 | ||
267 | if ( __copy_from_user( env, buf, 7 * sizeof(long) ) ) | |
268 | return 1; | |
269 | ||
270 | fxsave->cwd = (unsigned short)(env[0] & 0xffff); | |
271 | fxsave->swd = (unsigned short)(env[1] & 0xffff); | |
272 | fxsave->twd = twd_i387_to_fxsr((unsigned short)(env[2] & 0xffff)); | |
273 | fxsave->fip = env[3]; | |
274 | fxsave->fop = (unsigned short)((env[4] & 0xffff0000ul) >> 16); | |
275 | fxsave->fcs = (env[4] & 0xffff); | |
276 | fxsave->foo = env[5]; | |
277 | fxsave->fos = env[6]; | |
278 | ||
279 | to = (struct _fpxreg *) &fxsave->st_space[0]; | |
280 | from = &buf->_st[0]; | |
281 | for ( i = 0 ; i < 8 ; i++, to++, from++ ) { | |
282 | unsigned long *t = (unsigned long *)to; | |
283 | unsigned long __user *f = (unsigned long __user *)from; | |
284 | ||
285 | if (__get_user(*t, f) || | |
286 | __get_user(*(t + 1), f + 1) || | |
287 | __get_user(to->exponent, &from->exponent)) | |
288 | return 1; | |
289 | } | |
290 | return 0; | |
291 | } | |
292 | ||
293 | /* | |
294 | * Signal frame handlers. | |
295 | */ | |
296 | ||
297 | static inline int save_i387_fsave( struct _fpstate __user *buf ) | |
298 | { | |
299 | struct task_struct *tsk = current; | |
300 | ||
301 | unlazy_fpu( tsk ); | |
302 | tsk->thread.i387.fsave.status = tsk->thread.i387.fsave.swd; | |
303 | if ( __copy_to_user( buf, &tsk->thread.i387.fsave, | |
304 | sizeof(struct i387_fsave_struct) ) ) | |
305 | return -1; | |
306 | return 1; | |
307 | } | |
308 | ||
309 | static int save_i387_fxsave( struct _fpstate __user *buf ) | |
310 | { | |
311 | struct task_struct *tsk = current; | |
312 | int err = 0; | |
313 | ||
314 | unlazy_fpu( tsk ); | |
315 | ||
316 | if ( convert_fxsr_to_user( buf, &tsk->thread.i387.fxsave ) ) | |
317 | return -1; | |
318 | ||
319 | err |= __put_user( tsk->thread.i387.fxsave.swd, &buf->status ); | |
320 | err |= __put_user( X86_FXSR_MAGIC, &buf->magic ); | |
321 | if ( err ) | |
322 | return -1; | |
323 | ||
324 | if ( __copy_to_user( &buf->_fxsr_env[0], &tsk->thread.i387.fxsave, | |
325 | sizeof(struct i387_fxsave_struct) ) ) | |
326 | return -1; | |
327 | return 1; | |
328 | } | |
329 | ||
330 | int save_i387( struct _fpstate __user *buf ) | |
331 | { | |
332 | if ( !used_math() ) | |
333 | return 0; | |
334 | ||
335 | /* This will cause a "finit" to be triggered by the next | |
336 | * attempted FPU operation by the 'current' process. | |
337 | */ | |
338 | clear_used_math(); | |
339 | ||
340 | if ( HAVE_HWFP ) { | |
341 | if ( cpu_has_fxsr ) { | |
342 | return save_i387_fxsave( buf ); | |
343 | } else { | |
344 | return save_i387_fsave( buf ); | |
345 | } | |
346 | } else { | |
347 | return save_i387_soft( ¤t->thread.i387.soft, buf ); | |
348 | } | |
349 | } | |
350 | ||
351 | static inline int restore_i387_fsave( struct _fpstate __user *buf ) | |
352 | { | |
353 | struct task_struct *tsk = current; | |
354 | clear_fpu( tsk ); | |
355 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
356 | sizeof(struct i387_fsave_struct) ); | |
357 | } | |
358 | ||
359 | static int restore_i387_fxsave( struct _fpstate __user *buf ) | |
360 | { | |
361 | int err; | |
362 | struct task_struct *tsk = current; | |
363 | clear_fpu( tsk ); | |
364 | err = __copy_from_user( &tsk->thread.i387.fxsave, &buf->_fxsr_env[0], | |
365 | sizeof(struct i387_fxsave_struct) ); | |
366 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
367 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
368 | return err ? 1 : convert_fxsr_from_user( &tsk->thread.i387.fxsave, buf ); | |
369 | } | |
370 | ||
371 | int restore_i387( struct _fpstate __user *buf ) | |
372 | { | |
373 | int err; | |
374 | ||
375 | if ( HAVE_HWFP ) { | |
376 | if ( cpu_has_fxsr ) { | |
377 | err = restore_i387_fxsave( buf ); | |
378 | } else { | |
379 | err = restore_i387_fsave( buf ); | |
380 | } | |
381 | } else { | |
382 | err = restore_i387_soft( ¤t->thread.i387.soft, buf ); | |
383 | } | |
384 | set_used_math(); | |
385 | return err; | |
386 | } | |
387 | ||
388 | /* | |
389 | * ptrace request handlers. | |
390 | */ | |
391 | ||
392 | static inline int get_fpregs_fsave( struct user_i387_struct __user *buf, | |
393 | struct task_struct *tsk ) | |
394 | { | |
395 | return __copy_to_user( buf, &tsk->thread.i387.fsave, | |
396 | sizeof(struct user_i387_struct) ); | |
397 | } | |
398 | ||
399 | static inline int get_fpregs_fxsave( struct user_i387_struct __user *buf, | |
400 | struct task_struct *tsk ) | |
401 | { | |
402 | return convert_fxsr_to_user( (struct _fpstate __user *)buf, | |
403 | &tsk->thread.i387.fxsave ); | |
404 | } | |
405 | ||
406 | int get_fpregs( struct user_i387_struct __user *buf, struct task_struct *tsk ) | |
407 | { | |
408 | if ( HAVE_HWFP ) { | |
409 | if ( cpu_has_fxsr ) { | |
410 | return get_fpregs_fxsave( buf, tsk ); | |
411 | } else { | |
412 | return get_fpregs_fsave( buf, tsk ); | |
413 | } | |
414 | } else { | |
415 | return save_i387_soft( &tsk->thread.i387.soft, | |
416 | (struct _fpstate __user *)buf ); | |
417 | } | |
418 | } | |
419 | ||
420 | static inline int set_fpregs_fsave( struct task_struct *tsk, | |
421 | struct user_i387_struct __user *buf ) | |
422 | { | |
423 | return __copy_from_user( &tsk->thread.i387.fsave, buf, | |
424 | sizeof(struct user_i387_struct) ); | |
425 | } | |
426 | ||
427 | static inline int set_fpregs_fxsave( struct task_struct *tsk, | |
428 | struct user_i387_struct __user *buf ) | |
429 | { | |
430 | return convert_fxsr_from_user( &tsk->thread.i387.fxsave, | |
431 | (struct _fpstate __user *)buf ); | |
432 | } | |
433 | ||
434 | int set_fpregs( struct task_struct *tsk, struct user_i387_struct __user *buf ) | |
435 | { | |
436 | if ( HAVE_HWFP ) { | |
437 | if ( cpu_has_fxsr ) { | |
438 | return set_fpregs_fxsave( tsk, buf ); | |
439 | } else { | |
440 | return set_fpregs_fsave( tsk, buf ); | |
441 | } | |
442 | } else { | |
443 | return restore_i387_soft( &tsk->thread.i387.soft, | |
444 | (struct _fpstate __user *)buf ); | |
445 | } | |
446 | } | |
447 | ||
448 | int get_fpxregs( struct user_fxsr_struct __user *buf, struct task_struct *tsk ) | |
449 | { | |
450 | if ( cpu_has_fxsr ) { | |
451 | if (__copy_to_user( buf, &tsk->thread.i387.fxsave, | |
452 | sizeof(struct user_fxsr_struct) )) | |
453 | return -EFAULT; | |
454 | return 0; | |
455 | } else { | |
456 | return -EIO; | |
457 | } | |
458 | } | |
459 | ||
460 | int set_fpxregs( struct task_struct *tsk, struct user_fxsr_struct __user *buf ) | |
461 | { | |
462 | int ret = 0; | |
463 | ||
464 | if ( cpu_has_fxsr ) { | |
465 | if (__copy_from_user( &tsk->thread.i387.fxsave, buf, | |
466 | sizeof(struct user_fxsr_struct) )) | |
467 | ret = -EFAULT; | |
468 | /* mxcsr reserved bits must be masked to zero for security reasons */ | |
469 | tsk->thread.i387.fxsave.mxcsr &= mxcsr_feature_mask; | |
470 | } else { | |
471 | ret = -EIO; | |
472 | } | |
473 | return ret; | |
474 | } | |
475 | ||
476 | /* | |
477 | * FPU state for core dumps. | |
478 | */ | |
479 | ||
480 | static inline void copy_fpu_fsave( struct task_struct *tsk, | |
481 | struct user_i387_struct *fpu ) | |
482 | { | |
483 | memcpy( fpu, &tsk->thread.i387.fsave, | |
484 | sizeof(struct user_i387_struct) ); | |
485 | } | |
486 | ||
487 | static inline void copy_fpu_fxsave( struct task_struct *tsk, | |
488 | struct user_i387_struct *fpu ) | |
489 | { | |
490 | unsigned short *to; | |
491 | unsigned short *from; | |
492 | int i; | |
493 | ||
494 | memcpy( fpu, &tsk->thread.i387.fxsave, 7 * sizeof(long) ); | |
495 | ||
496 | to = (unsigned short *)&fpu->st_space[0]; | |
497 | from = (unsigned short *)&tsk->thread.i387.fxsave.st_space[0]; | |
498 | for ( i = 0 ; i < 8 ; i++, to += 5, from += 8 ) { | |
499 | memcpy( to, from, 5 * sizeof(unsigned short) ); | |
500 | } | |
501 | } | |
502 | ||
503 | int dump_fpu( struct pt_regs *regs, struct user_i387_struct *fpu ) | |
504 | { | |
505 | int fpvalid; | |
506 | struct task_struct *tsk = current; | |
507 | ||
508 | fpvalid = !!used_math(); | |
509 | if ( fpvalid ) { | |
510 | unlazy_fpu( tsk ); | |
511 | if ( cpu_has_fxsr ) { | |
512 | copy_fpu_fxsave( tsk, fpu ); | |
513 | } else { | |
514 | copy_fpu_fsave( tsk, fpu ); | |
515 | } | |
516 | } | |
517 | ||
518 | return fpvalid; | |
519 | } | |
129f6946 | 520 | EXPORT_SYMBOL(dump_fpu); |
1da177e4 LT |
521 | |
522 | int dump_task_fpu(struct task_struct *tsk, struct user_i387_struct *fpu) | |
523 | { | |
524 | int fpvalid = !!tsk_used_math(tsk); | |
525 | ||
526 | if (fpvalid) { | |
527 | if (tsk == current) | |
528 | unlazy_fpu(tsk); | |
529 | if (cpu_has_fxsr) | |
530 | copy_fpu_fxsave(tsk, fpu); | |
531 | else | |
532 | copy_fpu_fsave(tsk, fpu); | |
533 | } | |
534 | return fpvalid; | |
535 | } | |
536 | ||
537 | int dump_task_extended_fpu(struct task_struct *tsk, struct user_fxsr_struct *fpu) | |
538 | { | |
539 | int fpvalid = tsk_used_math(tsk) && cpu_has_fxsr; | |
540 | ||
541 | if (fpvalid) { | |
542 | if (tsk == current) | |
543 | unlazy_fpu(tsk); | |
544 | memcpy(fpu, &tsk->thread.i387.fxsave, sizeof(*fpu)); | |
545 | } | |
546 | return fpvalid; | |
547 | } |