]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __ASM_SYSTEM_H |
2 | #define __ASM_SYSTEM_H | |
3 | ||
4 | #include <linux/config.h> | |
5 | #include <linux/kernel.h> | |
6 | #include <asm/segment.h> | |
7 | #include <asm/cpufeature.h> | |
8 | #include <linux/bitops.h> /* for LOCK_PREFIX */ | |
9 | ||
10 | #ifdef __KERNEL__ | |
11 | ||
12 | struct task_struct; /* one of the stranger aspects of C forward declarations.. */ | |
13 | extern struct task_struct * FASTCALL(__switch_to(struct task_struct *prev, struct task_struct *next)); | |
14 | ||
15 | #define switch_to(prev,next,last) do { \ | |
16 | unsigned long esi,edi; \ | |
17 | asm volatile("pushfl\n\t" \ | |
18 | "pushl %%ebp\n\t" \ | |
19 | "movl %%esp,%0\n\t" /* save ESP */ \ | |
20 | "movl %5,%%esp\n\t" /* restore ESP */ \ | |
21 | "movl $1f,%1\n\t" /* save EIP */ \ | |
22 | "pushl %6\n\t" /* restore EIP */ \ | |
23 | "jmp __switch_to\n" \ | |
24 | "1:\t" \ | |
25 | "popl %%ebp\n\t" \ | |
26 | "popfl" \ | |
27 | :"=m" (prev->thread.esp),"=m" (prev->thread.eip), \ | |
28 | "=a" (last),"=S" (esi),"=D" (edi) \ | |
29 | :"m" (next->thread.esp),"m" (next->thread.eip), \ | |
30 | "2" (prev), "d" (next)); \ | |
31 | } while (0) | |
32 | ||
33 | #define _set_base(addr,base) do { unsigned long __pr; \ | |
34 | __asm__ __volatile__ ("movw %%dx,%1\n\t" \ | |
35 | "rorl $16,%%edx\n\t" \ | |
36 | "movb %%dl,%2\n\t" \ | |
37 | "movb %%dh,%3" \ | |
38 | :"=&d" (__pr) \ | |
39 | :"m" (*((addr)+2)), \ | |
40 | "m" (*((addr)+4)), \ | |
41 | "m" (*((addr)+7)), \ | |
42 | "0" (base) \ | |
43 | ); } while(0) | |
44 | ||
45 | #define _set_limit(addr,limit) do { unsigned long __lr; \ | |
46 | __asm__ __volatile__ ("movw %%dx,%1\n\t" \ | |
47 | "rorl $16,%%edx\n\t" \ | |
48 | "movb %2,%%dh\n\t" \ | |
49 | "andb $0xf0,%%dh\n\t" \ | |
50 | "orb %%dh,%%dl\n\t" \ | |
51 | "movb %%dl,%2" \ | |
52 | :"=&d" (__lr) \ | |
53 | :"m" (*(addr)), \ | |
54 | "m" (*((addr)+6)), \ | |
55 | "0" (limit) \ | |
56 | ); } while(0) | |
57 | ||
58 | #define set_base(ldt,base) _set_base( ((char *)&(ldt)) , (base) ) | |
59 | #define set_limit(ldt,limit) _set_limit( ((char *)&(ldt)) , ((limit)-1)>>12 ) | |
60 | ||
61 | static inline unsigned long _get_base(char * addr) | |
62 | { | |
63 | unsigned long __base; | |
64 | __asm__("movb %3,%%dh\n\t" | |
65 | "movb %2,%%dl\n\t" | |
66 | "shll $16,%%edx\n\t" | |
67 | "movw %1,%%dx" | |
68 | :"=&d" (__base) | |
69 | :"m" (*((addr)+2)), | |
70 | "m" (*((addr)+4)), | |
71 | "m" (*((addr)+7))); | |
72 | return __base; | |
73 | } | |
74 | ||
75 | #define get_base(ldt) _get_base( ((char *)&(ldt)) ) | |
76 | ||
77 | /* | |
78 | * Load a segment. Fall back on loading the zero | |
79 | * segment if something goes wrong.. | |
80 | */ | |
81 | #define loadsegment(seg,value) \ | |
82 | asm volatile("\n" \ | |
83 | "1:\t" \ | |
fd51f666 | 84 | "mov %0,%%" #seg "\n" \ |
1da177e4 LT |
85 | "2:\n" \ |
86 | ".section .fixup,\"ax\"\n" \ | |
87 | "3:\t" \ | |
88 | "pushl $0\n\t" \ | |
89 | "popl %%" #seg "\n\t" \ | |
90 | "jmp 2b\n" \ | |
91 | ".previous\n" \ | |
92 | ".section __ex_table,\"a\"\n\t" \ | |
93 | ".align 4\n\t" \ | |
94 | ".long 1b,3b\n" \ | |
95 | ".previous" \ | |
4d37e7e3 | 96 | : :"rm" (value)) |
1da177e4 LT |
97 | |
98 | /* | |
99 | * Save a segment register away | |
100 | */ | |
101 | #define savesegment(seg, value) \ | |
4d37e7e3 | 102 | asm volatile("mov %%" #seg ",%0":"=rm" (value)) |
1da177e4 LT |
103 | |
104 | /* | |
105 | * Clear and set 'TS' bit respectively | |
106 | */ | |
107 | #define clts() __asm__ __volatile__ ("clts") | |
108 | #define read_cr0() ({ \ | |
109 | unsigned int __dummy; \ | |
4bb0d3ec | 110 | __asm__ __volatile__( \ |
1da177e4 LT |
111 | "movl %%cr0,%0\n\t" \ |
112 | :"=r" (__dummy)); \ | |
113 | __dummy; \ | |
114 | }) | |
115 | #define write_cr0(x) \ | |
4bb0d3ec ZA |
116 | __asm__ __volatile__("movl %0,%%cr0": :"r" (x)); |
117 | ||
118 | #define read_cr2() ({ \ | |
119 | unsigned int __dummy; \ | |
120 | __asm__ __volatile__( \ | |
121 | "movl %%cr2,%0\n\t" \ | |
122 | :"=r" (__dummy)); \ | |
123 | __dummy; \ | |
124 | }) | |
125 | #define write_cr2(x) \ | |
126 | __asm__ __volatile__("movl %0,%%cr2": :"r" (x)); | |
127 | ||
128 | #define read_cr3() ({ \ | |
129 | unsigned int __dummy; \ | |
130 | __asm__ ( \ | |
131 | "movl %%cr3,%0\n\t" \ | |
132 | :"=r" (__dummy)); \ | |
133 | __dummy; \ | |
134 | }) | |
135 | #define write_cr3(x) \ | |
136 | __asm__ __volatile__("movl %0,%%cr3": :"r" (x)); | |
1da177e4 LT |
137 | |
138 | #define read_cr4() ({ \ | |
139 | unsigned int __dummy; \ | |
140 | __asm__( \ | |
141 | "movl %%cr4,%0\n\t" \ | |
142 | :"=r" (__dummy)); \ | |
143 | __dummy; \ | |
144 | }) | |
145 | #define write_cr4(x) \ | |
4bb0d3ec | 146 | __asm__ __volatile__("movl %0,%%cr4": :"r" (x)); |
1da177e4 LT |
147 | #define stts() write_cr0(8 | read_cr0()) |
148 | ||
149 | #endif /* __KERNEL__ */ | |
150 | ||
151 | #define wbinvd() \ | |
152 | __asm__ __volatile__ ("wbinvd": : :"memory"); | |
153 | ||
154 | static inline unsigned long get_limit(unsigned long segment) | |
155 | { | |
156 | unsigned long __limit; | |
157 | __asm__("lsll %1,%0" | |
158 | :"=r" (__limit):"r" (segment)); | |
159 | return __limit+1; | |
160 | } | |
161 | ||
162 | #define nop() __asm__ __volatile__ ("nop") | |
163 | ||
164 | #define xchg(ptr,v) ((__typeof__(*(ptr)))__xchg((unsigned long)(v),(ptr),sizeof(*(ptr)))) | |
165 | ||
166 | #define tas(ptr) (xchg((ptr),1)) | |
167 | ||
168 | struct __xchg_dummy { unsigned long a[100]; }; | |
169 | #define __xg(x) ((struct __xchg_dummy *)(x)) | |
170 | ||
171 | ||
172 | /* | |
173 | * The semantics of XCHGCMP8B are a bit strange, this is why | |
174 | * there is a loop and the loading of %%eax and %%edx has to | |
175 | * be inside. This inlines well in most cases, the cached | |
176 | * cost is around ~38 cycles. (in the future we might want | |
177 | * to do an SIMD/3DNOW!/MMX/FPU 64-bit store here, but that | |
178 | * might have an implicit FPU-save as a cost, so it's not | |
179 | * clear which path to go.) | |
180 | * | |
181 | * cmpxchg8b must be used with the lock prefix here to allow | |
182 | * the instruction to be executed atomically, see page 3-102 | |
183 | * of the instruction set reference 24319102.pdf. We need | |
184 | * the reader side to see the coherent 64bit value. | |
185 | */ | |
186 | static inline void __set_64bit (unsigned long long * ptr, | |
187 | unsigned int low, unsigned int high) | |
188 | { | |
189 | __asm__ __volatile__ ( | |
190 | "\n1:\t" | |
191 | "movl (%0), %%eax\n\t" | |
192 | "movl 4(%0), %%edx\n\t" | |
193 | "lock cmpxchg8b (%0)\n\t" | |
194 | "jnz 1b" | |
195 | : /* no outputs */ | |
196 | : "D"(ptr), | |
197 | "b"(low), | |
198 | "c"(high) | |
199 | : "ax","dx","memory"); | |
200 | } | |
201 | ||
202 | static inline void __set_64bit_constant (unsigned long long *ptr, | |
203 | unsigned long long value) | |
204 | { | |
205 | __set_64bit(ptr,(unsigned int)(value), (unsigned int)((value)>>32ULL)); | |
206 | } | |
207 | #define ll_low(x) *(((unsigned int*)&(x))+0) | |
208 | #define ll_high(x) *(((unsigned int*)&(x))+1) | |
209 | ||
210 | static inline void __set_64bit_var (unsigned long long *ptr, | |
211 | unsigned long long value) | |
212 | { | |
213 | __set_64bit(ptr,ll_low(value), ll_high(value)); | |
214 | } | |
215 | ||
216 | #define set_64bit(ptr,value) \ | |
217 | (__builtin_constant_p(value) ? \ | |
218 | __set_64bit_constant(ptr, value) : \ | |
219 | __set_64bit_var(ptr, value) ) | |
220 | ||
221 | #define _set_64bit(ptr,value) \ | |
222 | (__builtin_constant_p(value) ? \ | |
223 | __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \ | |
224 | __set_64bit(ptr, ll_low(value), ll_high(value)) ) | |
225 | ||
226 | /* | |
227 | * Note: no "lock" prefix even on SMP: xchg always implies lock anyway | |
228 | * Note 2: xchg has side effect, so that attribute volatile is necessary, | |
229 | * but generally the primitive is invalid, *ptr is output argument. --ANK | |
230 | */ | |
231 | static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) | |
232 | { | |
233 | switch (size) { | |
234 | case 1: | |
235 | __asm__ __volatile__("xchgb %b0,%1" | |
236 | :"=q" (x) | |
237 | :"m" (*__xg(ptr)), "0" (x) | |
238 | :"memory"); | |
239 | break; | |
240 | case 2: | |
241 | __asm__ __volatile__("xchgw %w0,%1" | |
242 | :"=r" (x) | |
243 | :"m" (*__xg(ptr)), "0" (x) | |
244 | :"memory"); | |
245 | break; | |
246 | case 4: | |
247 | __asm__ __volatile__("xchgl %0,%1" | |
248 | :"=r" (x) | |
249 | :"m" (*__xg(ptr)), "0" (x) | |
250 | :"memory"); | |
251 | break; | |
252 | } | |
253 | return x; | |
254 | } | |
255 | ||
256 | /* | |
257 | * Atomic compare and exchange. Compare OLD with MEM, if identical, | |
258 | * store NEW in MEM. Return the initial value in MEM. Success is | |
259 | * indicated by comparing RETURN with OLD. | |
260 | */ | |
261 | ||
262 | #ifdef CONFIG_X86_CMPXCHG | |
263 | #define __HAVE_ARCH_CMPXCHG 1 | |
264 | #endif | |
265 | ||
266 | static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, | |
267 | unsigned long new, int size) | |
268 | { | |
269 | unsigned long prev; | |
270 | switch (size) { | |
271 | case 1: | |
272 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" | |
273 | : "=a"(prev) | |
274 | : "q"(new), "m"(*__xg(ptr)), "0"(old) | |
275 | : "memory"); | |
276 | return prev; | |
277 | case 2: | |
278 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" | |
279 | : "=a"(prev) | |
280 | : "q"(new), "m"(*__xg(ptr)), "0"(old) | |
281 | : "memory"); | |
282 | return prev; | |
283 | case 4: | |
284 | __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" | |
285 | : "=a"(prev) | |
286 | : "q"(new), "m"(*__xg(ptr)), "0"(old) | |
287 | : "memory"); | |
288 | return prev; | |
289 | } | |
290 | return old; | |
291 | } | |
292 | ||
293 | #define cmpxchg(ptr,o,n)\ | |
294 | ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ | |
295 | (unsigned long)(n),sizeof(*(ptr)))) | |
296 | ||
297 | #ifdef __KERNEL__ | |
298 | struct alt_instr { | |
299 | __u8 *instr; /* original instruction */ | |
300 | __u8 *replacement; | |
301 | __u8 cpuid; /* cpuid bit set for replacement */ | |
302 | __u8 instrlen; /* length of original instruction */ | |
303 | __u8 replacementlen; /* length of new instruction, <= instrlen */ | |
304 | __u8 pad; | |
305 | }; | |
306 | #endif | |
307 | ||
308 | /* | |
309 | * Alternative instructions for different CPU types or capabilities. | |
310 | * | |
311 | * This allows to use optimized instructions even on generic binary | |
312 | * kernels. | |
313 | * | |
314 | * length of oldinstr must be longer or equal the length of newinstr | |
315 | * It can be padded with nops as needed. | |
316 | * | |
317 | * For non barrier like inlines please define new variants | |
318 | * without volatile and memory clobber. | |
319 | */ | |
320 | #define alternative(oldinstr, newinstr, feature) \ | |
321 | asm volatile ("661:\n\t" oldinstr "\n662:\n" \ | |
322 | ".section .altinstructions,\"a\"\n" \ | |
323 | " .align 4\n" \ | |
324 | " .long 661b\n" /* label */ \ | |
325 | " .long 663f\n" /* new instruction */ \ | |
326 | " .byte %c0\n" /* feature bit */ \ | |
327 | " .byte 662b-661b\n" /* sourcelen */ \ | |
328 | " .byte 664f-663f\n" /* replacementlen */ \ | |
329 | ".previous\n" \ | |
330 | ".section .altinstr_replacement,\"ax\"\n" \ | |
331 | "663:\n\t" newinstr "\n664:\n" /* replacement */ \ | |
332 | ".previous" :: "i" (feature) : "memory") | |
333 | ||
334 | /* | |
335 | * Alternative inline assembly with input. | |
336 | * | |
337 | * Pecularities: | |
338 | * No memory clobber here. | |
339 | * Argument numbers start with 1. | |
340 | * Best is to use constraints that are fixed size (like (%1) ... "r") | |
341 | * If you use variable sized constraints like "m" or "g" in the | |
342 | * replacement maake sure to pad to the worst case length. | |
343 | */ | |
344 | #define alternative_input(oldinstr, newinstr, feature, input...) \ | |
345 | asm volatile ("661:\n\t" oldinstr "\n662:\n" \ | |
346 | ".section .altinstructions,\"a\"\n" \ | |
347 | " .align 4\n" \ | |
348 | " .long 661b\n" /* label */ \ | |
349 | " .long 663f\n" /* new instruction */ \ | |
350 | " .byte %c0\n" /* feature bit */ \ | |
351 | " .byte 662b-661b\n" /* sourcelen */ \ | |
352 | " .byte 664f-663f\n" /* replacementlen */ \ | |
353 | ".previous\n" \ | |
354 | ".section .altinstr_replacement,\"ax\"\n" \ | |
355 | "663:\n\t" newinstr "\n664:\n" /* replacement */ \ | |
356 | ".previous" :: "i" (feature), ##input) | |
357 | ||
358 | /* | |
359 | * Force strict CPU ordering. | |
360 | * And yes, this is required on UP too when we're talking | |
361 | * to devices. | |
362 | * | |
363 | * For now, "wmb()" doesn't actually do anything, as all | |
364 | * Intel CPU's follow what Intel calls a *Processor Order*, | |
365 | * in which all writes are seen in the program order even | |
366 | * outside the CPU. | |
367 | * | |
368 | * I expect future Intel CPU's to have a weaker ordering, | |
369 | * but I'd also expect them to finally get their act together | |
370 | * and add some real memory barriers if so. | |
371 | * | |
372 | * Some non intel clones support out of order store. wmb() ceases to be a | |
373 | * nop for these. | |
374 | */ | |
375 | ||
376 | ||
377 | /* | |
378 | * Actually only lfence would be needed for mb() because all stores done | |
379 | * by the kernel should be already ordered. But keep a full barrier for now. | |
380 | */ | |
381 | ||
382 | #define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2) | |
383 | #define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2) | |
384 | ||
385 | /** | |
386 | * read_barrier_depends - Flush all pending reads that subsequents reads | |
387 | * depend on. | |
388 | * | |
389 | * No data-dependent reads from memory-like regions are ever reordered | |
390 | * over this barrier. All reads preceding this primitive are guaranteed | |
391 | * to access memory (but not necessarily other CPUs' caches) before any | |
392 | * reads following this primitive that depend on the data return by | |
393 | * any of the preceding reads. This primitive is much lighter weight than | |
394 | * rmb() on most CPUs, and is never heavier weight than is | |
395 | * rmb(). | |
396 | * | |
397 | * These ordering constraints are respected by both the local CPU | |
398 | * and the compiler. | |
399 | * | |
400 | * Ordering is not guaranteed by anything other than these primitives, | |
401 | * not even by data dependencies. See the documentation for | |
402 | * memory_barrier() for examples and URLs to more information. | |
403 | * | |
404 | * For example, the following code would force ordering (the initial | |
405 | * value of "a" is zero, "b" is one, and "p" is "&a"): | |
406 | * | |
407 | * <programlisting> | |
408 | * CPU 0 CPU 1 | |
409 | * | |
410 | * b = 2; | |
411 | * memory_barrier(); | |
412 | * p = &b; q = p; | |
413 | * read_barrier_depends(); | |
414 | * d = *q; | |
415 | * </programlisting> | |
416 | * | |
417 | * because the read of "*q" depends on the read of "p" and these | |
418 | * two reads are separated by a read_barrier_depends(). However, | |
419 | * the following code, with the same initial values for "a" and "b": | |
420 | * | |
421 | * <programlisting> | |
422 | * CPU 0 CPU 1 | |
423 | * | |
424 | * a = 2; | |
425 | * memory_barrier(); | |
426 | * b = 3; y = b; | |
427 | * read_barrier_depends(); | |
428 | * x = a; | |
429 | * </programlisting> | |
430 | * | |
431 | * does not enforce ordering, since there is no data dependency between | |
432 | * the read of "a" and the read of "b". Therefore, on some CPUs, such | |
433 | * as Alpha, "y" could be set to 3 and "x" to 0. Use rmb() | |
434 | * in cases like thiswhere there are no data dependencies. | |
435 | **/ | |
436 | ||
437 | #define read_barrier_depends() do { } while(0) | |
438 | ||
439 | #ifdef CONFIG_X86_OOSTORE | |
440 | /* Actually there are no OOO store capable CPUs for now that do SSE, | |
441 | but make it already an possibility. */ | |
442 | #define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM) | |
443 | #else | |
444 | #define wmb() __asm__ __volatile__ ("": : :"memory") | |
445 | #endif | |
446 | ||
447 | #ifdef CONFIG_SMP | |
448 | #define smp_mb() mb() | |
449 | #define smp_rmb() rmb() | |
450 | #define smp_wmb() wmb() | |
451 | #define smp_read_barrier_depends() read_barrier_depends() | |
452 | #define set_mb(var, value) do { xchg(&var, value); } while (0) | |
453 | #else | |
454 | #define smp_mb() barrier() | |
455 | #define smp_rmb() barrier() | |
456 | #define smp_wmb() barrier() | |
457 | #define smp_read_barrier_depends() do { } while(0) | |
458 | #define set_mb(var, value) do { var = value; barrier(); } while (0) | |
459 | #endif | |
460 | ||
461 | #define set_wmb(var, value) do { var = value; wmb(); } while (0) | |
462 | ||
463 | /* interrupt control.. */ | |
464 | #define local_save_flags(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushfl ; popl %0":"=g" (x): /* no input */); } while (0) | |
465 | #define local_irq_restore(x) do { typecheck(unsigned long,x); __asm__ __volatile__("pushl %0 ; popfl": /* no output */ :"g" (x):"memory", "cc"); } while (0) | |
466 | #define local_irq_disable() __asm__ __volatile__("cli": : :"memory") | |
467 | #define local_irq_enable() __asm__ __volatile__("sti": : :"memory") | |
468 | /* used in the idle loop; sti takes one instruction cycle to complete */ | |
469 | #define safe_halt() __asm__ __volatile__("sti; hlt": : :"memory") | |
4bb0d3ec ZA |
470 | /* used when interrupts are already enabled or to shutdown the processor */ |
471 | #define halt() __asm__ __volatile__("hlt": : :"memory") | |
1da177e4 LT |
472 | |
473 | #define irqs_disabled() \ | |
474 | ({ \ | |
475 | unsigned long flags; \ | |
476 | local_save_flags(flags); \ | |
477 | !(flags & (1<<9)); \ | |
478 | }) | |
479 | ||
480 | /* For spinlocks etc */ | |
481 | #define local_irq_save(x) __asm__ __volatile__("pushfl ; popl %0 ; cli":"=g" (x): /* no input */ :"memory") | |
482 | ||
483 | /* | |
484 | * disable hlt during certain critical i/o operations | |
485 | */ | |
486 | #define HAVE_DISABLE_HLT | |
487 | void disable_hlt(void); | |
488 | void enable_hlt(void); | |
489 | ||
490 | extern int es7000_plat; | |
491 | void cpu_idle_wait(void); | |
492 | ||
493 | extern unsigned long arch_align_stack(unsigned long sp); | |
494 | ||
495 | #endif |