[mirror_ubuntu-artful-kernel.git] / arch / x86 / include / asm / string_32.h

#ifndef _ASM_X86_STRING_32_H
#define _ASM_X86_STRING_32_H

#ifdef __KERNEL__

/* Let gcc decide whether to inline or use the out of line functions */

#define __HAVE_ARCH_STRCPY
extern char *strcpy(char *dest, const char *src);

#define __HAVE_ARCH_STRNCPY
extern char *strncpy(char *dest, const char *src, size_t count);

#define __HAVE_ARCH_STRCAT
extern char *strcat(char *dest, const char *src);

#define __HAVE_ARCH_STRNCAT
extern char *strncat(char *dest, const char *src, size_t count);

#define __HAVE_ARCH_STRCMP
extern int strcmp(const char *cs, const char *ct);

#define __HAVE_ARCH_STRNCMP
extern int strncmp(const char *cs, const char *ct, size_t count);

#define __HAVE_ARCH_STRCHR
extern char *strchr(const char *s, int c);

#define __HAVE_ARCH_STRLEN
extern size_t strlen(const char *s);

static __always_inline void *__memcpy(void *to, const void *from, size_t n)
{
	int d0, d1, d2;
	asm volatile("rep ; movsl\n\t"
		     "movl %4,%%ecx\n\t"
		     "andl $3,%%ecx\n\t"
		     "jz 1f\n\t"
		     "rep ; movsb\n\t"
		     "1:"
		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
		     : "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from)
		     : "memory");
	return to;
}

/*
 * This looks ugly, but the compiler can optimize it totally,
 * as the count is constant.
 */
static __always_inline void *__constant_memcpy(void *to, const void *from,
					       size_t n)
{
	long esi, edi;
	if (!n)
		return to;

	switch (n) {
	case 1:
		*(char *)to = *(char *)from;
		return to;
	case 2:
		*(short *)to = *(short *)from;
		return to;
	case 4:
		*(int *)to = *(int *)from;
		return to;

	case 3:
		*(short *)to = *(short *)from;
		*((char *)to + 2) = *((char *)from + 2);
		return to;
	case 5:
		*(int *)to = *(int *)from;
		*((char *)to + 4) = *((char *)from + 4);
		return to;
	case 6:
		*(int *)to = *(int *)from;
		*((short *)to + 2) = *((short *)from + 2);
		return to;
	case 8:
		*(int *)to = *(int *)from;
		*((int *)to + 1) = *((int *)from + 1);
		return to;
	}

	esi = (long)from;
	edi = (long)to;
	if (n >= 5 * 4) {
		/* large block: use rep prefix */
		int ecx;
		asm volatile("rep ; movsl"
			     : "=&c" (ecx), "=&D" (edi), "=&S" (esi)
			     : "0" (n / 4), "1" (edi), "2" (esi)
			     : "memory"
		);
	} else {
		/* small block: don't clobber ecx + smaller code */
		if (n >= 4 * 4)
			asm volatile("movsl"
				     : "=&D"(edi), "=&S"(esi)
				     : "0"(edi), "1"(esi)
				     : "memory");
		if (n >= 3 * 4)
			asm volatile("movsl"
				     : "=&D"(edi), "=&S"(esi)
				     : "0"(edi), "1"(esi)
				     : "memory");
		if (n >= 2 * 4)
			asm volatile("movsl"
				     : "=&D"(edi), "=&S"(esi)
				     : "0"(edi), "1"(esi)
				     : "memory");
		if (n >= 1 * 4)
			asm volatile("movsl"
				     : "=&D"(edi), "=&S"(esi)
				     : "0"(edi), "1"(esi)
				     : "memory");
	}
	switch (n % 4) {
		/* tail */
	case 0:
		return to;
	case 1:
		asm volatile("movsb"
			     : "=&D"(edi), "=&S"(esi)
			     : "0"(edi), "1"(esi)
			     : "memory");
		return to;
	case 2:
		asm volatile("movsw"
			     : "=&D"(edi), "=&S"(esi)
			     : "0"(edi), "1"(esi)
			     : "memory");
		return to;
	default:
		asm volatile("movsw\n\tmovsb"
			     : "=&D"(edi), "=&S"(esi)
			     : "0"(edi), "1"(esi)
			     : "memory");
		return to;
	}
}

#define __HAVE_ARCH_MEMCPY

#ifdef CONFIG_X86_USE_3DNOW

#include <asm/mmx.h>

/*
 *	This CPU favours 3DNow strongly (eg AMD Athlon)
 */

static inline void *__constant_memcpy3d(void *to, const void *from, size_t len)
{
	if (len < 512)
		return __constant_memcpy(to, from, len);
	return _mmx_memcpy(to, from, len);
}

static inline void *__memcpy3d(void *to, const void *from, size_t len)
{
	if (len < 512)
		return __memcpy(to, from, len);
	return _mmx_memcpy(to, from, len);
}

#define memcpy(t, f, n)				\
	(__builtin_constant_p((n))		\
	 ? __constant_memcpy3d((t), (f), (n))	\
	 : __memcpy3d((t), (f), (n)))

#else

/*
 *	No 3D Now!
 */

#define memcpy(t, f, n)				\
	(__builtin_constant_p((n))		\
	 ? __constant_memcpy((t), (f), (n))	\
	 : __memcpy((t), (f), (n)))

#endif

#define __HAVE_ARCH_MEMMOVE
void *memmove(void *dest, const void *src, size_t n);

#define memcmp __builtin_memcmp

#define __HAVE_ARCH_MEMCHR
extern void *memchr(const void *cs, int c, size_t count);

static inline void *__memset_generic(void *s, char c, size_t count)
{
	int d0, d1;
	asm volatile("rep\n\t"
		     "stosb"
		     : "=&c" (d0), "=&D" (d1)
		     : "a" (c), "1" (s), "0" (count)
		     : "memory");
	return s;
}

/* we might want to write optimized versions of these later */
#define __constant_count_memset(s, c, count) __memset_generic((s), (c), (count))

/*
 * memset(x, 0, y) is a reasonably common thing to do, so we want to fill
 * things 32 bits at a time even when we don't know the size of the
 * area at compile-time..
 */
static __always_inline
void *__constant_c_memset(void *s, unsigned long c, size_t count)
{
	int d0, d1;
	asm volatile("rep ; stosl\n\t"
		     "testb $2,%b3\n\t"
		     "je 1f\n\t"
		     "stosw\n"
		     "1:\ttestb $1,%b3\n\t"
		     "je 2f\n\t"
		     "stosb\n"
		     "2:"
		     : "=&c" (d0), "=&D" (d1)
		     : "a" (c), "q" (count), "0" (count/4), "1" ((long)s)
		     : "memory");
	return s;
}

/* Added by Gertjan van Wingerde to make minix and sysv module work */
#define __HAVE_ARCH_STRNLEN
extern size_t strnlen(const char *s, size_t count);
/* end of additional stuff */

#define __HAVE_ARCH_STRSTR
extern char *strstr(const char *cs, const char *ct);

/*
 * This looks horribly ugly, but the compiler can optimize it totally,
 * as we by now know that both pattern and count is constant..
 */
static __always_inline
void *__constant_c_and_count_memset(void *s, unsigned long pattern,
				    size_t count)
{
	switch (count) {
	case 0:
		return s;
	case 1:
		*(unsigned char *)s = pattern & 0xff;
		return s;
	case 2:
		*(unsigned short *)s = pattern & 0xffff;
		return s;
	case 3:
		*(unsigned short *)s = pattern & 0xffff;
		*((unsigned char *)s + 2) = pattern & 0xff;
		return s;
	case 4:
		*(unsigned long *)s = pattern;
		return s;
	}

#define COMMON(x)							\
	asm volatile("rep ; stosl"					\
		     x							\
		     : "=&c" (d0), "=&D" (d1)				\
		     : "a" (eax), "0" (count/4), "1" ((long)s)	\
		     : "memory")

	{
		int d0, d1;
#if __GNUC__ == 4 && __GNUC_MINOR__ == 0
		/* Workaround for broken gcc 4.0 */
		register unsigned long eax asm("%eax") = pattern;
#else
		unsigned long eax = pattern;
#endif

		switch (count % 4) {
		case 0:
			COMMON("");
			return s;
		case 1:
			COMMON("\n\tstosb");
			return s;
		case 2:
			COMMON("\n\tstosw");
			return s;
		default:
			COMMON("\n\tstosw\n\tstosb");
			return s;
		}
	}

#undef COMMON
}

#define __constant_c_x_memset(s, c, count)			\
	(__builtin_constant_p(count)				\
	 ? __constant_c_and_count_memset((s), (c), (count))	\
	 : __constant_c_memset((s), (c), (count)))

#define __memset(s, c, count)				\
	(__builtin_constant_p(count)			\
	 ? __constant_count_memset((s), (c), (count))	\
	 : __memset_generic((s), (c), (count)))

#define __HAVE_ARCH_MEMSET
#define memset(s, c, count)						\
	(__builtin_constant_p(c)					\
	 ? __constant_c_x_memset((s), (0x01010101UL * (unsigned char)(c)), \
				 (count))				\
	 : __memset((s), (c), (count)))

/*
 * find the first occurrence of byte 'c', or 1 past the area if none
 */
#define __HAVE_ARCH_MEMSCAN
extern void *memscan(void *addr, int c, size_t size);

#endif /* __KERNEL__ */

#endif /* _ASM_X86_STRING_32_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_STRING_32_H
1965aae3 PA	2	#define _ASM_X86_STRING_32_H
1da177e4 LT	3
1da177e4 LT	4	#ifdef __KERNEL__
1da177e4	5
06b0f574	6	/* Let gcc decide whether to inline or use the out of line functions */
1da177e4 LT	7
1da177e4 LT	8	#define __HAVE_ARCH_STRCPY
b520b85a	9	extern char strcpy(char dest, const char *src);
1da177e4 LT	10
1da177e4 LT	11	#define __HAVE_ARCH_STRNCPY
b520b85a	12	extern char strncpy(char dest, const char *src, size_t count);
1da177e4 LT	13
1da177e4 LT	14	#define __HAVE_ARCH_STRCAT
b520b85a	15	extern char strcat(char dest, const char *src);
1da177e4 LT	16
1da177e4 LT	17	#define __HAVE_ARCH_STRNCAT
b520b85a	18	extern char strncat(char dest, const char *src, size_t count);
1da177e4 LT	19
1da177e4 LT	20	#define __HAVE_ARCH_STRCMP
b520b85a	21	extern int strcmp(const char cs, const char ct);
1da177e4 LT	22
1da177e4 LT	23	#define __HAVE_ARCH_STRNCMP
b520b85a	24	extern int strncmp(const char cs, const char ct, size_t count);
1da177e4 LT	25
1da177e4 LT	26	#define __HAVE_ARCH_STRCHR
b520b85a	27	extern char strchr(const char s, int c);
1da177e4	28
1da177e4	29	#define __HAVE_ARCH_STRLEN
b520b85a	30	extern size_t strlen(const char *s);
1da177e4	31
78d64fc2	32	static __always_inline void __memcpy(void to, const void *from, size_t n)
1da177e4	33	{
78d64fc2 JP	34	int d0, d1, d2;
	35	asm volatile("rep ; movsl\n\t"
	36	"movl %4,%%ecx\n\t"
	37	"andl $3,%%ecx\n\t"
	38	"jz 1f\n\t"
	39	"rep ; movsb\n\t"
	40	"1:"
	41	: "=&c" (d0), "=&D" (d1), "=&S" (d2)
	42	: "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from)
	43	: "memory");
	44	return to;
1da177e4 LT	45	}
	46
	47	/*
d5b63d78	48	* This looks ugly, but the compiler can optimize it totally,
1da177e4 LT	49	* as the count is constant.
1da177e4 LT	50	*/
78d64fc2 JP	51	static __always_inline void __constant_memcpy(void to, const void *from,
78d64fc2 JP	52	size_t n)
1da177e4	53	{
d5b63d78	54	long esi, edi;
78d64fc2 JP	55	if (!n)
	56	return to;
	57
d5b63d78	58	switch (n) {
78d64fc2 JP	59	case 1:
	60	(char )to = (char )from;
	61	return to;
	62	case 2:
	63	(short )to = (short )from;
	64	return to;
	65	case 4:
	66	(int )to = (int )from;
	67	return to;
	68
	69	case 3:
	70	(short )to = (short )from;
	71	((char )to + 2) = ((char )from + 2);
	72	return to;
	73	case 5:
	74	(int )to = (int )from;
	75	((char )to + 4) = ((char )from + 4);
	76	return to;
	77	case 6:
	78	(int )to = (int )from;
	79	((short )to + 2) = ((short )from + 2);
	80	return to;
	81	case 8:
	82	(int )to = (int )from;
	83	((int )to + 1) = ((int )from + 1);
	84	return to;
d5b63d78	85	}
78d64fc2 JP	86
	87	esi = (long)from;
	88	edi = (long)to;
	89	if (n >= 5 * 4) {
d5b63d78 DV	90	/* large block: use rep prefix */
d5b63d78 DV	91	int ecx;
78d64fc2 JP	92	asm volatile("rep ; movsl"
	93	: "=&c" (ecx), "=&D" (edi), "=&S" (esi)
	94	: "0" (n / 4), "1" (edi), "2" (esi)
	95	: "memory"
d5b63d78 DV	96	);
	97	} else {
	98	/* small block: don't clobber ecx + smaller code */
78d64fc2 JP	99	if (n >= 4 * 4)
	100	asm volatile("movsl"
	101	: "=&D"(edi), "=&S"(esi)
	102	: "0"(edi), "1"(esi)
	103	: "memory");
	104	if (n >= 3 * 4)
	105	asm volatile("movsl"
	106	: "=&D"(edi), "=&S"(esi)
	107	: "0"(edi), "1"(esi)
	108	: "memory");
	109	if (n >= 2 * 4)
	110	asm volatile("movsl"
	111	: "=&D"(edi), "=&S"(esi)
	112	: "0"(edi), "1"(esi)
	113	: "memory");
	114	if (n >= 1 * 4)
	115	asm volatile("movsl"
	116	: "=&D"(edi), "=&S"(esi)
	117	: "0"(edi), "1"(esi)
	118	: "memory");
d5b63d78	119	}
1da177e4	120	switch (n % 4) {
d5b63d78	121	/* tail */
78d64fc2 JP	122	case 0:
	123	return to;
	124	case 1:
	125	asm volatile("movsb"
	126	: "=&D"(edi), "=&S"(esi)
	127	: "0"(edi), "1"(esi)
	128	: "memory");
	129	return to;
	130	case 2:
	131	asm volatile("movsw"
	132	: "=&D"(edi), "=&S"(esi)
	133	: "0"(edi), "1"(esi)
	134	: "memory");
	135	return to;
	136	default:
	137	asm volatile("movsw\n\tmovsb"
	138	: "=&D"(edi), "=&S"(esi)
	139	: "0"(edi), "1"(esi)
	140	: "memory");
	141	return to;
1da177e4 LT	142	}
1da177e4 LT	143	}
1da177e4 LT	144
	145	#define __HAVE_ARCH_MEMCPY
	146
	147	#ifdef CONFIG_X86_USE_3DNOW
	148
	149	#include <asm/mmx.h>
	150
	151	/*
	152	* This CPU favours 3DNow strongly (eg AMD Athlon)
	153	*/
	154
78d64fc2	155	static inline void __constant_memcpy3d(void to, const void *from, size_t len)
1da177e4 LT	156	{
	157	if (len < 512)
	158	return __constant_memcpy(to, from, len);
	159	return _mmx_memcpy(to, from, len);
	160	}
	161
78d64fc2	162	static inline void __memcpy3d(void to, const void *from, size_t len)
1da177e4 LT	163	{
	164	if (len < 512)
	165	return __memcpy(to, from, len);
	166	return _mmx_memcpy(to, from, len);
	167	}
	168
78d64fc2 JP	169	#define memcpy(t, f, n) \
	170	(__builtin_constant_p((n)) \
	171	? __constant_memcpy3d((t), (f), (n)) \
	172	: __memcpy3d((t), (f), (n)))
1da177e4 LT	173
	174	#else
	175
	176	/*
	177	* No 3D Now!
	178	*/
78d64fc2 JP	179
	180	#define memcpy(t, f, n) \
	181	(__builtin_constant_p((n)) \
	182	? __constant_memcpy((t), (f), (n)) \
	183	: __memcpy((t), (f), (n)))
1da177e4 LT	184
	185	#endif
	186
	187	#define __HAVE_ARCH_MEMMOVE
78d64fc2	188	void memmove(void dest, const void *src, size_t n);
1da177e4 LT	189
	190	#define memcmp __builtin_memcmp
	191
	192	#define __HAVE_ARCH_MEMCHR
78d64fc2	193	extern void memchr(const void cs, int c, size_t count);
1da177e4	194
78d64fc2	195	static inline void __memset_generic(void s, char c, size_t count)
1da177e4	196	{
78d64fc2 JP	197	int d0, d1;
	198	asm volatile("rep\n\t"
	199	"stosb"
	200	: "=&c" (d0), "=&D" (d1)
	201	: "a" (c), "1" (s), "0" (count)
	202	: "memory");
	203	return s;
1da177e4 LT	204	}
	205
	206	/* we might want to write optimized versions of these later */
78d64fc2	207	#define __constant_count_memset(s, c, count) __memset_generic((s), (c), (count))
1da177e4 LT	208
1da177e4 LT	209	/*
78d64fc2	210	* memset(x, 0, y) is a reasonably common thing to do, so we want to fill
1da177e4 LT	211	* things 32 bits at a time even when we don't know the size of the
	212	* area at compile-time..
	213	*/
78d64fc2 JP	214	static __always_inline
78d64fc2 JP	215	void __constant_c_memset(void s, unsigned long c, size_t count)
1da177e4	216	{
78d64fc2 JP	217	int d0, d1;
	218	asm volatile("rep ; stosl\n\t"
	219	"testb $2,%b3\n\t"
	220	"je 1f\n\t"
	221	"stosw\n"
	222	"1:\ttestb $1,%b3\n\t"
	223	"je 2f\n\t"
	224	"stosb\n"
	225	"2:"
	226	: "=&c" (d0), "=&D" (d1)
	227	: "a" (c), "q" (count), "0" (count/4), "1" ((long)s)
	228	: "memory");
	229	return s;
1da177e4 LT	230	}
	231
	232	/* Added by Gertjan van Wingerde to make minix and sysv module work */
	233	#define __HAVE_ARCH_STRNLEN
78d64fc2	234	extern size_t strnlen(const char *s, size_t count);
1da177e4 LT	235	/* end of additional stuff */
	236
	237	#define __HAVE_ARCH_STRSTR
1da177e4 LT	238	extern char strstr(const char cs, const char *ct);
	239
	240	/*
	241	* This looks horribly ugly, but the compiler can optimize it totally,
	242	* as we by now know that both pattern and count is constant..
	243	*/
78d64fc2 JP	244	static __always_inline
	245	void __constant_c_and_count_memset(void s, unsigned long pattern,
	246	size_t count)
1da177e4 LT	247	{
1da177e4 LT	248	switch (count) {
78d64fc2 JP	249	case 0:
	250	return s;
	251	case 1:
	252	(unsigned char )s = pattern & 0xff;
	253	return s;
	254	case 2:
	255	(unsigned short )s = pattern & 0xffff;
	256	return s;
	257	case 3:
	258	(unsigned short )s = pattern & 0xffff;
	259	((unsigned char )s + 2) = pattern & 0xff;
	260	return s;
	261	case 4:
	262	(unsigned long )s = pattern;
	263	return s;
	264	}
	265
	266	#define COMMON(x) \
	267	asm volatile("rep ; stosl" \
	268	x \
	269	: "=&c" (d0), "=&D" (d1) \
1a20d3ec	270	: "a" (eax), "0" (count/4), "1" ((long)s) \
78d64fc2 JP	271	: "memory")
	272
	273	{
	274	int d0, d1;
1a20d3ec PA	275	#if __GNUC__ == 4 && __GNUC_MINOR__ == 0
	276	/* Workaround for broken gcc 4.0 */
	277	register unsigned long eax asm("%eax") = pattern;
	278	#else
	279	unsigned long eax = pattern;
	280	#endif
	281
78d64fc2	282	switch (count % 4) {
1da177e4	283	case 0:
78d64fc2	284	COMMON("");
1da177e4 LT	285	return s;
1da177e4 LT	286	case 1:
78d64fc2	287	COMMON("\n\tstosb");
1da177e4 LT	288	return s;
1da177e4 LT	289	case 2:
78d64fc2	290	COMMON("\n\tstosw");
1da177e4	291	return s;
78d64fc2 JP	292	default:
78d64fc2 JP	293	COMMON("\n\tstosw\n\tstosb");
1da177e4	294	return s;
78d64fc2	295	}
1da177e4	296	}
78d64fc2	297
1da177e4 LT	298	#undef COMMON
	299	}
	300
78d64fc2 JP	301	#define __constant_c_x_memset(s, c, count) \
	302	(__builtin_constant_p(count) \
	303	? __constant_c_and_count_memset((s), (c), (count)) \
	304	: __constant_c_memset((s), (c), (count)))
1da177e4	305
78d64fc2 JP	306	#define __memset(s, c, count) \
	307	(__builtin_constant_p(count) \
	308	? __constant_count_memset((s), (c), (count)) \
	309	: __memset_generic((s), (c), (count)))
1da177e4 LT	310
1da177e4 LT	311	#define __HAVE_ARCH_MEMSET
78d64fc2 JP	312	#define memset(s, c, count) \
	313	(__builtin_constant_p(c) \
	314	? __constant_c_x_memset((s), (0x01010101UL * (unsigned char)(c)), \
	315	(count)) \
	316	: __memset((s), (c), (count)))
1da177e4 LT	317
	318	/*
	319	* find the first occurrence of byte 'c', or 1 past the area if none
	320	*/
	321	#define __HAVE_ARCH_MEMSCAN
78d64fc2	322	extern void memscan(void addr, int c, size_t size);
1da177e4 LT	323
	324	#endif /* __KERNEL__ */
	325
1965aae3	326	#endif /* _ASM_X86_STRING_32_H */