[mirror_ubuntu-bionic-kernel.git] / arch / alpha / lib / memcpy.c

// SPDX-License-Identifier: GPL-2.0
/*
 *  linux/arch/alpha/lib/memcpy.c
 *
 *  Copyright (C) 1995  Linus Torvalds
 */

/*
 * This is a reasonably optimized memcpy() routine.
 */

/*
 * Note that the C code is written to be optimized into good assembly. However,
 * at this point gcc is unable to sanely compile "if (n >= 0)", resulting in a
 * explicit compare against 0 (instead of just using the proper "blt reg, xx" or
 * "bge reg, xx"). I hope alpha-gcc will be fixed to notice this eventually..
 */

#include <linux/types.h>
#include <linux/export.h>

/*
 * This should be done in one go with ldq_u*2/mask/stq_u. Do it
 * with a macro so that we can fix it up later..
 */
#define ALIGN_DEST_TO8_UP(d,s,n) \
	while (d & 7) { \
		if (n <= 0) return; \
		n--; \
		*(char *) d = *(char *) s; \
		d++; s++; \
	}
#define ALIGN_DEST_TO8_DN(d,s,n) \
	while (d & 7) { \
		if (n <= 0) return; \
		n--; \
		d--; s--; \
		*(char *) d = *(char *) s; \
	}

/*
 * This should similarly be done with ldq_u*2/mask/stq. The destination
 * is aligned, but we don't fill in a full quad-word
 */
#define DO_REST_UP(d,s,n) \
	while (n > 0) { \
		n--; \
		*(char *) d = *(char *) s; \
		d++; s++; \
	}
#define DO_REST_DN(d,s,n) \
	while (n > 0) { \
		n--; \
		d--; s--; \
		*(char *) d = *(char *) s; \
	}

/*
 * This should be done with ldq/mask/stq. The source and destination are
 * aligned, but we don't fill in a full quad-word
 */
#define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n)
#define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n)

/*
 * This does unaligned memory copies. We want to avoid storing to
 * an unaligned address, as that would do a read-modify-write cycle.
 * We also want to avoid double-reading the unaligned reads.
 *
 * Note the ordering to try to avoid load (and address generation) latencies.
 */
static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s,
					  long n)
{
	ALIGN_DEST_TO8_UP(d,s,n);
	n -= 8;			/* to avoid compare against 8 in the loop */
	if (n >= 0) {
		unsigned long low_word, high_word;
		__asm__("ldq_u %0,%1":"=r" (low_word):"m" (*(unsigned long *) s));
		do {
			unsigned long tmp;
			__asm__("ldq_u %0,%1":"=r" (high_word):"m" (*(unsigned long *)(s+8)));
			n -= 8;
			__asm__("extql %1,%2,%0"
				:"=r" (low_word)
				:"r" (low_word), "r" (s));
			__asm__("extqh %1,%2,%0"
				:"=r" (tmp)
				:"r" (high_word), "r" (s));
			s += 8;
			*(unsigned long *) d = low_word | tmp;
			d += 8;
			low_word = high_word;
		} while (n >= 0);
	}
	n += 8;
	DO_REST_UP(d,s,n);
}

static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s,
					  long n)
{
	/* I don't understand AXP assembler well enough for this. -Tim */
	s += n;
	d += n;
	while (n--)
		* (char *) --d = * (char *) --s;
}

/*
 * Hmm.. Strange. The __asm__ here is there to make gcc use an integer register
 * for the load-store. I don't know why, but it would seem that using a floating
 * point register for the move seems to slow things down (very small difference,
 * though).
 *
 * Note the ordering to try to avoid load (and address generation) latencies.
 */
static inline void __memcpy_aligned_up (unsigned long d, unsigned long s,
					long n)
{
	ALIGN_DEST_TO8_UP(d,s,n);
	n -= 8;
	while (n >= 0) {
		unsigned long tmp;
		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
		n -= 8;
		s += 8;
		*(unsigned long *) d = tmp;
		d += 8;
	}
	n += 8;
	DO_REST_ALIGNED_UP(d,s,n);
}
static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s,
					long n)
{
	s += n;
	d += n;
	ALIGN_DEST_TO8_DN(d,s,n);
	n -= 8;
	while (n >= 0) {
		unsigned long tmp;
		s -= 8;
		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));
		n -= 8;
		d -= 8;
		*(unsigned long *) d = tmp;
	}
	n += 8;
	DO_REST_ALIGNED_DN(d,s,n);
}

void * memcpy(void * dest, const void *src, size_t n)
{
	if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) {
		__memcpy_aligned_up ((unsigned long) dest, (unsigned long) src,
				     n);
		return dest;
	}
	__memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n);
	return dest;
}
EXPORT_SYMBOL(memcpy);
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* linux/arch/alpha/lib/memcpy.c
	4	*
	5	* Copyright (C) 1995 Linus Torvalds
	6	*/
	7
	8	/*
	9	* This is a reasonably optimized memcpy() routine.
	10	*/
	11
	12	/*
	13	* Note that the C code is written to be optimized into good assembly. However,
	14	* at this point gcc is unable to sanely compile "if (n >= 0)", resulting in a
	15	* explicit compare against 0 (instead of just using the proper "blt reg, xx" or
	16	* "bge reg, xx"). I hope alpha-gcc will be fixed to notice this eventually..
	17	*/
	18
	19	#include <linux/types.h>
00fc0e0d	20	#include <linux/export.h>
1da177e4 LT	21
	22	/*
	23	* This should be done in one go with ldq_u*2/mask/stq_u. Do it
	24	* with a macro so that we can fix it up later..
	25	*/
	26	#define ALIGN_DEST_TO8_UP(d,s,n) \
	27	while (d & 7) { \
	28	if (n <= 0) return; \
	29	n--; \
	30	(char ) d = (char ) s; \
	31	d++; s++; \
	32	}
	33	#define ALIGN_DEST_TO8_DN(d,s,n) \
	34	while (d & 7) { \
	35	if (n <= 0) return; \
	36	n--; \
	37	d--; s--; \
	38	(char ) d = (char ) s; \
	39	}
	40
	41	/*
	42	* This should similarly be done with ldq_u*2/mask/stq. The destination
	43	* is aligned, but we don't fill in a full quad-word
	44	*/
	45	#define DO_REST_UP(d,s,n) \
	46	while (n > 0) { \
	47	n--; \
	48	(char ) d = (char ) s; \
	49	d++; s++; \
	50	}
	51	#define DO_REST_DN(d,s,n) \
	52	while (n > 0) { \
	53	n--; \
	54	d--; s--; \
	55	(char ) d = (char ) s; \
	56	}
	57
	58	/*
	59	* This should be done with ldq/mask/stq. The source and destination are
	60	* aligned, but we don't fill in a full quad-word
	61	*/
	62	#define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n)
	63	#define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n)
	64
	65	/*
	66	* This does unaligned memory copies. We want to avoid storing to
	67	* an unaligned address, as that would do a read-modify-write cycle.
	68	* We also want to avoid double-reading the unaligned reads.
	69	*
	70	* Note the ordering to try to avoid load (and address generation) latencies.
	71	*/
	72	static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s,
	73	long n)
	74	{
	75	ALIGN_DEST_TO8_UP(d,s,n);
	76	n -= 8; /* to avoid compare against 8 in the loop */
	77	if (n >= 0) {
	78	unsigned long low_word, high_word;
	79	__asm__("ldq_u %0,%1":"=r" (low_word):"m" ((unsigned long ) s));
	80	do {
	81	unsigned long tmp;
	82	__asm__("ldq_u %0,%1":"=r" (high_word):"m" ((unsigned long )(s+8)));
	83	n -= 8;
	84	__asm__("extql %1,%2,%0"
85	:"=r" (low_word)
86	:"r" (low_word), "r" (s));
87	__asm__("extqh %1,%2,%0"
88	:"=r" (tmp)
89	:"r" (high_word), "r" (s));
90	s += 8;
91	(unsigned long ) d = low_word \| tmp;
92	d += 8;
93	low_word = high_word;
94	} while (n >= 0);
95	}
96	n += 8;
97	DO_REST_UP(d,s,n);
98	}
99
100	static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s,
101	long n)
102	{
103	/* I don't understand AXP assembler well enough for this. -Tim */
104	s += n;
105	d += n;
106	while (n--)
107	* (char ) --d = (char *) --s;
108	}
109
110	/*
111	* Hmm.. Strange. The __asm__ here is there to make gcc use an integer register
112	* for the load-store. I don't know why, but it would seem that using a floating
113	* point register for the move seems to slow things down (very small difference,
114	* though).
115	*
116	* Note the ordering to try to avoid load (and address generation) latencies.
117	*/
118	static inline void __memcpy_aligned_up (unsigned long d, unsigned long s,
119	long n)
120	{
121	ALIGN_DEST_TO8_UP(d,s,n);
122	n -= 8;
123	while (n >= 0) {
124	unsigned long tmp;
125	__asm__("ldq %0,%1":"=r" (tmp):"m" ((unsigned long ) s));
126	n -= 8;
127	s += 8;
128	(unsigned long ) d = tmp;
129	d += 8;
130	}
131	n += 8;
132	DO_REST_ALIGNED_UP(d,s,n);
133	}
134	static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s,
135	long n)
136	{
137	s += n;
138	d += n;
139	ALIGN_DEST_TO8_DN(d,s,n);
140	n -= 8;
141	while (n >= 0) {
142	unsigned long tmp;
143	s -= 8;
144	__asm__("ldq %0,%1":"=r" (tmp):"m" ((unsigned long ) s));
145	n -= 8;
146	d -= 8;
147	(unsigned long ) d = tmp;
148	}
149	n += 8;
150	DO_REST_ALIGNED_DN(d,s,n);
151	}
152
153	void * memcpy(void * dest, const void *src, size_t n)
154	{
155	if (!(((unsigned long) dest ^ (unsigned long) src) & 7)) {
156	__memcpy_aligned_up ((unsigned long) dest, (unsigned long) src,
157	n);
158	return dest;
159	}
160	__memcpy_unaligned_up ((unsigned long) dest, (unsigned long) src, n);
161	return dest;
162	}
00fc0e0d	163	EXPORT_SYMBOL(memcpy);