[mirror_ubuntu-bionic-kernel.git] / arch / ia64 / lib / do_csum.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 *
 * Optmized version of the standard do_csum() function
 *
 * Return: a 64bit quantity containing the 16bit Internet checksum
 *
 * Inputs:
 *	in0: address of buffer to checksum (char *)
 *	in1: length of the buffer (int)
 *
 * Copyright (C) 1999, 2001-2002 Hewlett-Packard Co
 *	Stephane Eranian <eranian@hpl.hp.com>
 *
 * 02/04/22	Ken Chen <kenneth.w.chen@intel.com>
 *		Data locality study on the checksum buffer.
 *		More optimization cleanup - remove excessive stop bits.
 * 02/04/08	David Mosberger <davidm@hpl.hp.com>
 *		More cleanup and tuning.
 * 01/04/18	Jun Nakajima <jun.nakajima@intel.com>
 *		Clean up and optimize and the software pipeline, loading two
 *		back-to-back 8-byte words per loop. Clean up the initialization
 *		for the loop. Support the cases where load latency = 1 or 2.
 *		Set CONFIG_IA64_LOAD_LATENCY to 1 or 2 (default).
 */

#include <asm/asmmacro.h>

//
// Theory of operations:
//	The goal is to go as quickly as possible to the point where
//	we can checksum 16 bytes/loop. Before reaching that point we must
//	take care of incorrect alignment of first byte.
//
//	The code hereafter also takes care of the "tail" part of the buffer
//	before entering the core loop, if any. The checksum is a sum so it
//	allows us to commute operations. So we do the "head" and "tail"
//	first to finish at full speed in the body. Once we get the head and
//	tail values, we feed them into the pipeline, very handy initialization.
//
//	Of course we deal with the special case where the whole buffer fits
//	into one 8 byte word. In this case we have only one entry in the pipeline.
//
//	We use a (LOAD_LATENCY+2)-stage pipeline in the loop to account for
//	possible load latency and also to accommodate for head and tail.
//
//	The end of the function deals with folding the checksum from 64bits
//	down to 16bits taking care of the carry.
//
//	This version avoids synchronization in the core loop by also using a
//	pipeline for the accumulation of the checksum in resultx[] (x=1,2).
//
//	 wordx[] (x=1,2)
//	|---|
//      |   | 0			: new value loaded in pipeline
//	|---|
//      |   | -			: in transit data
//	|---|
//      |   | LOAD_LATENCY	: current value to add to checksum
//	|---|
//      |   | LOAD_LATENCY+1	: previous value added to checksum
//      |---|			(previous iteration)
//
//	resultx[] (x=1,2)
//	|---|
//      |   | 0			: initial value
//	|---|
//      |   | LOAD_LATENCY-1	: new checksum
//	|---|
//      |   | LOAD_LATENCY	: previous value of checksum
//	|---|
//      |   | LOAD_LATENCY+1	: final checksum when out of the loop
//      |---|
//
//
//	See RFC1071 "Computing the Internet Checksum" for various techniques for
//	calculating the Internet checksum.
//
// NOT YET DONE:
//	- Maybe another algorithm which would take care of the folding at the
//	  end in a different manner
//	- Work with people more knowledgeable than me on the network stack
//	  to figure out if we could not split the function depending on the
//	  type of packet or alignment we get. Like the ip_fast_csum() routine
//	  where we know we have at least 20bytes worth of data to checksum.
//	- Do a better job of handling small packets.
//	- Note on prefetching: it was found that under various load, i.e. ftp read/write,
//	  nfs read/write, the L1 cache hit rate is at 60% and L2 cache hit rate is at 99.8%
//	  on the data that buffer points to (partly because the checksum is often preceded by
//	  a copy_from_user()).  This finding indiate that lfetch will not be beneficial since
//	  the data is already in the cache.
//

#define saved_pfs	r11
#define hmask		r16
#define tmask		r17
#define first1		r18
#define firstval	r19
#define firstoff	r20
#define last		r21
#define lastval		r22
#define lastoff		r23
#define saved_lc	r24
#define saved_pr	r25
#define tmp1		r26
#define tmp2		r27
#define tmp3		r28
#define carry1		r29
#define carry2		r30
#define first2		r31

#define buf		in0
#define len		in1

#define LOAD_LATENCY	2	// XXX fix me

#if (LOAD_LATENCY != 1) && (LOAD_LATENCY != 2)
# error "Only 1 or 2 is supported/tested for LOAD_LATENCY."
#endif

#define PIPE_DEPTH			(LOAD_LATENCY+2)
#define ELD	p[LOAD_LATENCY]		// end of load
#define ELD_1	p[LOAD_LATENCY+1]	// and next stage

// unsigned long do_csum(unsigned char *buf,long len)

GLOBAL_ENTRY(do_csum)
	.prologue
	.save ar.pfs, saved_pfs
	alloc saved_pfs=ar.pfs,2,16,0,16
	.rotr word1[4], word2[4],result1[LOAD_LATENCY+2],result2[LOAD_LATENCY+2]
	.rotp p[PIPE_DEPTH], pC1[2], pC2[2]
	mov ret0=r0		// in case we have zero length
	cmp.lt p0,p6=r0,len	// check for zero length or negative (32bit len)
	;;
	add tmp1=buf,len	// last byte's address
	.save pr, saved_pr
	mov saved_pr=pr		// preserve predicates (rotation)
(p6)	br.ret.spnt.many rp	// return if zero or negative length

	mov hmask=-1		// initialize head mask
	tbit.nz p15,p0=buf,0	// is buf an odd address?
	and first1=-8,buf	// 8-byte align down address of first1 element

	and firstoff=7,buf	// how many bytes off for first1 element
	mov tmask=-1		// initialize tail mask

	;;
	adds tmp2=-1,tmp1	// last-1
	and lastoff=7,tmp1	// how many bytes off for last element
	;;
	sub tmp1=8,lastoff	// complement to lastoff
	and last=-8,tmp2	// address of word containing last byte
	;;
	sub tmp3=last,first1	// tmp3=distance from first1 to last
	.save ar.lc, saved_lc
	mov saved_lc=ar.lc	// save lc
	cmp.eq p8,p9=last,first1	// everything fits in one word ?

	ld8 firstval=[first1],8	// load, ahead of time, "first1" word
	and tmp1=7, tmp1	// make sure that if tmp1==8 -> tmp1=0
	shl tmp2=firstoff,3	// number of bits
	;;
(p9)	ld8 lastval=[last]	// load, ahead of time, "last" word, if needed
	shl tmp1=tmp1,3		// number of bits
(p9)	adds tmp3=-8,tmp3	// effectively loaded
	;;
(p8)	mov lastval=r0		// we don't need lastval if first1==last
	shl hmask=hmask,tmp2	// build head mask, mask off [0,first1off[
	shr.u tmask=tmask,tmp1	// build tail mask, mask off ]8,lastoff]
	;;
	.body
#define count tmp3

(p8)	and hmask=hmask,tmask	// apply tail mask to head mask if 1 word only
(p9)	and word2[0]=lastval,tmask	// mask last it as appropriate
	shr.u count=count,3	// how many 8-byte?
	;;
	// If count is odd, finish this 8-byte word so that we can
	// load two back-to-back 8-byte words per loop thereafter.
	and word1[0]=firstval,hmask	// and mask it as appropriate
	tbit.nz p10,p11=count,0		// if (count is odd)
	;;
(p8)	mov result1[0]=word1[0]
(p9)	add result1[0]=word1[0],word2[0]
	;;
	cmp.ltu p6,p0=result1[0],word1[0]	// check the carry
	cmp.eq.or.andcm p8,p0=0,count		// exit if zero 8-byte
	;;
(p6)	adds result1[0]=1,result1[0]
(p8)	br.cond.dptk .do_csum_exit	// if (within an 8-byte word)
(p11)	br.cond.dptk .do_csum16		// if (count is even)

	// Here count is odd.
	ld8 word1[1]=[first1],8		// load an 8-byte word
	cmp.eq p9,p10=1,count		// if (count == 1)
	adds count=-1,count		// loaded an 8-byte word
	;;
	add result1[0]=result1[0],word1[1]
	;;
	cmp.ltu p6,p0=result1[0],word1[1]
	;;
(p6)	adds result1[0]=1,result1[0]
(p9)	br.cond.sptk .do_csum_exit	// if (count == 1) exit
	// Fall through to calculate the checksum, feeding result1[0] as
	// the initial value in result1[0].
	//
	// Calculate the checksum loading two 8-byte words per loop.
	//
.do_csum16:
	add first2=8,first1
	shr.u count=count,1	// we do 16 bytes per loop
	;;
	adds count=-1,count
	mov carry1=r0
	mov carry2=r0
	brp.loop.imp 1f,2f
	;;
	mov ar.ec=PIPE_DEPTH
	mov ar.lc=count	// set lc
	mov pr.rot=1<<16
	// result1[0] must be initialized in advance.
	mov result2[0]=r0
	;;
	.align 32
1:
(ELD_1)	cmp.ltu pC1[0],p0=result1[LOAD_LATENCY],word1[LOAD_LATENCY+1]
(pC1[1])adds carry1=1,carry1
(ELD_1)	cmp.ltu pC2[0],p0=result2[LOAD_LATENCY],word2[LOAD_LATENCY+1]
(pC2[1])adds carry2=1,carry2
(ELD)	add result1[LOAD_LATENCY-1]=result1[LOAD_LATENCY],word1[LOAD_LATENCY]
(ELD)	add result2[LOAD_LATENCY-1]=result2[LOAD_LATENCY],word2[LOAD_LATENCY]
2:
(p[0])	ld8 word1[0]=[first1],16
(p[0])	ld8 word2[0]=[first2],16
	br.ctop.sptk 1b
	;;
	// Since len is a 32-bit value, carry cannot be larger than a 64-bit value.
(pC1[1])adds carry1=1,carry1	// since we miss the last one
(pC2[1])adds carry2=1,carry2
	;;
	add result1[LOAD_LATENCY+1]=result1[LOAD_LATENCY+1],carry1
	add result2[LOAD_LATENCY+1]=result2[LOAD_LATENCY+1],carry2
	;;
	cmp.ltu p6,p0=result1[LOAD_LATENCY+1],carry1
	cmp.ltu p7,p0=result2[LOAD_LATENCY+1],carry2
	;;
(p6)	adds result1[LOAD_LATENCY+1]=1,result1[LOAD_LATENCY+1]
(p7)	adds result2[LOAD_LATENCY+1]=1,result2[LOAD_LATENCY+1]
	;;
	add result1[0]=result1[LOAD_LATENCY+1],result2[LOAD_LATENCY+1]
	;;
	cmp.ltu p6,p0=result1[0],result2[LOAD_LATENCY+1]
	;;
(p6)	adds result1[0]=1,result1[0]
	;;
.do_csum_exit:
	//
	// now fold 64 into 16 bits taking care of carry
	// that's not very good because it has lots of sequentiality
	//
	mov tmp3=0xffff
	zxt4 tmp1=result1[0]
	shr.u tmp2=result1[0],32
	;;
	add result1[0]=tmp1,tmp2
	;;
	and tmp1=result1[0],tmp3
	shr.u tmp2=result1[0],16
	;;
	add result1[0]=tmp1,tmp2
	;;
	and tmp1=result1[0],tmp3
	shr.u tmp2=result1[0],16
	;;
	add result1[0]=tmp1,tmp2
	;;
	and tmp1=result1[0],tmp3
	shr.u tmp2=result1[0],16
	;;
	add ret0=tmp1,tmp2
	mov pr=saved_pr,0xffffffffffff0000
	;;
	// if buf was odd then swap bytes
	mov ar.pfs=saved_pfs		// restore ar.ec
(p15)	mux1 ret0=ret0,@rev		// reverse word
	;;
	mov ar.lc=saved_lc
(p15)	shr.u ret0=ret0,64-16	// + shift back to position = swap bytes
	br.ret.sptk.many rp

//	I (Jun Nakajima) wrote an equivalent code (see below), but it was
//	not much better than the original. So keep the original there so that
//	someone else can challenge.
//
//	shr.u word1[0]=result1[0],32
//	zxt4 result1[0]=result1[0]
//	;;
//	add result1[0]=result1[0],word1[0]
//	;;
//	zxt2 result2[0]=result1[0]
//	extr.u word1[0]=result1[0],16,16
//	shr.u carry1=result1[0],32
//	;;
//	add result2[0]=result2[0],word1[0]
//	;;
//	add result2[0]=result2[0],carry1
//	;;
//	extr.u ret0=result2[0],16,16
//	;;
//	add ret0=ret0,result2[0]
//	;;
//	zxt2 ret0=ret0
//	mov ar.pfs=saved_pfs		 // restore ar.ec
//	mov pr=saved_pr,0xffffffffffff0000
//	;;
//	// if buf was odd then swap bytes
//	mov ar.lc=saved_lc
//(p15)	mux1 ret0=ret0,@rev		// reverse word
//	;;
//(p15)	shr.u ret0=ret0,64-16	// + shift back to position = swap bytes
//	br.ret.sptk.many rp

END(do_csum)
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
1da177e4 LT	2	/*
	3	*
	4	* Optmized version of the standard do_csum() function
	5	*
	6	* Return: a 64bit quantity containing the 16bit Internet checksum
	7	*
	8	* Inputs:
	9	* in0: address of buffer to checksum (char *)
	10	* in1: length of the buffer (int)
	11	*
	12	* Copyright (C) 1999, 2001-2002 Hewlett-Packard Co
	13	* Stephane Eranian <eranian@hpl.hp.com>
	14	*
	15	* 02/04/22 Ken Chen <kenneth.w.chen@intel.com>
	16	* Data locality study on the checksum buffer.
	17	* More optimization cleanup - remove excessive stop bits.
	18	* 02/04/08 David Mosberger <davidm@hpl.hp.com>
	19	* More cleanup and tuning.
	20	* 01/04/18 Jun Nakajima <jun.nakajima@intel.com>
	21	* Clean up and optimize and the software pipeline, loading two
	22	* back-to-back 8-byte words per loop. Clean up the initialization
	23	* for the loop. Support the cases where load latency = 1 or 2.
	24	* Set CONFIG_IA64_LOAD_LATENCY to 1 or 2 (default).
	25	*/
	26
	27	#include <asm/asmmacro.h>
	28
	29	//
	30	// Theory of operations:
	31	// The goal is to go as quickly as possible to the point where
	32	// we can checksum 16 bytes/loop. Before reaching that point we must
	33	// take care of incorrect alignment of first byte.
	34	//
	35	// The code hereafter also takes care of the "tail" part of the buffer
	36	// before entering the core loop, if any. The checksum is a sum so it
	37	// allows us to commute operations. So we do the "head" and "tail"
	38	// first to finish at full speed in the body. Once we get the head and
	39	// tail values, we feed them into the pipeline, very handy initialization.
	40	//
	41	// Of course we deal with the special case where the whole buffer fits
	42	// into one 8 byte word. In this case we have only one entry in the pipeline.
	43	//
	44	// We use a (LOAD_LATENCY+2)-stage pipeline in the loop to account for
	45	// possible load latency and also to accommodate for head and tail.
	46	//
	47	// The end of the function deals with folding the checksum from 64bits
	48	// down to 16bits taking care of the carry.
	49	//
	50	// This version avoids synchronization in the core loop by also using a
	51	// pipeline for the accumulation of the checksum in resultx[] (x=1,2).
	52	//
	53	// wordx[] (x=1,2)
	54	// \|---\|
	55	// \| \| 0 : new value loaded in pipeline
	56	// \|---\|
	57	// \| \| - : in transit data
	58	// \|---\|
	59	// \| \| LOAD_LATENCY : current value to add to checksum
	60	// \|---\|
	61	// \| \| LOAD_LATENCY+1 : previous value added to checksum
	62	// \|---\| (previous iteration)
	63	//
	64	// resultx[] (x=1,2)
	65	// \|---\|
66	// \| \| 0 : initial value
67	// \|---\|
68	// \| \| LOAD_LATENCY-1 : new checksum
69	// \|---\|
70	// \| \| LOAD_LATENCY : previous value of checksum
71	// \|---\|
72	// \| \| LOAD_LATENCY+1 : final checksum when out of the loop
73	// \|---\|
74	//
75	//
76	// See RFC1071 "Computing the Internet Checksum" for various techniques for
77	// calculating the Internet checksum.
78	//
79	// NOT YET DONE:
80	// - Maybe another algorithm which would take care of the folding at the
81	// end in a different manner
82	// - Work with people more knowledgeable than me on the network stack
83	// to figure out if we could not split the function depending on the
84	// type of packet or alignment we get. Like the ip_fast_csum() routine
85	// where we know we have at least 20bytes worth of data to checksum.
86	// - Do a better job of handling small packets.
87	// - Note on prefetching: it was found that under various load, i.e. ftp read/write,
88	// nfs read/write, the L1 cache hit rate is at 60% and L2 cache hit rate is at 99.8%
89	// on the data that buffer points to (partly because the checksum is often preceded by
90	// a copy_from_user()). This finding indiate that lfetch will not be beneficial since
91	// the data is already in the cache.
92	//
93
94	#define saved_pfs r11
95	#define hmask r16
96	#define tmask r17
97	#define first1 r18
98	#define firstval r19
99	#define firstoff r20
100	#define last r21
101	#define lastval r22
102	#define lastoff r23
103	#define saved_lc r24
104	#define saved_pr r25
105	#define tmp1 r26
106	#define tmp2 r27
107	#define tmp3 r28
108	#define carry1 r29
109	#define carry2 r30
110	#define first2 r31
111
112	#define buf in0
113	#define len in1
114
115	#define LOAD_LATENCY 2 // XXX fix me
116
117	#if (LOAD_LATENCY != 1) && (LOAD_LATENCY != 2)
118	# error "Only 1 or 2 is supported/tested for LOAD_LATENCY."
119	#endif
120
121	#define PIPE_DEPTH (LOAD_LATENCY+2)
122	#define ELD p[LOAD_LATENCY] // end of load
123	#define ELD_1 p[LOAD_LATENCY+1] // and next stage
124
125	// unsigned long do_csum(unsigned char *buf,long len)
126
127	GLOBAL_ENTRY(do_csum)
128	.prologue
129	.save ar.pfs, saved_pfs
130	alloc saved_pfs=ar.pfs,2,16,0,16
131	.rotr word1[4], word2[4],result1[LOAD_LATENCY+2],result2[LOAD_LATENCY+2]
132	.rotp p[PIPE_DEPTH], pC1[2], pC2[2]
133	mov ret0=r0 // in case we have zero length
134	cmp.lt p0,p6=r0,len // check for zero length or negative (32bit len)
135	;;
136	add tmp1=buf,len // last byte's address
137	.save pr, saved_pr
138	mov saved_pr=pr // preserve predicates (rotation)
139	(p6) br.ret.spnt.many rp // return if zero or negative length
140
141	mov hmask=-1 // initialize head mask
142	tbit.nz p15,p0=buf,0 // is buf an odd address?
143	and first1=-8,buf // 8-byte align down address of first1 element
144
145	and firstoff=7,buf // how many bytes off for first1 element
146	mov tmask=-1 // initialize tail mask
147
148	;;
149	adds tmp2=-1,tmp1 // last-1
150	and lastoff=7,tmp1 // how many bytes off for last element
151	;;
152	sub tmp1=8,lastoff // complement to lastoff
153	and last=-8,tmp2 // address of word containing last byte
154	;;
155	sub tmp3=last,first1 // tmp3=distance from first1 to last
156	.save ar.lc, saved_lc
157	mov saved_lc=ar.lc // save lc
158	cmp.eq p8,p9=last,first1 // everything fits in one word ?
159
160	ld8 firstval=[first1],8 // load, ahead of time, "first1" word
161	and tmp1=7, tmp1 // make sure that if tmp1==8 -> tmp1=0
162	shl tmp2=firstoff,3 // number of bits
163	;;
164	(p9) ld8 lastval=[last] // load, ahead of time, "last" word, if needed
165	shl tmp1=tmp1,3 // number of bits
166	(p9) adds tmp3=-8,tmp3 // effectively loaded
167	;;
168	(p8) mov lastval=r0 // we don't need lastval if first1==last
169	shl hmask=hmask,tmp2 // build head mask, mask off [0,first1off[
170	shr.u tmask=tmask,tmp1 // build tail mask, mask off ]8,lastoff]
171	;;
172	.body
173	#define count tmp3
174
175	(p8) and hmask=hmask,tmask // apply tail mask to head mask if 1 word only
176	(p9) and word2[0]=lastval,tmask // mask last it as appropriate
177	shr.u count=count,3 // how many 8-byte?
178	;;
179	// If count is odd, finish this 8-byte word so that we can
180	// load two back-to-back 8-byte words per loop thereafter.
181	and word1[0]=firstval,hmask // and mask it as appropriate
182	tbit.nz p10,p11=count,0 // if (count is odd)
183	;;
184	(p8) mov result1[0]=word1[0]
185	(p9) add result1[0]=word1[0],word2[0]
186	;;
187	cmp.ltu p6,p0=result1[0],word1[0] // check the carry
188	cmp.eq.or.andcm p8,p0=0,count // exit if zero 8-byte
189	;;
190	(p6) adds result1[0]=1,result1[0]
191	(p8) br.cond.dptk .do_csum_exit // if (within an 8-byte word)
192	(p11) br.cond.dptk .do_csum16 // if (count is even)
193
194	// Here count is odd.
195	ld8 word1[1]=[first1],8 // load an 8-byte word
196	cmp.eq p9,p10=1,count // if (count == 1)
197	adds count=-1,count // loaded an 8-byte word
198	;;
199	add result1[0]=result1[0],word1[1]
200	;;
201	cmp.ltu p6,p0=result1[0],word1[1]
202	;;
203	(p6) adds result1[0]=1,result1[0]
204	(p9) br.cond.sptk .do_csum_exit // if (count == 1) exit
25985edc	205	// Fall through to calculate the checksum, feeding result1[0] as
1da177e4 LT	206	// the initial value in result1[0].
	207	//
	208	// Calculate the checksum loading two 8-byte words per loop.
	209	//
	210	.do_csum16:
	211	add first2=8,first1
	212	shr.u count=count,1 // we do 16 bytes per loop
	213	;;
	214	adds count=-1,count
	215	mov carry1=r0
	216	mov carry2=r0
	217	brp.loop.imp 1f,2f
	218	;;
	219	mov ar.ec=PIPE_DEPTH
	220	mov ar.lc=count // set lc
	221	mov pr.rot=1<<16
	222	// result1[0] must be initialized in advance.
	223	mov result2[0]=r0
	224	;;
	225	.align 32
	226	1:
	227	(ELD_1) cmp.ltu pC1[0],p0=result1[LOAD_LATENCY],word1[LOAD_LATENCY+1]
	228	(pC1[1])adds carry1=1,carry1
	229	(ELD_1) cmp.ltu pC2[0],p0=result2[LOAD_LATENCY],word2[LOAD_LATENCY+1]
	230	(pC2[1])adds carry2=1,carry2
	231	(ELD) add result1[LOAD_LATENCY-1]=result1[LOAD_LATENCY],word1[LOAD_LATENCY]
	232	(ELD) add result2[LOAD_LATENCY-1]=result2[LOAD_LATENCY],word2[LOAD_LATENCY]
	233	2:
	234	(p[0]) ld8 word1[0]=[first1],16
	235	(p[0]) ld8 word2[0]=[first2],16
	236	br.ctop.sptk 1b
	237	;;
	238	// Since len is a 32-bit value, carry cannot be larger than a 64-bit value.
	239	(pC1[1])adds carry1=1,carry1 // since we miss the last one
	240	(pC2[1])adds carry2=1,carry2
	241	;;
	242	add result1[LOAD_LATENCY+1]=result1[LOAD_LATENCY+1],carry1
	243	add result2[LOAD_LATENCY+1]=result2[LOAD_LATENCY+1],carry2
	244	;;
	245	cmp.ltu p6,p0=result1[LOAD_LATENCY+1],carry1
	246	cmp.ltu p7,p0=result2[LOAD_LATENCY+1],carry2
	247	;;
	248	(p6) adds result1[LOAD_LATENCY+1]=1,result1[LOAD_LATENCY+1]
	249	(p7) adds result2[LOAD_LATENCY+1]=1,result2[LOAD_LATENCY+1]
	250	;;
	251	add result1[0]=result1[LOAD_LATENCY+1],result2[LOAD_LATENCY+1]
	252	;;
	253	cmp.ltu p6,p0=result1[0],result2[LOAD_LATENCY+1]
	254	;;
	255	(p6) adds result1[0]=1,result1[0]
	256	;;
	257	.do_csum_exit:
	258	//
	259	// now fold 64 into 16 bits taking care of carry
	260	// that's not very good because it has lots of sequentiality
	261	//
	262	mov tmp3=0xffff
	263	zxt4 tmp1=result1[0]
	264	shr.u tmp2=result1[0],32
	265	;;
	266	add result1[0]=tmp1,tmp2
	267	;;
	268	and tmp1=result1[0],tmp3
	269	shr.u tmp2=result1[0],16
270	;;
271	add result1[0]=tmp1,tmp2
272	;;
273	and tmp1=result1[0],tmp3
274	shr.u tmp2=result1[0],16
275	;;
276	add result1[0]=tmp1,tmp2
277	;;
278	and tmp1=result1[0],tmp3
279	shr.u tmp2=result1[0],16
280	;;
281	add ret0=tmp1,tmp2
282	mov pr=saved_pr,0xffffffffffff0000
283	;;
284	// if buf was odd then swap bytes
285	mov ar.pfs=saved_pfs // restore ar.ec
286	(p15) mux1 ret0=ret0,@rev // reverse word
287	;;
288	mov ar.lc=saved_lc
289	(p15) shr.u ret0=ret0,64-16 // + shift back to position = swap bytes
290	br.ret.sptk.many rp
291
292	// I (Jun Nakajima) wrote an equivalent code (see below), but it was
293	// not much better than the original. So keep the original there so that
294	// someone else can challenge.
295	//
296	// shr.u word1[0]=result1[0],32
297	// zxt4 result1[0]=result1[0]
298	// ;;
299	// add result1[0]=result1[0],word1[0]
300	// ;;
301	// zxt2 result2[0]=result1[0]
302	// extr.u word1[0]=result1[0],16,16
303	// shr.u carry1=result1[0],32
304	// ;;
305	// add result2[0]=result2[0],word1[0]
306	// ;;
307	// add result2[0]=result2[0],carry1
308	// ;;
309	// extr.u ret0=result2[0],16,16
310	// ;;
311	// add ret0=ret0,result2[0]
312	// ;;
313	// zxt2 ret0=ret0
314	// mov ar.pfs=saved_pfs // restore ar.ec
315	// mov pr=saved_pr,0xffffffffffff0000
316	// ;;
317	// // if buf was odd then swap bytes
318	// mov ar.lc=saved_lc
319	//(p15) mux1 ret0=ret0,@rev // reverse word
320	// ;;
321	//(p15) shr.u ret0=ret0,64-16 // + shift back to position = swap bytes
322	// br.ret.sptk.many rp
323
324	END(do_csum)