obj-y := io.o iomap_copy_64.o
obj-$(CONFIG_SMP) += msr-on-cpu.o
-lib-y := csum-partial.o csum-copy.o csum-wrappers.o delay.o \
+lib-y := csum-partial.o csum-copy_64.o csum-wrappers.o delay.o \
usercopy.o getuser.o putuser.o \
thunk_64.o clear_page.o copy_page_64.o bitstr_64.o bitops.o
lib-y += memcpy.o memmove.o memset.o copy_user.o rwlock.o copy_user_nocache.o
+++ /dev/null
-/*
- * Copyright 2002,2003 Andi Kleen, SuSE Labs.
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file COPYING in the main directory of this archive
- * for more details. No warranty for anything given at all.
- */
-#include <linux/linkage.h>
-#include <asm/dwarf2.h>
-#include <asm/errno.h>
-
-/*
- * Checksum copy with exception handling.
- * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
- * destination is zeroed.
- *
- * Input
- * rdi source
- * rsi destination
- * edx len (32bit)
- * ecx sum (32bit)
- * r8 src_err_ptr (int)
- * r9 dst_err_ptr (int)
- *
- * Output
- * eax 64bit sum. undefined in case of exception.
- *
- * Wrappers need to take care of valid exception sum and zeroing.
- * They also should align source or destination to 8 bytes.
- */
-
- .macro source
-10:
- .section __ex_table,"a"
- .align 8
- .quad 10b,.Lbad_source
- .previous
- .endm
-
- .macro dest
-20:
- .section __ex_table,"a"
- .align 8
- .quad 20b,.Lbad_dest
- .previous
- .endm
-
- .macro ignore L=.Lignore
-30:
- .section __ex_table,"a"
- .align 8
- .quad 30b,\L
- .previous
- .endm
-
-
-ENTRY(csum_partial_copy_generic)
- CFI_STARTPROC
- cmpl $3*64,%edx
- jle .Lignore
-
-.Lignore:
- subq $7*8,%rsp
- CFI_ADJUST_CFA_OFFSET 7*8
- movq %rbx,2*8(%rsp)
- CFI_REL_OFFSET rbx, 2*8
- movq %r12,3*8(%rsp)
- CFI_REL_OFFSET r12, 3*8
- movq %r14,4*8(%rsp)
- CFI_REL_OFFSET r14, 4*8
- movq %r13,5*8(%rsp)
- CFI_REL_OFFSET r13, 5*8
- movq %rbp,6*8(%rsp)
- CFI_REL_OFFSET rbp, 6*8
-
- movq %r8,(%rsp)
- movq %r9,1*8(%rsp)
-
- movl %ecx,%eax
- movl %edx,%ecx
-
- xorl %r9d,%r9d
- movq %rcx,%r12
-
- shrq $6,%r12
- jz .Lhandle_tail /* < 64 */
-
- clc
-
- /* main loop. clear in 64 byte blocks */
- /* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
- /* r11: temp3, rdx: temp4, r12 loopcnt */
- /* r10: temp5, rbp: temp6, r14 temp7, r13 temp8 */
- .p2align 4
-.Lloop:
- source
- movq (%rdi),%rbx
- source
- movq 8(%rdi),%r8
- source
- movq 16(%rdi),%r11
- source
- movq 24(%rdi),%rdx
-
- source
- movq 32(%rdi),%r10
- source
- movq 40(%rdi),%rbp
- source
- movq 48(%rdi),%r14
- source
- movq 56(%rdi),%r13
-
- ignore 2f
- prefetcht0 5*64(%rdi)
-2:
- adcq %rbx,%rax
- adcq %r8,%rax
- adcq %r11,%rax
- adcq %rdx,%rax
- adcq %r10,%rax
- adcq %rbp,%rax
- adcq %r14,%rax
- adcq %r13,%rax
-
- decl %r12d
-
- dest
- movq %rbx,(%rsi)
- dest
- movq %r8,8(%rsi)
- dest
- movq %r11,16(%rsi)
- dest
- movq %rdx,24(%rsi)
-
- dest
- movq %r10,32(%rsi)
- dest
- movq %rbp,40(%rsi)
- dest
- movq %r14,48(%rsi)
- dest
- movq %r13,56(%rsi)
-
-3:
-
- leaq 64(%rdi),%rdi
- leaq 64(%rsi),%rsi
-
- jnz .Lloop
-
- adcq %r9,%rax
-
- /* do last upto 56 bytes */
-.Lhandle_tail:
- /* ecx: count */
- movl %ecx,%r10d
- andl $63,%ecx
- shrl $3,%ecx
- jz .Lfold
- clc
- .p2align 4
-.Lloop_8:
- source
- movq (%rdi),%rbx
- adcq %rbx,%rax
- decl %ecx
- dest
- movq %rbx,(%rsi)
- leaq 8(%rsi),%rsi /* preserve carry */
- leaq 8(%rdi),%rdi
- jnz .Lloop_8
- adcq %r9,%rax /* add in carry */
-
-.Lfold:
- /* reduce checksum to 32bits */
- movl %eax,%ebx
- shrq $32,%rax
- addl %ebx,%eax
- adcl %r9d,%eax
-
- /* do last upto 6 bytes */
-.Lhandle_7:
- movl %r10d,%ecx
- andl $7,%ecx
- shrl $1,%ecx
- jz .Lhandle_1
- movl $2,%edx
- xorl %ebx,%ebx
- clc
- .p2align 4
-.Lloop_1:
- source
- movw (%rdi),%bx
- adcl %ebx,%eax
- decl %ecx
- dest
- movw %bx,(%rsi)
- leaq 2(%rdi),%rdi
- leaq 2(%rsi),%rsi
- jnz .Lloop_1
- adcl %r9d,%eax /* add in carry */
-
- /* handle last odd byte */
-.Lhandle_1:
- testl $1,%r10d
- jz .Lende
- xorl %ebx,%ebx
- source
- movb (%rdi),%bl
- dest
- movb %bl,(%rsi)
- addl %ebx,%eax
- adcl %r9d,%eax /* carry */
-
- CFI_REMEMBER_STATE
-.Lende:
- movq 2*8(%rsp),%rbx
- CFI_RESTORE rbx
- movq 3*8(%rsp),%r12
- CFI_RESTORE r12
- movq 4*8(%rsp),%r14
- CFI_RESTORE r14
- movq 5*8(%rsp),%r13
- CFI_RESTORE r13
- movq 6*8(%rsp),%rbp
- CFI_RESTORE rbp
- addq $7*8,%rsp
- CFI_ADJUST_CFA_OFFSET -7*8
- ret
- CFI_RESTORE_STATE
-
- /* Exception handlers. Very simple, zeroing is done in the wrappers */
-.Lbad_source:
- movq (%rsp),%rax
- testq %rax,%rax
- jz .Lende
- movl $-EFAULT,(%rax)
- jmp .Lende
-
-.Lbad_dest:
- movq 8(%rsp),%rax
- testq %rax,%rax
- jz .Lende
- movl $-EFAULT,(%rax)
- jmp .Lende
- CFI_ENDPROC
-ENDPROC(csum_partial_copy_generic)
--- /dev/null
+/*
+ * Copyright 2002,2003 Andi Kleen, SuSE Labs.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details. No warranty for anything given at all.
+ */
+#include <linux/linkage.h>
+#include <asm/dwarf2.h>
+#include <asm/errno.h>
+
+/*
+ * Checksum copy with exception handling.
+ * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the
+ * destination is zeroed.
+ *
+ * Input
+ * rdi source
+ * rsi destination
+ * edx len (32bit)
+ * ecx sum (32bit)
+ * r8 src_err_ptr (int)
+ * r9 dst_err_ptr (int)
+ *
+ * Output
+ * eax 64bit sum. undefined in case of exception.
+ *
+ * Wrappers need to take care of valid exception sum and zeroing.
+ * They also should align source or destination to 8 bytes.
+ */
+
+ .macro source
+10:
+ .section __ex_table,"a"
+ .align 8
+ .quad 10b,.Lbad_source
+ .previous
+ .endm
+
+ .macro dest
+20:
+ .section __ex_table,"a"
+ .align 8
+ .quad 20b,.Lbad_dest
+ .previous
+ .endm
+
+ .macro ignore L=.Lignore
+30:
+ .section __ex_table,"a"
+ .align 8
+ .quad 30b,\L
+ .previous
+ .endm
+
+
+ENTRY(csum_partial_copy_generic)
+ CFI_STARTPROC
+ cmpl $3*64,%edx
+ jle .Lignore
+
+.Lignore:
+ subq $7*8,%rsp
+ CFI_ADJUST_CFA_OFFSET 7*8
+ movq %rbx,2*8(%rsp)
+ CFI_REL_OFFSET rbx, 2*8
+ movq %r12,3*8(%rsp)
+ CFI_REL_OFFSET r12, 3*8
+ movq %r14,4*8(%rsp)
+ CFI_REL_OFFSET r14, 4*8
+ movq %r13,5*8(%rsp)
+ CFI_REL_OFFSET r13, 5*8
+ movq %rbp,6*8(%rsp)
+ CFI_REL_OFFSET rbp, 6*8
+
+ movq %r8,(%rsp)
+ movq %r9,1*8(%rsp)
+
+ movl %ecx,%eax
+ movl %edx,%ecx
+
+ xorl %r9d,%r9d
+ movq %rcx,%r12
+
+ shrq $6,%r12
+ jz .Lhandle_tail /* < 64 */
+
+ clc
+
+ /* main loop. clear in 64 byte blocks */
+ /* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
+ /* r11: temp3, rdx: temp4, r12 loopcnt */
+ /* r10: temp5, rbp: temp6, r14 temp7, r13 temp8 */
+ .p2align 4
+.Lloop:
+ source
+ movq (%rdi),%rbx
+ source
+ movq 8(%rdi),%r8
+ source
+ movq 16(%rdi),%r11
+ source
+ movq 24(%rdi),%rdx
+
+ source
+ movq 32(%rdi),%r10
+ source
+ movq 40(%rdi),%rbp
+ source
+ movq 48(%rdi),%r14
+ source
+ movq 56(%rdi),%r13
+
+ ignore 2f
+ prefetcht0 5*64(%rdi)
+2:
+ adcq %rbx,%rax
+ adcq %r8,%rax
+ adcq %r11,%rax
+ adcq %rdx,%rax
+ adcq %r10,%rax
+ adcq %rbp,%rax
+ adcq %r14,%rax
+ adcq %r13,%rax
+
+ decl %r12d
+
+ dest
+ movq %rbx,(%rsi)
+ dest
+ movq %r8,8(%rsi)
+ dest
+ movq %r11,16(%rsi)
+ dest
+ movq %rdx,24(%rsi)
+
+ dest
+ movq %r10,32(%rsi)
+ dest
+ movq %rbp,40(%rsi)
+ dest
+ movq %r14,48(%rsi)
+ dest
+ movq %r13,56(%rsi)
+
+3:
+
+ leaq 64(%rdi),%rdi
+ leaq 64(%rsi),%rsi
+
+ jnz .Lloop
+
+ adcq %r9,%rax
+
+ /* do last upto 56 bytes */
+.Lhandle_tail:
+ /* ecx: count */
+ movl %ecx,%r10d
+ andl $63,%ecx
+ shrl $3,%ecx
+ jz .Lfold
+ clc
+ .p2align 4
+.Lloop_8:
+ source
+ movq (%rdi),%rbx
+ adcq %rbx,%rax
+ decl %ecx
+ dest
+ movq %rbx,(%rsi)
+ leaq 8(%rsi),%rsi /* preserve carry */
+ leaq 8(%rdi),%rdi
+ jnz .Lloop_8
+ adcq %r9,%rax /* add in carry */
+
+.Lfold:
+ /* reduce checksum to 32bits */
+ movl %eax,%ebx
+ shrq $32,%rax
+ addl %ebx,%eax
+ adcl %r9d,%eax
+
+ /* do last upto 6 bytes */
+.Lhandle_7:
+ movl %r10d,%ecx
+ andl $7,%ecx
+ shrl $1,%ecx
+ jz .Lhandle_1
+ movl $2,%edx
+ xorl %ebx,%ebx
+ clc
+ .p2align 4
+.Lloop_1:
+ source
+ movw (%rdi),%bx
+ adcl %ebx,%eax
+ decl %ecx
+ dest
+ movw %bx,(%rsi)
+ leaq 2(%rdi),%rdi
+ leaq 2(%rsi),%rsi
+ jnz .Lloop_1
+ adcl %r9d,%eax /* add in carry */
+
+ /* handle last odd byte */
+.Lhandle_1:
+ testl $1,%r10d
+ jz .Lende
+ xorl %ebx,%ebx
+ source
+ movb (%rdi),%bl
+ dest
+ movb %bl,(%rsi)
+ addl %ebx,%eax
+ adcl %r9d,%eax /* carry */
+
+ CFI_REMEMBER_STATE
+.Lende:
+ movq 2*8(%rsp),%rbx
+ CFI_RESTORE rbx
+ movq 3*8(%rsp),%r12
+ CFI_RESTORE r12
+ movq 4*8(%rsp),%r14
+ CFI_RESTORE r14
+ movq 5*8(%rsp),%r13
+ CFI_RESTORE r13
+ movq 6*8(%rsp),%rbp
+ CFI_RESTORE rbp
+ addq $7*8,%rsp
+ CFI_ADJUST_CFA_OFFSET -7*8
+ ret
+ CFI_RESTORE_STATE
+
+ /* Exception handlers. Very simple, zeroing is done in the wrappers */
+.Lbad_source:
+ movq (%rsp),%rax
+ testq %rax,%rax
+ jz .Lende
+ movl $-EFAULT,(%rax)
+ jmp .Lende
+
+.Lbad_dest:
+ movq 8(%rsp),%rax
+ testq %rax,%rax
+ jz .Lende
+ movl $-EFAULT,(%rax)
+ jmp .Lende
+ CFI_ENDPROC
+ENDPROC(csum_partial_copy_generic)
projects / mirror_ubuntu-kernels.git / commitdiff