BaseMemoryLibSse2: Take advantage of write combining buffers

The current SSE2 implementation of the ZeroMem(), SetMem(),
SetMem16(), SetMem32 and SetMem64 functions is writing 16 bytes per 16
bytes. It hurts the performances so bad that this is even slower than
a simple 'rep stos' (4% slower) in regular DRAM.

To take full advantages of the 'movntdq' instruction it is better to
"queue" a total of 64 bytes in the write combining buffers.  This
patch implement such a change.  Below is a table where I measured
(with 'rdtsc') the time to write an entire 100MB RAM buffer. These
functions operate almost two times faster.

| Function | Arch | Untouched | 64 bytes | Result |
|----------+------+-----------+----------+--------|
| ZeroMem  | Ia32 |  17765947 |  9136062 | 1.945x |
| ZeroMem  | X64  |  17525170 |  9233391 | 1.898x |
| SetMem   | Ia32 |  17522291 |  9137272 | 1.918x |
| SetMem   | X64  |  17949261 |  9176978 | 1.956x |
| SetMem16 | Ia32 |  18219673 |  9372062 | 1.944x |
| SetMem16 | X64  |  17523331 |  9275184 | 1.889x |
| SetMem32 | Ia32 |  18495036 |  9273053 | 1.994x |
| SetMem32 | X64  |  17368864 |  9285885 | 1.870x |
| SetMem64 | Ia32 |  18564473 |  9241362 | 2.009x |
| SetMem64 | X64  |  17506951 |  9280148 | 1.886x |

Signed-off-by: Jeremy Compostella <jeremy.compostella@intel.com>
Reviewed-by: Liming Gao <gaoliming@byosoft.com.cn>

diff --git a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem.nasm b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem.nasm
index 24313cb4b3d2c15bc36ce5a0f90995a930012d32..a8744300c6e80bf215f08a84c03c320aa6054b17 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem.nasm
@@ -34,7 +34,7 @@ ASM_PFX(InternalMemSetMem):
     mov     al, [esp + 16]              ; al <- Value\r
     xor     ecx, ecx\r
     sub     ecx, edi\r
-    and     ecx, 15                     ; ecx + edi aligns on 16-byte boundary\r
+    and     ecx, 63                     ; ecx + edi aligns on 16-byte boundary\r
     jz      .0\r
     cmp     ecx, edx\r
     cmova   ecx, edx\r
@@ -42,8 +42,8 @@ ASM_PFX(InternalMemSetMem):
     rep     stosb\r
 .0:\r
     mov     ecx, edx\r
-    and     edx, 15\r
-    shr     ecx, 4                      ; ecx <- # of DQwords to set\r
+    and     edx, 63\r
+    shr     ecx, 6                      ; ecx <- # of DQwords to set\r
     jz      @SetBytes\r
     mov     ah, al                      ; ax <- Value | (Value << 8)\r
     add     esp, -16\r
@@ -53,7 +53,10 @@ ASM_PFX(InternalMemSetMem):
     movlhps xmm0, xmm0                  ; xmm0 <- Value repeats 16 times\r
 .1:\r
     movntdq [edi], xmm0                 ; edi should be 16-byte aligned\r
-    add     edi, 16\r
+    movntdq [edi + 16], xmm0\r
+    movntdq [edi + 32], xmm0\r
+    movntdq [edi + 48], xmm0\r
+    add     edi, 64\r
     loop    .1\r
     mfence\r
     movdqu  xmm0, [esp]                 ; restore xmm0\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem16.nasm b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem16.nasm
index 6e308b559405d658542b3cd331064ae59ba1db94..d461ee086c195f118b2fe89ac2ce98e807abf986 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem16.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem16.nasm
@@ -33,7 +33,7 @@ ASM_PFX(InternalMemSetMem16):
     mov     edi, [esp + 8]\r
     xor     ecx, ecx\r
     sub     ecx, edi\r
-    and     ecx, 15                     ; ecx + edi aligns on 16-byte boundary\r
+    and     ecx, 63                     ; ecx + edi aligns on 16-byte boundary\r
     mov     eax, [esp + 16]\r
     jz      .0\r
     shr     ecx, 1\r
@@ -43,15 +43,18 @@ ASM_PFX(InternalMemSetMem16):
     rep     stosw\r
 .0:\r
     mov     ecx, edx\r
-    and     edx, 7\r
-    shr     ecx, 3\r
+    and     edx, 31\r
+    shr     ecx, 5\r
     jz      @SetWords\r
     movd    xmm0, eax\r
     pshuflw xmm0, xmm0, 0\r
     movlhps xmm0, xmm0\r
 .1:\r
     movntdq [edi], xmm0                 ; edi should be 16-byte aligned\r
-    add     edi, 16\r
+    movntdq [edi + 16], xmm0\r
+    movntdq [edi + 32], xmm0\r
+    movntdq [edi + 48], xmm0\r
+    add     edi, 64\r
     loop    .1\r
     mfence\r
 @SetWords:\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem32.nasm b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem32.nasm
index 2cfc8cb0ddb24361efc27642d51c012baaad2fbe..3ffdcd07d71898d7cc6a3b3f899ce08769bcdcce 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem32.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem32.nasm
@@ -43,14 +43,17 @@ ASM_PFX(InternalMemSetMem32):
     rep     stosd\r
 .0:\r
     mov     ecx, edx\r
-    and     edx, 3\r
-    shr     ecx, 2\r
+    and     edx, 15\r
+    shr     ecx, 4\r
     jz      @SetDwords\r
     movd    xmm0, eax\r
     pshufd  xmm0, xmm0, 0\r
 .1:\r
     movntdq [edi], xmm0\r
-    add     edi, 16\r
+    movntdq [edi + 16], xmm0\r
+    movntdq [edi + 32], xmm0\r
+    movntdq [edi + 48], xmm0\r
+    add     edi, 64\r
     loop    .1\r
     mfence\r
 @SetDwords:\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem64.nasm b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem64.nasm
index e153495a687426eadcd517d4695eb642188a2714..cd000648ae7e4ca22b29f68e731684940dea96f9 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem64.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/Ia32/SetMem64.nasm
@@ -38,17 +38,29 @@ ASM_PFX(InternalMemSetMem64):
     add     edx, 8\r
     dec     ecx\r
 .0:\r
-    shr     ecx, 1\r
+    push    ebx\r
+    mov     ebx, ecx\r
+    and     ebx, 7\r
+    shr     ecx, 3\r
     jz      @SetQwords\r
     movlhps xmm0, xmm0\r
 .1:\r
     movntdq [edx], xmm0\r
-    lea     edx, [edx + 16]\r
+    movntdq [edx + 16], xmm0\r
+    movntdq [edx + 32], xmm0\r
+    movntdq [edx + 48], xmm0\r
+    lea     edx, [edx + 64]\r
     loop    .1\r
     mfence\r
 @SetQwords:\r
-    jnc     .2\r
+    test    ebx, ebx\r
+    jz .3\r
+    mov     ecx, ebx\r
+.2\r
     movq    qword [edx], xmm0\r
-.2:\r
+    lea     edx, [edx + 8]\r
+    loop    .2\r
+.3:\r
+    pop ebx\r
     ret\r
 \r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/Ia32/ZeroMem.nasm b/MdePkg/Library/BaseMemoryLibSse2/Ia32/ZeroMem.nasm
index cd34006f596a5d317eb41d327a5f5b143de9d6d7..0e0828551b9790095f8824afbbad13bcae47b685 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/Ia32/ZeroMem.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/Ia32/ZeroMem.nasm
@@ -33,7 +33,7 @@ ASM_PFX(InternalMemZeroMem):
     xor     ecx, ecx\r
     sub     ecx, edi\r
     xor     eax, eax\r
-    and     ecx, 15\r
+    and     ecx, 63\r
     jz      .0\r
     cmp     ecx, edx\r
     cmova   ecx, edx\r
@@ -41,13 +41,16 @@ ASM_PFX(InternalMemZeroMem):
     rep     stosb\r
 .0:\r
     mov     ecx, edx\r
-    and     edx, 15\r
-    shr     ecx, 4\r
+    and     edx, 63\r
+    shr     ecx, 6\r
     jz      @ZeroBytes\r
     pxor    xmm0, xmm0\r
 .1:\r
     movntdq [edi], xmm0\r
-    add     edi, 16\r
+    movntdq [edi + 16], xmm0\r
+    movntdq [edi + 32], xmm0\r
+    movntdq [edi + 48], xmm0\r
+    add     edi, 64\r
     loop    .1\r
     mfence\r
 @ZeroBytes:\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem.nasm b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem.nasm
index 5bd1c2262d3059386f76de5ba9584051f947a6ae..28b11ee586ba5cc95314ef682cda11af0e520931 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem.nasm
@@ -42,8 +42,8 @@ ASM_PFX(InternalMemSetMem):
     rep     stosb\r
 .0:\r
     mov     rcx, rdx\r
-    and     rdx, 15\r
-    shr     rcx, 4\r
+    and     rdx, 63\r
+    shr     rcx, 6\r
     jz      @SetBytes\r
     mov     ah, al                      ; ax <- Value repeats twice\r
     movdqa  [rsp + 0x10], xmm0           ; save xmm0\r
@@ -52,7 +52,10 @@ ASM_PFX(InternalMemSetMem):
     movlhps xmm0, xmm0                  ; xmm0 <- Value repeats 16 times\r
 .1:\r
     movntdq [rdi], xmm0                 ; rdi should be 16-byte aligned\r
-    add     rdi, 16\r
+    movntdq [rdi + 16], xmm0\r
+    movntdq [rdi + 32], xmm0\r
+    movntdq [rdi + 48], xmm0\r
+    add     rdi, 64\r
     loop    .1\r
     mfence\r
     movdqa  xmm0, [rsp + 0x10]           ; restore xmm0\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem16.nasm b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem16.nasm
index 90d159820aa161aae8039d7190a67e07a47a5d70..375be19313319a4e2ff693da472385a3947513d2 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem16.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem16.nasm
@@ -33,7 +33,7 @@ ASM_PFX(InternalMemSetMem16):
     mov     r9, rdi\r
     xor     rcx, rcx\r
     sub     rcx, rdi\r
-    and     rcx, 15\r
+    and     rcx, 63\r
     mov     rax, r8\r
     jz      .0\r
     shr     rcx, 1\r
@@ -43,15 +43,18 @@ ASM_PFX(InternalMemSetMem16):
     rep     stosw\r
 .0:\r
     mov     rcx, rdx\r
-    and     edx, 7\r
-    shr     rcx, 3\r
+    and     edx, 31\r
+    shr     rcx, 5\r
     jz      @SetWords\r
     movd    xmm0, eax\r
     pshuflw xmm0, xmm0, 0\r
     movlhps xmm0, xmm0\r
 .1:\r
     movntdq [rdi], xmm0\r
-    add     rdi, 16\r
+    movntdq [rdi + 16], xmm0\r
+    movntdq [rdi + 32], xmm0\r
+    movntdq [rdi + 48], xmm0\r
+    add     rdi, 64\r
     loop    .1\r
     mfence\r
 @SetWords:\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem32.nasm b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem32.nasm
index 928e086889c567c8cd5d99d1bb97b1eb35f7795c..5d12beaa9a23ac44e5b0c76c9a51ca12c7afa729 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem32.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem32.nasm
@@ -43,14 +43,17 @@ ASM_PFX(InternalMemSetMem32):
     rep     stosd\r
 .0:\r
     mov     rcx, rdx\r
-    and     edx, 3\r
-    shr     rcx, 2\r
+    and     edx, 15\r
+    shr     rcx, 4\r
     jz      @SetDwords\r
     movd    xmm0, eax\r
     pshufd  xmm0, xmm0, 0\r
 .1:\r
     movntdq [rdi], xmm0\r
-    add     rdi, 16\r
+    movntdq [rdi + 16], xmm0\r
+    movntdq [rdi + 32], xmm0\r
+    movntdq [rdi + 48], xmm0\r
+    add     rdi, 64\r
     loop    .1\r
     mfence\r
 @SetDwords:\r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem64.nasm b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem64.nasm
index d77181054265609b6914538d30fcb34da358edce..485f74ddac451dff5989b6aa93ddd1012d3648be 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem64.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/X64/SetMem64.nasm
@@ -37,17 +37,28 @@ ASM_PFX(InternalMemSetMem64):
     add     rdx, 8\r
     dec     rcx\r
 .0:\r
-    shr     rcx, 1\r
+    push    rbx\r
+    mov     rbx, rcx\r
+    and     rbx, 7\r
+    shr     rcx, 3\r
     jz      @SetQwords\r
     movlhps xmm0, xmm0\r
 .1:\r
     movntdq [rdx], xmm0\r
-    lea     rdx, [rdx + 16]\r
+    movntdq [rdx + 16], xmm0\r
+    movntdq [rdx + 32], xmm0\r
+    movntdq [rdx + 48], xmm0\r
+    lea     rdx, [rdx + 64]\r
     loop    .1\r
     mfence\r
 @SetQwords:\r
-    jnc     .2\r
-    mov     [rdx], r8\r
+    push    rdi\r
+    mov     rcx, rbx\r
+    mov     rax, r8\r
+    mov     rdi, rdx\r
+    rep     stosq\r
+    pop     rdi\r
 .2:\r
+    pop rbx\r
     ret\r
 \r

diff --git a/MdePkg/Library/BaseMemoryLibSse2/X64/ZeroMem.nasm b/MdePkg/Library/BaseMemoryLibSse2/X64/ZeroMem.nasm
index 5ddcae9ca52affc3ee18497249944c8eb7424909..21f504e3b70892f7eebbbfd25daed5fd5b0dc787 100644 (file)
--- a/MdePkg/Library/BaseMemoryLibSse2/X64/ZeroMem.nasm
+++ b/MdePkg/Library/BaseMemoryLibSse2/X64/ZeroMem.nasm
@@ -32,7 +32,7 @@ ASM_PFX(InternalMemZeroMem):
     xor     rcx, rcx\r
     xor     eax, eax\r
     sub     rcx, rdi\r
-    and     rcx, 15\r
+    and     rcx, 63\r
     mov     r8, rdi\r
     jz      .0\r
     cmp     rcx, rdx\r
@@ -41,13 +41,16 @@ ASM_PFX(InternalMemZeroMem):
     rep     stosb\r
 .0:\r
     mov     rcx, rdx\r
-    and     edx, 15\r
-    shr     rcx, 4\r
+    and     edx, 63\r
+    shr     rcx, 6\r
     jz      @ZeroBytes\r
     pxor    xmm0, xmm0\r
 .1:\r
-    movntdq [rdi], xmm0                 ; rdi should be 16-byte aligned\r
-    add     rdi, 16\r
+    movntdq [rdi], xmm0\r
+    movntdq [rdi + 16], xmm0\r
+    movntdq [rdi + 32], xmm0\r
+    movntdq [rdi + 48], xmm0\r
+    add     rdi, 64\r
     loop    .1\r
     mfence\r
 @ZeroBytes:\r