arch/x86/mm/mem_encrypt_boot.S

   1 /*
   2  * AMD Memory Encryption Support
   3  *
   4  * Copyright (C) 2016 Advanced Micro Devices, Inc.
   5  *
   6  * Author: Tom Lendacky <thomas.lendacky@amd.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as
  10  * published by the Free Software Foundation.
  11  */
  12
  13 #include <linux/linkage.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/page.h>
  16 #include <asm/processor-flags.h>
  17 #include <asm/msr-index.h>
  18
  19         .text
  20         .code64
  21 ENTRY(sme_encrypt_execute)
  22
  23         /*
  24          * Entry parameters:
  25          *   RDI - virtual address for the encrypted kernel mapping
  26          *   RSI - virtual address for the decrypted kernel mapping
  27          *   RDX - length of kernel
  28          *   RCX - virtual address of the encryption workarea, including:
  29          *     - stack page (PAGE_SIZE)
  30          *     - encryption routine page (PAGE_SIZE)
  31          *     - intermediate copy buffer (PMD_PAGE_SIZE)
  32          *    R8 - physcial address of the pagetables to use for encryption
  33          */
  34
  35         push    %rbp
  36         movq    %rsp, %rbp              /* RBP now has original stack pointer */
  37
  38         /* Set up a one page stack in the non-encrypted memory area */
  39         movq    %rcx, %rax              /* Workarea stack page */
  40         leaq    PAGE_SIZE(%rax), %rsp   /* Set new stack pointer */
  41         addq    $PAGE_SIZE, %rax        /* Workarea encryption routine */
  42
  43         push    %r12
  44         movq    %rdi, %r10              /* Encrypted kernel */
  45         movq    %rsi, %r11              /* Decrypted kernel */
  46         movq    %rdx, %r12              /* Kernel length */
  47
  48         /* Copy encryption routine into the workarea */
  49         movq    %rax, %rdi                              /* Workarea encryption routine */
  50         leaq    __enc_copy(%rip), %rsi                  /* Encryption routine */
  51         movq    $(.L__enc_copy_end - __enc_copy), %rcx  /* Encryption routine length */
  52         rep     movsb
  53
  54         /* Setup registers for call */
  55         movq    %r10, %rdi              /* Encrypted kernel */
  56         movq    %r11, %rsi              /* Decrypted kernel */
  57         movq    %r8, %rdx               /* Pagetables used for encryption */
  58         movq    %r12, %rcx              /* Kernel length */
  59         movq    %rax, %r8               /* Workarea encryption routine */
  60         addq    $PAGE_SIZE, %r8         /* Workarea intermediate copy buffer */
  61
  62         call    *%rax                   /* Call the encryption routine */
  63
  64         pop     %r12
  65
  66         movq    %rbp, %rsp              /* Restore original stack pointer */
  67         pop     %rbp
  68
  69         ret
  70 ENDPROC(sme_encrypt_execute)
  71
  72 ENTRY(__enc_copy)
  73 /*
  74  * Routine used to encrypt kernel.
  75  *   This routine must be run outside of the kernel proper since
  76  *   the kernel will be encrypted during the process. So this
  77  *   routine is defined here and then copied to an area outside
  78  *   of the kernel where it will remain and run decrypted
  79  *   during execution.
  80  *
  81  *   On entry the registers must be:
  82  *     RDI - virtual address for the encrypted kernel mapping
  83  *     RSI - virtual address for the decrypted kernel mapping
  84  *     RDX - address of the pagetables to use for encryption
  85  *     RCX - length of kernel
  86  *      R8 - intermediate copy buffer
  87  *
  88  *     RAX - points to this routine
  89  *
  90  * The kernel will be encrypted by copying from the non-encrypted
  91  * kernel space to an intermediate buffer and then copying from the
  92  * intermediate buffer back to the encrypted kernel space. The physical
  93  * addresses of the two kernel space mappings are the same which
  94  * results in the kernel being encrypted "in place".
  95  */
  96         /* Enable the new page tables */
  97         mov     %rdx, %cr3
  98
  99         /* Flush any global TLBs */
 100         mov     %cr4, %rdx
 101         andq    $~X86_CR4_PGE, %rdx
 102         mov     %rdx, %cr4
 103         orq     $X86_CR4_PGE, %rdx
 104         mov     %rdx, %cr4
 105
 106         /* Set the PAT register PA5 entry to write-protect */
 107         push    %rcx
 108         movl    $MSR_IA32_CR_PAT, %ecx
 109         rdmsr
 110         push    %rdx                    /* Save original PAT value */
 111         andl    $0xffff00ff, %edx       /* Clear PA5 */
 112         orl     $0x00000500, %edx       /* Set PA5 to WP */
 113         wrmsr
 114         pop     %rdx                    /* RDX contains original PAT value */
 115         pop     %rcx
 116
 117         movq    %rcx, %r9               /* Save kernel length */
 118         movq    %rdi, %r10              /* Save encrypted kernel address */
 119         movq    %rsi, %r11              /* Save decrypted kernel address */
 120
 121         wbinvd                          /* Invalidate any cache entries */
 122
 123         /* Copy/encrypt 2MB at a time */
 124 1:
 125         movq    %r11, %rsi              /* Source - decrypted kernel */
 126         movq    %r8, %rdi               /* Dest   - intermediate copy buffer */
 127         movq    $PMD_PAGE_SIZE, %rcx    /* 2MB length */
 128         rep     movsb
 129
 130         movq    %r8, %rsi               /* Source - intermediate copy buffer */
 131         movq    %r10, %rdi              /* Dest   - encrypted kernel */
 132         movq    $PMD_PAGE_SIZE, %rcx    /* 2MB length */
 133         rep     movsb
 134
 135         addq    $PMD_PAGE_SIZE, %r11
 136         addq    $PMD_PAGE_SIZE, %r10
 137         subq    $PMD_PAGE_SIZE, %r9     /* Kernel length decrement */
 138         jnz     1b                      /* Kernel length not zero? */
 139
 140         /* Restore PAT register */
 141         push    %rdx                    /* Save original PAT value */
 142         movl    $MSR_IA32_CR_PAT, %ecx
 143         rdmsr
 144         pop     %rdx                    /* Restore original PAT value */
 145         wrmsr
 146
 147         ret
 148 .L__enc_copy_end:
 149 ENDPROC(__enc_copy)