Fix several issues in BaseCryptLib:

[mirror_edk2.git] / CryptoPkg / Library / BaseCryptLib / Pk / CryptRsaExt.c

diff --git a/CryptoPkg/Library/BaseCryptLib/Pk/CryptRsaExt.c b/CryptoPkg/Library/BaseCryptLib/Pk/CryptRsaExt.c
index 67d8826e6c4a706d9db930dc2aeb86f4ebbd8b5b..b4faafa0c350e7db61442efc85d6e58c9d7d616b 100644 (file)
--- a/CryptoPkg/Library/BaseCryptLib/Pk/CryptRsaExt.c
+++ b/CryptoPkg/Library/BaseCryptLib/Pk/CryptRsaExt.c
@@ -231,22 +231,32 @@ RsaGenerateKey (
   //\r
   // Check input parameters.\r
   //\r
-  if (RsaContext == NULL) {\r
+  if (RsaContext == NULL || ModulusLength > INT_MAX || PublicExponentSize > INT_MAX) {\r
     return FALSE;\r
   }\r
   \r
   KeyE = BN_new ();\r
+  if (KeyE == NULL) {\r
+    return FALSE;\r
+  }\r
+\r
+  RetVal = FALSE;\r
+  \r
   if (PublicExponent == NULL) {\r
-    BN_set_word (KeyE, 0x10001);\r
+    if (BN_set_word (KeyE, 0x10001) == 0) {\r
+      goto _Exit;\r
+    }\r
   } else {\r
-    BN_bin2bn (PublicExponent, (UINT32) PublicExponentSize, KeyE);\r
+    if (BN_bin2bn (PublicExponent, (UINT32) PublicExponentSize, KeyE) == NULL) {\r
+      goto _Exit;\r
+    }\r
   }\r
 \r
-  RetVal = FALSE;\r
   if (RSA_generate_key_ex ((RSA *) RsaContext, (UINT32) ModulusLength, KeyE, NULL) == 1) {\r
    RetVal = TRUE;\r
   }\r
 \r
+_Exit:\r
   BN_free (KeyE);\r
   return RetVal;\r
 }\r
@@ -299,18 +309,24 @@ RsaCheckKey (
 /**\r
   Performs the PKCS1-v1_5 encoding methods defined in RSA PKCS #1.\r
 \r
-  @param  Message      Message buffer to be encoded.\r
-  @param  MessageSize  Size of message buffer in bytes.\r
-  @param  DigestInfo   Pointer to buffer of digest info for output.\r
+  @param[in]     Message        Message buffer to be encoded.\r
+  @param[in]     MessageSize    Size of message buffer in bytes.\r
+  @param[out]    DigestInfo     Pointer to buffer of digest info for output.\r
+  @param[in,out] DigestInfoSize On input, the size of DigestInfo buffer in bytes.\r
+                                On output, the size of data returned in DigestInfo\r
+                                buffer in bytes.\r
 \r
-  @return  Size of DigestInfo in bytes.\r
+  @retval TRUE   PKCS1-v1_5 encoding finished successfully.\r
+  @retval FALSE  Any input parameter is invalid.\r
+  @retval FALSE  DigestInfo buffer is not large enough.\r
 \r
 **/  \r
-UINTN\r
+BOOLEAN\r
 DigestInfoEncoding (\r
-  IN   CONST UINT8  *Message,\r
-  IN   UINTN        MessageSize,\r
-  OUT  UINT8        *DigestInfo\r
+  IN CONST UINT8  *Message,\r
+  IN       UINTN  MessageSize,\r
+  OUT      UINT8  *DigestInfo,\r
+  IN OUT   UINTN  *DigestInfoSize\r
   )\r
 {\r
   CONST UINT8  *HashDer;\r
@@ -319,7 +335,7 @@ DigestInfoEncoding (
   //\r
   // Check input parameters.\r
   //\r
-  if (Message == NULL || DigestInfo == NULL) {\r
+  if (Message == NULL || DigestInfo == NULL || DigestInfoSize == NULL) {\r
     return FALSE;\r
   }\r
 \r
@@ -347,10 +363,16 @@ DigestInfoEncoding (
     return FALSE;\r
   }\r
 \r
+  if (*DigestInfoSize < DerSize + MessageSize) {\r
+    *DigestInfoSize = DerSize + MessageSize;\r
+    return FALSE;\r
+  }\r
+\r
   CopyMem (DigestInfo, HashDer, DerSize);\r
   CopyMem (DigestInfo + DerSize, Message, MessageSize);\r
 \r
-  return (DerSize + MessageSize);\r
+  *DigestInfoSize = DerSize + MessageSize;\r
+  return TRUE;\r
 }\r
 \r
 /**\r
@@ -412,21 +434,23 @@ RsaPkcs1Sign (
     return FALSE;\r
   }\r
 \r
-  Size = DigestInfoEncoding (MessageHash, HashSize, Signature);\r
+  if (!DigestInfoEncoding (MessageHash, HashSize, Signature, SigSize)) {\r
+    return FALSE;\r
+  }\r
 \r
   ReturnVal = RSA_private_encrypt (\r
-                (UINT32) Size,\r
+                (UINT32) *SigSize,\r
                 Signature,\r
                 Signature,\r
                 Rsa,\r
                 RSA_PKCS1_PADDING\r
                 );\r
 \r
-  if (ReturnVal < (INTN) Size) {\r
+  if (ReturnVal < (INTN) *SigSize) {\r
     return FALSE;\r
   }\r
 \r
-  *SigSize = (UINTN)ReturnVal;\r
+  *SigSize = (UINTN) ReturnVal;\r
   return TRUE;\r
 }\r
 \r