Make calloc set ENOMEM when failing due to overflow.

This fixes a bug in upstream dlmalloc, where it doesn't set errno to
ENOMEM in overflow or footprint overrun cases.

diff --git a/dlmalloc/src/malloc.c b/dlmalloc/src/malloc.c
index 490d898da8b78f178d75f751b930a73f2963d04f..29ce55698a1e641ed05a8741e71e26ad704b85b7 100644 (file)
--- a/dlmalloc/src/malloc.c
+++ b/dlmalloc/src/malloc.c
@@ -4055,12 +4055,26 @@ static void* sys_alloc(mstate m, size_t nb) {
   }
 
   asize = granularity_align(nb + SYS_ALLOC_PADDING);
+#ifdef __wasilibc_unmodified_upstream // Bug fix: set ENOMEM on size overflow
   if (asize <= nb)
     return 0; /* wraparound */
+#else
+  if (asize <= nb) {
+    MALLOC_FAILURE_ACTION;
+    return 0; /* wraparound */
+  }
+#endif
   if (m->footprint_limit != 0) {
     size_t fp = m->footprint + asize;
+#ifdef __wasilibc_unmodified_upstream // Bug fix: set ENOMEM on footprint overrun
     if (fp <= m->footprint || fp > m->footprint_limit)
       return 0;
+#else
+    if (fp <= m->footprint || fp > m->footprint_limit) {
+      MALLOC_FAILURE_ACTION;
+      return 0;
+    }
+#endif
   }
 
   /*