make DMU_OT_IS_METADATA and DMU_OT_IS_ENCRYPTED return B_TRUE or B_FALSE

Without this patch, the

    ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf));

at the beginning of dbuf_assign_arcbuf can panic
if the object type is a DMU_OT_NEWTYPE that has
DMU_OT_METADATA set.

While we're at it, fix DMU_OT_IS_ENCRYPTED as well.

Reviewed-by: Richard Yao <richard.yao@alumni.stonybrook.edu>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Signed-off-by: Christian Schwarz <christian.schwarz@nutanix.com>
Closes #13842

diff --git a/include/sys/dmu.h b/include/sys/dmu.h
index 5a3d7d6a50559893f2fdbc63ad784894a580449b..0a4827e5ec3ed2c488e8c36f4f1cdcc6dfb1e018 100644 (file)
--- a/include/sys/dmu.h
+++ b/include/sys/dmu.h
@@ -136,7 +136,7 @@ typedef enum dmu_object_byteswap {
 #endif
 
 #define        DMU_OT_IS_METADATA(ot) (((ot) & DMU_OT_NEWTYPE) ? \
-       ((ot) & DMU_OT_METADATA) : \
+       (((ot) & DMU_OT_METADATA) != 0) : \
        DMU_OT_IS_METADATA_IMPL(ot))
 
 #define        DMU_OT_IS_DDT(ot) \
@@ -147,7 +147,7 @@ typedef enum dmu_object_byteswap {
        ((ot) == DMU_OT_PLAIN_FILE_CONTENTS || (ot) == DMU_OT_UINT64_OTHER)
 
 #define        DMU_OT_IS_ENCRYPTED(ot) (((ot) & DMU_OT_NEWTYPE) ? \
-       ((ot) & DMU_OT_ENCRYPTED) : \
+       (((ot) & DMU_OT_ENCRYPTED) != 0) : \
        DMU_OT_IS_ENCRYPTED_IMPL(ot))
 
 /*