Add BLAKE3 support to OpenZFS

Add another checksum to OpenZFS that has similar performance to
Edon-R, but without the caveats around the latter.

Add micro benchmarking for all advanced hashing functions like it's
done for fletcher-4.

Linux statistics are here: '/proc/spl/kstat/zfs/chksum_bench'

'''
20 0 0x01 -1 0 865606240142 1529555928674
implementation   1k     4k     16k    64k    256k   1m     4m
fletcher-4       4772   13872  25625  32300  34418  33477  20507
edonr-generic    1240   1600   1750   1749   1741   1751   1750
skein-generic    548    600    612    618    618    611    615
sha256-generic   284    309    318    319    320    294    320
sha512-generic   404    461    481    485    484    484    485
blake3-generic   295    302    302    302    298    298    298
blake3-sse2      356    1272   1434   1470   1469   1471   1436
blake3-sse41     425    1368   1626   1694   1703   1676   1651
blake3-avx2      423    1895   3112   3346   3391   3372   3189
blake3-avx512    457    2668   5029   5833   6013   5891   5504
'''

FreeBSD statistics via 'sysctl -n kstat.zfs.misc.chksum_bench'

'''
implementation   1k     4k     16k    64k    256k   1m     4m
fletcher-4       2995   8686   18052  25044  27950  28649  17091
edonr-generic    1273   1609   1693   1704   1738   1714   1729
skein-generic    219    233    235    236    236    235    236
sha256-generic   259    275    283    281    283    283    282
sha512-generic   380    419    437    445    445    445    444
blake3-generic   518    514    509    516    519    515    508
blake3-sse2      272    1203   1298   1320   1345   1329   1311
...
'''

The units are MiB/s for each method.

TODO
Split up the patch into into patchset with multiple parts.

Signed-off-by: Rich Ercolani <[email protected]>
Signed-off-by: Tino Reichardt <[email protected]>

Author: mcmilk

AUTHORS

@@ -285,6 +285,7 @@ CONTRIBUTORS:
     Tim Connors <[email protected]>
     Tim Crawford <[email protected]>
     Tim Haley <[email protected]>
+    Tino Reichardt <[email protected]>
     Tobin Harding <[email protected]>
     Tom Caputi <[email protected]>
     Tom Matthews <[email protected]>

include/os/freebsd/spl/sys/ccompile.h

@@ -77,10 +77,12 @@ extern "C" {
 
 #ifndef LOCORE
 #ifndef HAVE_RPC_TYPES
+#ifndef _KERNEL
 typedef int bool_t;
 typedef int enum_t;
 #endif
 #endif
+#endif
 
 #ifndef __cplusplus
 #define	__init

include/sys/Makefile.am

@@ -9,6 +9,7 @@ COMMON_H = \
 	avl.h \
 	avl_impl.h \
 	bitops.h \
+	blake3.h \
 	blkptr.h \
 	bplist.h \
 	bpobj.h \
@@ -104,6 +105,7 @@ COMMON_H = \
 	zfs_acl.h \
 	zfs_bootenv.h \
 	zfs_context.h \
+	zfs_chksum.h \
 	zfs_debug.h \
 	zfs_delay.h \
 	zfs_file.h \

include/sys/blake3.h

@@ -0,0 +1,114 @@
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License, Version 1.0 only
+ * (the "License").  You may not use this file except in compliance
+ * with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Based on BLAKE3 v1.2.0, https://github.com/BLAKE3-team/BLAKE3
+ * Copyright (c) 2019-2020 Samuel Neves and Jack O'Connor
+ * Copyright (c) 2021 Tino Reichardt <[email protected]>
+ */
+
+#ifndef BLAKE3_H
+#define	BLAKE3_H
+
+#ifdef  _KERNEL
+#include <sys/types.h>
+#else
+#include <stdint.h>
+#include <stdlib.h>
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BLAKE3_KEY_LEN		32
+#define	BLAKE3_OUT_LEN		32
+#define	BLAKE3_MAX_DEPTH	54
+#define	BLAKE3_BLOCK_LEN	64
+#define	BLAKE3_CHUNK_LEN	1024
+
+/*
+ * This struct is a private implementation detail.
+ * It has to be here because it's part of BLAKE3_CTX below.
+ */
+typedef struct {
+	uint32_t cv[8];
+	uint64_t chunk_counter;
+	uint8_t buf[BLAKE3_BLOCK_LEN];
+	uint8_t buf_len;
+	uint8_t blocks_compressed;
+	uint8_t flags;
+} blake3_chunk_state_t;
+
+typedef struct {
+	uint32_t key[8];
+	blake3_chunk_state_t chunk;
+	uint8_t cv_stack_len;
+
+	/*
+	 * The stack size is MAX_DEPTH + 1 because we do lazy merging. For
+	 * example, with 7 chunks, we have 3 entries in the stack. Adding an
+	 * 8th chunk requires a 4th entry, rather than merging everything down
+	 * to 1, because we don't know whether more input is coming. This is
+	 * different from how the reference implementation does things.
+	 */
+	uint8_t cv_stack[(BLAKE3_MAX_DEPTH + 1) * BLAKE3_OUT_LEN];
+} BLAKE3_CTX;
+
+/* init the context for hash operation */
+void Blake3_Init(BLAKE3_CTX *ctx);
+
+/* init the context for a MAC and/or tree hash operation */
+void Blake3_InitKeyed(BLAKE3_CTX *ctx, const uint8_t key[BLAKE3_KEY_LEN]);
+
+/* process the input bytes */
+void Blake3_Update(BLAKE3_CTX *ctx, const void *input, size_t input_len);
+
+/* finalize the hash computation and output the result */
+void Blake3_Final(const BLAKE3_CTX *ctx, uint8_t *out);
+
+/* finalize the hash computation and output the result */
+void Blake3_FinalSeek(const BLAKE3_CTX *ctx, uint64_t seek, uint8_t *out,
+    size_t out_len);
+
+/*
+ * Returns the number of supported BLAKE3 implementations
+ */
+extern int blake3_get_impl_count(void);
+
+/*
+ * Returns the name of selected BLAKE3 implementation
+ */
+extern const char *blake3_get_impl_name(void);
+
+/*
+ * Set BLAKE3 implementation via id or name
+ */
+extern void blake3_set_impl_id(int id);
+extern void blake3_set_impl_name(char *name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif	/* BLAKE3_H */

include/sys/zfs_chksum.h

@@ -0,0 +1,49 @@
+
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2021 Tino Reichardt <[email protected]>
+ */
+
+#ifndef	_ZFS_CHKSUM_H
+#define	_ZFS_CHKSUM_H
+
+#ifdef  _KERNEL
+#include <sys/types.h>
+#else
+#include <stdint.h>
+#include <stdlib.h>
+#endif
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+/* Benchmark the chksums of ZFS when the module is loading */
+void chksum_init(void);
+void chksum_fini(void);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _ZFS_CHKSUM_H */

include/sys/zfs_ioctl.h

@@ -124,6 +124,7 @@ typedef enum drr_headertype {
  * default use of "zfs send" won't encounter the bug mentioned above.
  */
 #define	DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS (1 << 27)
+#define	DMU_BACKUP_FEATURE_BLAKE3		(1 << 28)
 
 /*
  * Mask of all supported backup features
@@ -134,7 +135,7 @@ typedef enum drr_headertype {
     DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_LARGE_DNODE | \
     DMU_BACKUP_FEATURE_RAW | DMU_BACKUP_FEATURE_HOLDS | \
     DMU_BACKUP_FEATURE_REDACTED | DMU_BACKUP_FEATURE_SWITCH_TO_LARGE_BLOCKS | \
-    DMU_BACKUP_FEATURE_ZSTD)
+    DMU_BACKUP_FEATURE_ZSTD | DMU_BACKUP_FEATURE_BLAKE3)
 
 /* Are all features in the given flag word currently supported? */
 #define	DMU_STREAM_SUPPORTED(x)	(!((x) & ~DMU_BACKUP_FEATURE_MASK))

include/sys/zio.h

@@ -89,6 +89,7 @@ enum zio_checksum {
 	ZIO_CHECKSUM_SHA512,
 	ZIO_CHECKSUM_SKEIN,
 	ZIO_CHECKSUM_EDONR,
+	ZIO_CHECKSUM_BLAKE3,
 	ZIO_CHECKSUM_FUNCTIONS
 };
 

include/sys/zio_checksum.h

@@ -21,7 +21,8 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2016 by Delphix. All rights reserved.
- * Copyright Saso Kiselkov 2013, All rights reserved.
+ * Copyright (c) 2013 Saso Kiselkov, All rights reserved.
+ * Copyright (c) 2021 Tino Reichardt <[email protected]>
  */
 
 #ifndef _SYS_ZIO_CHECKSUM_H
@@ -107,6 +108,13 @@ _SYS_ZIO_CHECKSUM_H zio_checksum_info_t
 /*
  * Checksum routines.
  */
+
+/* Fletcher 4 */
+extern zio_abd_checksum_func_t fletcher_4_abd_ops;
+extern zio_checksum_t abd_fletcher_4_native;
+extern zio_checksum_t abd_fletcher_4_byteswap;
+
+/* SHA2 */
 extern zio_checksum_t abd_checksum_SHA256;
 extern zio_checksum_t abd_checksum_SHA512_native;
 extern zio_checksum_t abd_checksum_SHA512_byteswap;
@@ -123,6 +131,12 @@ extern zio_checksum_t abd_checksum_edonr_byteswap;
 extern zio_checksum_tmpl_init_t abd_checksum_edonr_tmpl_init;
 extern zio_checksum_tmpl_free_t abd_checksum_edonr_tmpl_free;
 
+/* BLAKE3 */
+extern zio_checksum_t abd_checksum_blake3_native;
+extern zio_checksum_t abd_checksum_blake3_byteswap;
+extern zio_checksum_tmpl_init_t abd_checksum_blake3_tmpl_init;
+extern zio_checksum_tmpl_free_t abd_checksum_blake3_tmpl_free;
+
 _SYS_ZIO_CHECKSUM_H zio_abd_checksum_func_t fletcher_4_abd_ops;
 extern zio_checksum_t abd_fletcher_4_native;
 extern zio_checksum_t abd_fletcher_4_byteswap;

include/zfeature_common.h

@@ -75,6 +75,7 @@ typedef enum spa_feature {
 	SPA_FEATURE_DEVICE_REBUILD,
 	SPA_FEATURE_ZSTD_COMPRESS,
 	SPA_FEATURE_DRAID,
+	SPA_FEATURE_BLAKE3,
 	SPA_FEATURES
 } spa_feature_t;
 

lib/libicp/Makefile.am

@@ -14,16 +14,26 @@ ASM_SOURCES_C = asm-x86_64/aes/aeskey.c
 ASM_SOURCES_AS = \
 	asm-x86_64/aes/aes_amd64.S \
 	asm-x86_64/aes/aes_aesni.S \
+	asm-x86_64/blake3/blake3_avx2.S \
+	asm-x86_64/blake3/blake3_avx512.S \
+	asm-x86_64/blake3/blake3_sse2.S \
+	asm-x86_64/blake3/blake3_sse41.S \
 	asm-x86_64/modes/gcm_pclmulqdq.S \
 	asm-x86_64/modes/aesni-gcm-x86_64.S \
 	asm-x86_64/modes/ghash-x86_64.S \
 	asm-x86_64/sha1/sha1-x86_64.S \
 	asm-x86_64/sha2/sha256_impl.S \
 	asm-x86_64/sha2/sha512_impl.S
+else 
+if TARGET_CPU_AARCH64
+ASM_SOURCES_C =
+ASM_SOURCES_AS = \
+	asm-aarch64/blake3/blake3_neon.S
 else
 ASM_SOURCES_C =
 ASM_SOURCES_AS =
 endif
+endif
 
 KERNEL_C = \
 	spi/kcf_spi.c \
@@ -37,6 +47,11 @@ KERNEL_C = \
 	algs/aes/aes_impl_x86-64.c \
 	algs/aes/aes_impl.c \
 	algs/aes/aes_modes.c \
+	algs/blake3/blake3.c \
+	algs/blake3/blake3_generic.c \
+	algs/blake3/blake3_impl.c \
+	algs/blake3/blake3_neon.c \
+	algs/blake3/blake3_x86-64.c \
 	algs/edonr/edonr.c \
 	algs/modes/modes.c \
 	algs/modes/cbc.c \

lib/libzfs/libzfs.abi

@@ -599,7 +599,7 @@
     <elf-symbol name='fletcher_4_superscalar_ops' size='64' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libspl_assert_ok' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='libzfs_config_ops' size='16' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
-    <elf-symbol name='spa_feature_table' size='1904' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='spa_feature_table' size='1960' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfeature_checks_disable' size='4' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_deleg_perm_tab' size='512' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='zfs_history_event_names' size='328' type='object-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1862,8 +1862,8 @@
     </function-decl>
   </abi-instr>
   <abi-instr address-size='64' path='../../module/zcommon/zfeature_common.c' language='LANG_C99'>
-    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='15232' id='d96379d0'>
-      <subrange length='34' type-id='7359adad' id='6a6a7e00'/>
+    <array-type-def dimensions='1' type-id='83f29ca2' size-in-bits='15680' id='d96379d0'>
+      <subrange length='35' type-id='7359adad' id='6a6a7e00'/>
     </array-type-def>
     <enum-decl name='spa_feature' id='33ecb627'>
       <underlying-type type-id='9cac1fee'/>
@@ -1902,7 +1902,8 @@
       <enumerator name='SPA_FEATURE_DEVICE_REBUILD' value='31'/>
       <enumerator name='SPA_FEATURE_ZSTD_COMPRESS' value='32'/>
       <enumerator name='SPA_FEATURE_DRAID' value='33'/>
-      <enumerator name='SPA_FEATURES' value='34'/>
+      <enumerator name='SPA_FEATURE_BLAKE3' value='34'/>
+      <enumerator name='SPA_FEATURES' value='35'/>
     </enum-decl>
     <typedef-decl name='spa_feature_t' type-id='33ecb627' id='d6618c78'/>
     <enum-decl name='zfeature_flags' id='6db816a4'>

lib/libzpool/Makefile.am

@@ -54,6 +54,7 @@ KERNEL_C = \
 	aggsum.c \
 	arc.c \
 	arc_os.c \
+	blake3_zfs.c \
 	blkptr.c \
 	bplist.c \
 	bpobj.c \
@@ -160,6 +161,7 @@ KERNEL_C = \
 	zcp_synctask.c \
 	zfeature.c \
 	zfs_byteswap.c \
+	zfs_chksum.c \
 	zfs_debug.c \
 	zfs_fm.c \
 	zfs_fuid.c \

man/man7/zfsprops.7

@@ -743,7 +743,7 @@ This property is not inherited.
 .It Xo
 .Sy checksum Ns = Ns Sy on Ns | Ns Sy off Ns | Ns Sy fletcher2 Ns | Ns
 .Sy fletcher4 Ns | Ns Sy sha256 Ns | Ns Sy noparity Ns | Ns
-.Sy sha512 Ns | Ns Sy skein Ns | Ns Sy edonr
+.Sy sha512 Ns | Ns Sy skein Ns | Ns Sy edonr Ns | Ns Sy blake3
 .Xc
 Controls the checksum used to verify data integrity.
 The default value is
@@ -768,6 +768,7 @@ a recommended practice.
 The
 .Sy sha512 ,
 .Sy skein ,
+.Sy blake3 ,
 and
 .Sy edonr
 checksum algorithms require enabling the appropriate features on the pool.
@@ -981,7 +982,7 @@ mount options.
 .It Xo
 .Sy dedup Ns = Ns Sy off Ns | Ns Sy on Ns | Ns Sy verify Ns | Ns
 .Sy sha256 Ns Oo , Ns Sy verify Oc Ns | Ns Sy sha512 Ns Oo , Ns Sy verify Oc Ns | Ns Sy skein Ns Oo , Ns Sy verify Oc Ns | Ns
-.Sy edonr , Ns Sy verify
+.Sy edonr , Ns Sy verify Ns | Ns Sy blake3 Ns Oo , Ns Sy verify Oc Ns
 .Xc
 Configures deduplication for a dataset.
 The default value is