ossl: Add support for armv7

OpenSSL provides implementations of several AES modes which use
bitslicing and can be accelerated on CPUs which support the NEON
extension.  This patch adds arm platform support to ossl(4) and provides
an AES-CBC implementation, though bsaes_cbc_encrypt() only implements
decryption.  The real goal is to provide an accelerated AES-GCM
implementation; this will be added in a subsequent patch.

Initially derived from https://reviews.freebsd.org/D37420.

Reviewed by:	jhb
Sponsored by:	Klara, Inc.
Sponsored by:	Stormshield
MFC after:	3 months
Differential Revision:	https://reviews.freebsd.org/D41304

sys/conf/files.arm

@@ -135,6 +135,16 @@ libkern/ucmpdi2.c		standard
 libkern/udivdi3.c		standard
 libkern/umoddi3.c		standard
 
+crypto/openssl/ossl_arm.c		optional ossl
+crypto/openssl/arm/aes-armv4.S		optional ossl
+crypto/openssl/arm/bsaes-armv7.S	optional ossl	\
+	compile-with	"${CC} -D__KERNEL__ -c ${CFLAGS:N-mgeneral-regs-only} ${WERROR} ${.IMPSRC}"
+crypto/openssl/arm/chacha-armv4.S	optional ossl
+crypto/openssl/arm/poly1305-armv4.S	optional ossl
+crypto/openssl/arm/sha1-armv4-large.S	optional ossl
+crypto/openssl/arm/sha256-armv4.S	optional ossl
+crypto/openssl/arm/sha512-armv4.S	optional ossl
+
 # Annapurna support
 arm/annapurna/alpine/alpine_ccu.c		optional al_ccu fdt
 arm/annapurna/alpine/alpine_nb_service.c	optional al_nb_service fdt

sys/crypto/openssl/arm/arm_arch.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright 2011-2018 The OpenSSL Project Authors. All Rights Reserved.
+ *
+ * Licensed under the OpenSSL license (the "License").  You may not use
+ * this file except in compliance with the License.  You can obtain a copy
+ * in the file LICENSE in the source distribution or at
+ * https://www.openssl.org/source/license.html
+ */
+
+#ifndef OSSL_CRYPTO_ARM_ARCH_H
+# define OSSL_CRYPTO_ARM_ARCH_H
+
+# if !defined(__ARM_ARCH__)
+#  if defined(__CC_ARM)
+#   define __ARM_ARCH__ __TARGET_ARCH_ARM
+#   if defined(__BIG_ENDIAN)
+#    define __ARMEB__
+#   else
+#    define __ARMEL__
+#   endif
+#  elif defined(__GNUC__)
+#   if   defined(__aarch64__)
+#    define __ARM_ARCH__ 8
+#    if __BYTE_ORDER__==__ORDER_BIG_ENDIAN__
+#     define __ARMEB__
+#    else
+#     define __ARMEL__
+#    endif
+  /*
+   * Why doesn't gcc define __ARM_ARCH__? Instead it defines
+   * bunch of below macros. See all_architectures[] table in
+   * gcc/config/arm/arm.c. On a side note it defines
+   * __ARMEL__/__ARMEB__ for little-/big-endian.
+   */
+#   elif defined(__ARM_ARCH)
+#    define __ARM_ARCH__ __ARM_ARCH
+#   elif defined(__ARM_ARCH_8A__)
+#    define __ARM_ARCH__ 8
+#   elif defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)     || \
+        defined(__ARM_ARCH_7R__)|| defined(__ARM_ARCH_7M__)     || \
+        defined(__ARM_ARCH_7EM__)
+#    define __ARM_ARCH__ 7
+#   elif defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__)     || \
+        defined(__ARM_ARCH_6K__)|| defined(__ARM_ARCH_6M__)     || \
+        defined(__ARM_ARCH_6Z__)|| defined(__ARM_ARCH_6ZK__)    || \
+        defined(__ARM_ARCH_6T2__)
+#    define __ARM_ARCH__ 6
+#   elif defined(__ARM_ARCH_5__) || defined(__ARM_ARCH_5T__)     || \
+        defined(__ARM_ARCH_5E__)|| defined(__ARM_ARCH_5TE__)    || \
+        defined(__ARM_ARCH_5TEJ__)
+#    define __ARM_ARCH__ 5
+#   elif defined(__ARM_ARCH_4__) || defined(__ARM_ARCH_4T__)
+#    define __ARM_ARCH__ 4
+#   else
+#    error "unsupported ARM architecture"
+#   endif
+#  endif
+# endif
+
+# if !defined(__ARM_MAX_ARCH__)
+#  define __ARM_MAX_ARCH__ __ARM_ARCH__
+# endif
+
+# if __ARM_MAX_ARCH__<__ARM_ARCH__
+#  error "__ARM_MAX_ARCH__ can't be less than __ARM_ARCH__"
+# elif __ARM_MAX_ARCH__!=__ARM_ARCH__
+#  if __ARM_ARCH__<7 && __ARM_MAX_ARCH__>=7 && defined(__ARMEB__)
+#   error "can't build universal big-endian binary"
+#  endif
+# endif
+
+# ifndef __ASSEMBLER__
+extern unsigned int OPENSSL_armcap_P;
+# endif
+
+# define ARMV7_NEON      (1<<0)
+# define ARMV7_TICK      (1<<1)
+# define ARMV8_AES       (1<<2)
+# define ARMV8_SHA1      (1<<3)
+# define ARMV8_SHA256    (1<<4)
+# define ARMV8_PMULL     (1<<5)
+# define ARMV8_SHA512    (1<<6)
+
+#endif

sys/crypto/openssl/ossl_aes.c

@@ -39,6 +39,8 @@
 #include <crypto/openssl/ossl_x86.h>
 #elif defined (__aarch64__)
 #include <crypto/openssl/ossl_aarch64.h>
+#elif defined (__arm__)
+#include <crypto/openssl/ossl_arm.h>
 #endif
 
 static ossl_cipher_process_t ossl_aes_cbc;

sys/crypto/openssl/ossl_aes_gcm.h

@@ -28,7 +28,6 @@
 #define	_OSSL_AES_GCM_H_
 
 #include <crypto/openssl/ossl_cipher.h>
-#include <crypto/rijndael/rijndael.h>
 
 struct ossl_gcm_context;
 
@@ -64,10 +63,7 @@ struct ossl_gcm_context {
 		unsigned int mres, ares;
 	} gcm;
 
-	struct {
-		uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
-		int rounds;
-	} aes_ks;
+	struct ossl_aes_keysched aes_ks;
 
 	const struct ossl_aes_gcm_ops *ops;
 };

sys/crypto/openssl/ossl_arm.c

@@ -0,0 +1,59 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2023 Stormshield
+ * Copyright (c) 2023 Semihalf
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+
+#include <machine/elf.h>
+#include <machine/md_var.h>
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+#include <crypto/openssl/arm/arm_arch.h>
+
+ossl_cipher_setkey_t AES_set_encrypt_key;
+ossl_cipher_setkey_t AES_set_decrypt_key;
+
+unsigned int OPENSSL_armcap_P;
+
+void
+ossl_cpuid(struct ossl_softc *sc)
+{
+	if (elf_hwcap & HWCAP_NEON) {
+		OPENSSL_armcap_P |= ARMV7_NEON;
+
+		sc->has_aes = true;
+		ossl_cipher_aes_cbc.set_encrypt_key = AES_set_encrypt_key;
+		ossl_cipher_aes_cbc.set_decrypt_key = AES_set_decrypt_key;
+	}
+}

sys/crypto/openssl/ossl_arm.h

@@ -0,0 +1,94 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
+ *
+ * Copyright (c) 2023 Stormshield
+ * Copyright (c) 2023 Semihalf
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification.
+ * 2. Redistributions in binary form must reproduce at minimum a disclaimer
+ *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
+ *    redistribution must be conditioned upon including a substantially
+ *    similar Disclaimer requirement for further binary redistribution.
+ *
+ * NO WARRANTY
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
+ * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGES.
+ */
+
+#ifndef __OSSL_ARM__
+#define __OSSL_ARM__
+
+#include <crypto/openssl/ossl.h>
+#include <crypto/openssl/ossl_cipher.h>
+
+#include <opencrypto/cryptodev.h>
+
+struct bsaes_key {
+	struct ossl_aes_keysched ks;
+	int converted;
+#define	BSAES_KEY_SIZE	(128 * (RIJNDAEL_MAXNR - 1) + 2 * AES_BLOCK_LEN)
+	uint8_t bitslice[BSAES_KEY_SIZE] __aligned(8);
+} __aligned(8);
+
+ossl_cipher_encrypt_t ossl_bsaes_cbc_encrypt;
+
+void AES_encrypt(const void *, void *, const void *);
+
+static inline void
+AES_CBC_ENCRYPT(const unsigned char *in, unsigned char *out,
+    size_t length, const void *key, unsigned char *iv, int encrypt)
+{
+	struct bsaes_key bsks;
+	uint32_t iv32[4], scratch[4];
+
+	/*
+	 * bsaes_cbc_encrypt has some special requirements w.r.t input data.
+	 * The key buffer, that normally holds round keys is used as a scratch
+	 * space. 128 bytes per round of extra space is required.
+	 * Another thing is that only decryption is supported.
+	 * In the case of encryption block chaining has to be done in C.
+	 */
+	if (!encrypt) {
+		memcpy(&bsks.ks, key, sizeof(bsks.ks));
+		bsks.converted = 0;
+		ossl_bsaes_cbc_encrypt(in, out, length, &bsks, iv, false);
+		return;
+	}
+
+	length /= AES_BLOCK_LEN;
+	memcpy(iv32, iv, AES_BLOCK_LEN);
+
+	while (length-- > 0) {
+		memcpy(scratch, in, AES_BLOCK_LEN);
+
+		/* XOR plaintext with IV. */
+		scratch[0] ^= iv32[0];
+		scratch[1] ^= iv32[1];
+		scratch[2] ^= iv32[2];
+		scratch[3] ^= iv32[3];
+
+		AES_encrypt(scratch, out, key);
+
+		memcpy(iv32, out, AES_BLOCK_LEN);
+		in += AES_BLOCK_LEN;
+		out += AES_BLOCK_LEN;
+	}
+
+	memcpy(iv, iv32, AES_BLOCK_LEN);
+}
+
+#endif /* __OSSL_ARM__ */

sys/crypto/openssl/ossl_cipher.h

@@ -28,6 +28,9 @@
 #ifndef __OSSL_CIPHER_H__
 #define __OSSL_CIPHER_H__
 
+#include <sys/types.h>
+#include <crypto/rijndael/rijndael.h>
+
 struct ossl_session_cipher;
 struct cryptop;
 struct crypto_session_params;
@@ -50,4 +53,9 @@ struct ossl_cipher {
 	ossl_cipher_process_t	*process;
 };
 
+struct ossl_aes_keysched {
+	uint32_t ks[4 * (RIJNDAEL_MAXNR + 1)];
+	int rounds;
+};
+
 #endif

sys/modules/Makefile

@@ -565,11 +565,15 @@ _iwlwifi=	iwlwifi
 .if ${MK_SOURCELESS_UCODE} != "no"
 _iwlwififw=	iwlwififw
 .endif
-_ossl=		ossl
 _rtw88=		rtw88
 _vmware=	vmware
 .endif
 
+.if ${MACHINE_CPUARCH} == "aarch64" || ${MACHINE_CPUARCH} == "amd64" || \
+    ${MACHINE_CPUARCH} == "i386" || ${MACHINE_ARCH} == "armv7"
+_ossl=		ossl
+.endif
+
 # MAC framework
 .if ${KERN_OPTS:MMAC} || defined(ALL_MODULES)
 _mac_biba=	mac_biba

sys/modules/ossl/Makefile

@@ -16,6 +16,16 @@ SRCS=	bus_if.h \
 	ossl_sha512.c \
 	${SRCS.${MACHINE_CPUARCH}}
 
+SRCS.arm= \
+	aes-armv4.S \
+	bsaes-armv7.S \
+	chacha-armv4.S \
+	poly1305-armv4.S  \
+	sha1-armv4-large.S \
+	sha256-armv4.S \
+	sha512-armv4.S \
+	ossl_arm.c
+
 SRCS.aarch64= \
 	chacha-armv8.S \
 	poly1305-armv8.S \
@@ -47,6 +57,8 @@ SRCS.i386= \
 	sha512-586.S \
 	ossl_x86.c
 
+CFLAGS.bsaes-armv7.S+=	-D__KERNEL__
+
 # For arm64, we are forced to rewrite the compiler invocation for the assembly
 # files, to remove -mgeneral-regs-only.
 ${SRCS.aarch64:M*.S:S/S/o/}: ${.TARGET:R}.S