lib/crypto/arm64/aes.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * AES block cipher, optimized for ARM64
 *
 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
 * Copyright 2026 Google LLC
 */

#include <asm/neon.h>
#include <asm/simd.h>
#include <linux/unaligned.h>
#include <linux/cpufeature.h>

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes);

struct aes_block {
	u8 b[AES_BLOCK_SIZE];
};

asmlinkage void __aes_arm64_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
				    const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_arm64_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
				    const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_ce_encrypt(const u32 rk[], u8 out[AES_BLOCK_SIZE],
				 const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage void __aes_ce_decrypt(const u32 inv_rk[], u8 out[AES_BLOCK_SIZE],
				 const u8 in[AES_BLOCK_SIZE], int rounds);
asmlinkage u32 __aes_ce_sub(u32 l);
asmlinkage void __aes_ce_invert(struct aes_block *out,
				const struct aes_block *in);
asmlinkage void neon_aes_mac_update(u8 const in[], u32 const rk[], int rounds,
				    size_t blocks, u8 dg[], int enc_before,
				    int enc_after);

/*
 * Expand an AES key using the crypto extensions if supported and usable or
 * generic code otherwise.  The expanded key format is compatible between the
 * two cases.  The outputs are @rndkeys (required) and @inv_rndkeys (optional).
 */
static void aes_expandkey_arm64(u32 rndkeys[], u32 *inv_rndkeys,
				const u8 *in_key, int key_len, int nrounds)
{
	/*
	 * The AES key schedule round constants
	 */
	static u8 const rcon[] = {
		0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36,
	};

	u32 kwords = key_len / sizeof(u32);
	struct aes_block *key_enc, *key_dec;
	int i, j;

	if (!static_branch_likely(&have_aes) || unlikely(!may_use_simd())) {
		aes_expandkey_generic(rndkeys, inv_rndkeys, in_key, key_len);
		return;
	}

	for (i = 0; i < kwords; i++)
		rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);

	scoped_ksimd() {
		for (i = 0; i < sizeof(rcon); i++) {
			u32 *rki = &rndkeys[i * kwords];
			u32 *rko = rki + kwords;

			rko[0] = ror32(__aes_ce_sub(rki[kwords - 1]), 8) ^
				 rcon[i] ^ rki[0];
			rko[1] = rko[0] ^ rki[1];
			rko[2] = rko[1] ^ rki[2];
			rko[3] = rko[2] ^ rki[3];

			if (key_len == AES_KEYSIZE_192) {
				if (i >= 7)
					break;
				rko[4] = rko[3] ^ rki[4];
				rko[5] = rko[4] ^ rki[5];
			} else if (key_len == AES_KEYSIZE_256) {
				if (i >= 6)
					break;
				rko[4] = __aes_ce_sub(rko[3]) ^ rki[4];
				rko[5] = rko[4] ^ rki[5];
				rko[6] = rko[5] ^ rki[6];
				rko[7] = rko[6] ^ rki[7];
			}
		}

		/*
		 * Generate the decryption keys for the Equivalent Inverse
		 * Cipher.  This involves reversing the order of the round
		 * keys, and applying the Inverse Mix Columns transformation on
		 * all but the first and the last one.
		 */
		if (inv_rndkeys) {
			key_enc = (struct aes_block *)rndkeys;
			key_dec = (struct aes_block *)inv_rndkeys;
			j = nrounds;

			key_dec[0] = key_enc[j];
			for (i = 1, j--; j > 0; i++, j--)
				__aes_ce_invert(key_dec + i, key_enc + j);
			key_dec[i] = key_enc[0];
		}
	}
}

static void aes_preparekey_arch(union aes_enckey_arch *k,
				union aes_invkey_arch *inv_k,
				const u8 *in_key, int key_len, int nrounds)
{
	aes_expandkey_arm64(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
			    in_key, key_len, nrounds);
}

/*
 * This is here temporarily until the remaining AES mode implementations are
 * migrated from arch/arm64/crypto/ to lib/crypto/arm64/.
 */
int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
		     unsigned int key_len)
{
	if (aes_check_keylen(key_len) != 0)
		return -EINVAL;
	ctx->key_length = key_len;
	aes_expandkey_arm64(ctx->key_enc, ctx->key_dec, in_key, key_len,
			    6 + key_len / 4);
	return 0;
}
EXPORT_SYMBOL(ce_aes_expandkey);

EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_ecb_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_ctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_xts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(neon_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");

EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_ecb_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_cbc_cts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_ctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xctr_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xts_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_xts_decrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_encrypt, "CRYPTO_INTERNAL");
EXPORT_SYMBOL_NS_GPL(ce_aes_essiv_cbc_decrypt, "CRYPTO_INTERNAL");
#if IS_MODULE(CONFIG_CRYPTO_AES_ARM64_CE_CCM)
EXPORT_SYMBOL_NS_GPL(ce_aes_mac_update, "CRYPTO_INTERNAL");
#endif

static void aes_encrypt_arch(const struct aes_enckey *key,
			     u8 out[AES_BLOCK_SIZE],
			     const u8 in[AES_BLOCK_SIZE])
{
	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
		scoped_ksimd()
			__aes_ce_encrypt(key->k.rndkeys, out, in, key->nrounds);
	} else {
		__aes_arm64_encrypt(key->k.rndkeys, out, in, key->nrounds);
	}
}

static void aes_decrypt_arch(const struct aes_key *key,
			     u8 out[AES_BLOCK_SIZE],
			     const u8 in[AES_BLOCK_SIZE])
{
	if (static_branch_likely(&have_aes) && likely(may_use_simd())) {
		scoped_ksimd()
			__aes_ce_decrypt(key->inv_k.inv_rndkeys, out, in,
					 key->nrounds);
	} else {
		__aes_arm64_decrypt(key->inv_k.inv_rndkeys, out, in,
				    key->nrounds);
	}
}

#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC_MACS)
#define aes_cbcmac_blocks_arch aes_cbcmac_blocks_arch
static bool aes_cbcmac_blocks_arch(u8 h[AES_BLOCK_SIZE],
				   const struct aes_enckey *key, const u8 *data,
				   size_t nblocks, bool enc_before,
				   bool enc_after)
{
	if (static_branch_likely(&have_neon) && likely(may_use_simd())) {
		scoped_ksimd() {
			if (static_branch_likely(&have_aes))
				ce_aes_mac_update(data, key->k.rndkeys,
						  key->nrounds, nblocks, h,
						  enc_before, enc_after);
			else
				neon_aes_mac_update(data, key->k.rndkeys,
						    key->nrounds, nblocks, h,
						    enc_before, enc_after);
		}
		return true;
	}
	return false;
}
#endif /* CONFIG_CRYPTO_LIB_AES_CBC_MACS */

#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
	if (cpu_have_named_feature(ASIMD)) {
		static_branch_enable(&have_neon);
		if (cpu_have_named_feature(AES))
			static_branch_enable(&have_aes);
	}
}