/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * BLAKE2b digest algorithm, NEON accelerated
 *
 * Copyright 2020 Google LLC
 */

#include <asm/neon.h>
#include <asm/simd.h>

static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);

asmlinkage void blake2b_compress_neon(struct blake2b_ctx *ctx,
				      const u8 *data, size_t nblocks, u32 inc);

static void blake2b_compress(struct blake2b_ctx *ctx,
			     const u8 *data, size_t nblocks, u32 inc)
{
	if (!static_branch_likely(&have_neon) || !may_use_simd()) {
		blake2b_compress_generic(ctx, data, nblocks, inc);
		return;
	}
	do {
		const size_t blocks = min_t(size_t, nblocks,
					    SZ_4K / BLAKE2B_BLOCK_SIZE);

		scoped_ksimd()
			blake2b_compress_neon(ctx, data, blocks, inc);

		data += blocks * BLAKE2B_BLOCK_SIZE;
		nblocks -= blocks;
	} while (nblocks);
}

#define blake2b_mod_init_arch blake2b_mod_init_arch
static void blake2b_mod_init_arch(void)
{
	if (elf_hwcap & HWCAP_NEON)
		static_branch_enable(&have_neon);
}