arch/x86/kernel/keylocker.c

// SPDX-License-Identifier: GPL-2.0-only

/*
 * Setup Key Locker feature and support internal wrapping key
 * management.
 */

#include <linux/random.h>
#include <linux/poison.h>

#include <asm/fpu/api.h>
#include <asm/keylocker.h>
#include <asm/tlbflush.h>
#include <asm/msr.h>

static struct keylocker_setup_data {
	bool initialized;
	struct iwkey key;
} kl_setup;

/*
 * This flag is set with IWKey load. When the key restore fails, it is
 * reset. This restore state is exported to the crypto library, then AES-KL
 * will not be used there. So, the feature is soft-disabled with this flag.
 */
static bool valid_kl;

bool valid_keylocker(void)
{
	return valid_kl;
}
EXPORT_SYMBOL_GPL(valid_keylocker);

static void __init generate_keylocker_data(void)
{
	get_random_bytes(&kl_setup.key.integrity_key,  sizeof(kl_setup.key.integrity_key));
	get_random_bytes(&kl_setup.key.encryption_key, sizeof(kl_setup.key.encryption_key));
}

void __init destroy_keylocker_data(void)
{
	if (!cpu_feature_enabled(X86_FEATURE_KEYLOCKER) ||
	    cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
	        return;

	memset(&kl_setup.key, KEY_DESTROY, sizeof(kl_setup.key));
	kl_setup.initialized = true;
	valid_kl = true;
}

static void load_keylocker(void)
{
	kernel_fpu_begin();
	load_xmm_iwkey(&kl_setup.key);
	kernel_fpu_end();
}

/**
 * copy_keylocker - Copy the internal wrapping key from the backup.
 *
 * Request hardware to copy the key in non-volatile storage to the CPU
 * state.
 *
 * Returns:	-EBUSY if the copy fails, 0 if successful.
 */
static int copy_keylocker(void)
{
	u64 status;

	wrmsrl(MSR_IA32_COPY_IWKEY_TO_LOCAL, 1);

	rdmsrl(MSR_IA32_IWKEY_COPY_STATUS, status);
	if (status & BIT(0))
		return 0;
	else
		return -EBUSY;
}

/**
 * setup_keylocker - Enable the feature.
 * @c:		A pointer to struct cpuinfo_x86
 */
void __ref setup_keylocker(struct cpuinfo_x86 *c)
{
	/* This feature is not compatible with a hypervisor. */
	if (!cpu_feature_enabled(X86_FEATURE_KEYLOCKER) ||
	    cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
		goto out;

	cr4_set_bits(X86_CR4_KEYLOCKER);

	if (c == &boot_cpu_data) {
		u32 eax, ebx, ecx, edx;
		bool backup_available;

		cpuid_count(KEYLOCKER_CPUID, 0, &eax, &ebx, &ecx, &edx);
		/*
		 * Check the feature readiness via CPUID. Note that the
		 * CPUID AESKLE bit is conditionally set only when CR4.KL
		 * is set.
		 */
		if (!(ebx & KEYLOCKER_CPUID_EBX_AESKLE) ||
		    !(eax & KEYLOCKER_CPUID_EAX_SUPERVISOR)) {
			pr_debug("x86/keylocker: Not fully supported.\n");
			goto disable;
		}

		backup_available = !!(ebx & KEYLOCKER_CPUID_EBX_BACKUP);
		/*
		 * The internal wrapping key in CPU state is volatile in
		 * S3/4 states. So ensure the backup capability along with
		 * S-states.
		 */
		if (!backup_available && IS_ENABLED(CONFIG_SUSPEND)) {
			pr_debug("x86/keylocker: No key backup support with possible S3/4.\n");
			goto disable;
		}

		generate_keylocker_data();
		load_keylocker();

		/* Backup an internal wrapping key in non-volatile media. */
		if (backup_available)
			wrmsrl(MSR_IA32_BACKUP_IWKEY_TO_PLATFORM, 1);
	} else {
		int rc;

		/*
		 * Load the internal wrapping key directly when available
		 * in memory, which is only possible at boot-time.
		 *
		 * NB: When system wakes up, this path also recovers the
		 * internal wrapping key.
		 */
		if (!kl_setup.initialized) {
			load_keylocker();
		} else if (valid_kl) {
			rc = copy_keylocker();
			/*
			 * The boot CPU was successful but the key copy
			 * fails here. Then, the subsequent feature use
			 * will have inconsistent keys and failures. So,
			 * invalidate the feature via the flag.
			 */
			if (rc) {
				valid_kl = false;
				pr_err_once("x86/keylocker: Invalid copy status (rc: %d).\n", rc);
			}
		}
	}

	pr_info_once("x86/keylocker: Enabled.\n");
	return;

disable:
	setup_clear_cpu_cap(X86_FEATURE_KEYLOCKER);
	pr_info_once("x86/keylocker: Disabled.\n");
out:
	/* Make sure the feature disabled for kexec-reboot. */
	cr4_clear_bits(X86_CR4_KEYLOCKER);
}

/**
 * restore_keylocker - Restore the internal wrapping key.
 *
 * The boot CPU executes this while other CPUs restore it through the setup
 * function.
 */
void restore_keylocker(void)
{
	u64 backup_status;
	int rc;

	if (!cpu_feature_enabled(X86_FEATURE_KEYLOCKER) || !valid_kl)
		return;

	/*
	 * The IA32_IWKEYBACKUP_STATUS MSR contains a bitmap that indicates
	 * an invalid backup if bit 0 is set and a read (or write) error if
	 * bit 2 is set.
	 */
	rdmsrl(MSR_IA32_IWKEY_BACKUP_STATUS, backup_status);
	if (backup_status & BIT(0)) {
		rc = copy_keylocker();
		if (rc)
			pr_err("x86/keylocker: Invalid copy state (rc: %d).\n", rc);
		else
			return;
	} else {
		pr_err("x86/keylocker: The key backup access failed with %s.\n",
		       (backup_status & BIT(2)) ? "read error" : "invalid status");
	}

	/*
	 * Now the backup key is not available. Invalidate the feature via
	 * the flag to avoid any subsequent use. But keep the feature with
	 * zero IWKeys instead of disabling it. The current users will see
	 * key handle integrity failure but that's because of the internal
	 * key change.
	 */
	pr_err("x86/keylocker: Failed to restore internal wrapping key.\n");
	valid_kl = false;
}