| // SPDX-License-Identifier: GPL-2.0-or-later |
| /* |
| * Cryptographic API. |
| * |
| * Support for VIA PadLock hardware crypto engine. |
| * |
| * Copyright (c) 2006 Michal Ludvig <michal@logix.cz> |
| */ |
| |
| #include <crypto/internal/hash.h> |
| #include <crypto/padlock.h> |
| #include <crypto/sha.h> |
| #include <linux/err.h> |
| #include <linux/module.h> |
| #include <linux/init.h> |
| #include <linux/errno.h> |
| #include <linux/interrupt.h> |
| #include <linux/kernel.h> |
| #include <linux/scatterlist.h> |
| #include <asm/cpu_device_id.h> |
| #include <asm/fpu/api.h> |
| |
| struct padlock_sha_desc { |
| struct shash_desc fallback; |
| }; |
| |
| struct padlock_sha_ctx { |
| struct crypto_shash *fallback; |
| }; |
| |
| static int padlock_sha_init(struct shash_desc *desc) |
| { |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); |
| |
| dctx->fallback.tfm = ctx->fallback; |
| return crypto_shash_init(&dctx->fallback); |
| } |
| |
| static int padlock_sha_update(struct shash_desc *desc, |
| const u8 *data, unsigned int length) |
| { |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| |
| return crypto_shash_update(&dctx->fallback, data, length); |
| } |
| |
| static int padlock_sha_export(struct shash_desc *desc, void *out) |
| { |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| |
| return crypto_shash_export(&dctx->fallback, out); |
| } |
| |
| static int padlock_sha_import(struct shash_desc *desc, const void *in) |
| { |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); |
| |
| dctx->fallback.tfm = ctx->fallback; |
| return crypto_shash_import(&dctx->fallback, in); |
| } |
| |
| static inline void padlock_output_block(uint32_t *src, |
| uint32_t *dst, size_t count) |
| { |
| while (count--) |
| *dst++ = swab32(*src++); |
| } |
| |
| static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, |
| unsigned int count, u8 *out) |
| { |
| /* We can't store directly to *out as it may be unaligned. */ |
| /* BTW Don't reduce the buffer size below 128 Bytes! |
| * PadLock microcode needs it that big. */ |
| char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
| ((aligned(STACK_ALIGN))); |
| char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| struct sha1_state state; |
| unsigned int space; |
| unsigned int leftover; |
| int err; |
| |
| err = crypto_shash_export(&dctx->fallback, &state); |
| if (err) |
| goto out; |
| |
| if (state.count + count > ULONG_MAX) |
| return crypto_shash_finup(&dctx->fallback, in, count, out); |
| |
| leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; |
| space = SHA1_BLOCK_SIZE - leftover; |
| if (space) { |
| if (count > space) { |
| err = crypto_shash_update(&dctx->fallback, in, space) ?: |
| crypto_shash_export(&dctx->fallback, &state); |
| if (err) |
| goto out; |
| count -= space; |
| in += space; |
| } else { |
| memcpy(state.buffer + leftover, in, count); |
| in = state.buffer; |
| count += leftover; |
| state.count &= ~(SHA1_BLOCK_SIZE - 1); |
| } |
| } |
| |
| memcpy(result, &state.state, SHA1_DIGEST_SIZE); |
| |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ |
| : \ |
| : "c"((unsigned long)state.count + count), \ |
| "a"((unsigned long)state.count), \ |
| "S"(in), "D"(result)); |
| |
| padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); |
| |
| out: |
| return err; |
| } |
| |
| static int padlock_sha1_final(struct shash_desc *desc, u8 *out) |
| { |
| u8 buf[4]; |
| |
| return padlock_sha1_finup(desc, buf, 0, out); |
| } |
| |
| static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, |
| unsigned int count, u8 *out) |
| { |
| /* We can't store directly to *out as it may be unaligned. */ |
| /* BTW Don't reduce the buffer size below 128 Bytes! |
| * PadLock microcode needs it that big. */ |
| char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
| ((aligned(STACK_ALIGN))); |
| char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
| struct padlock_sha_desc *dctx = shash_desc_ctx(desc); |
| struct sha256_state state; |
| unsigned int space; |
| unsigned int leftover; |
| int err; |
| |
| err = crypto_shash_export(&dctx->fallback, &state); |
| if (err) |
| goto out; |
| |
| if (state.count + count > ULONG_MAX) |
| return crypto_shash_finup(&dctx->fallback, in, count, out); |
| |
| leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; |
| space = SHA256_BLOCK_SIZE - leftover; |
| if (space) { |
| if (count > space) { |
| err = crypto_shash_update(&dctx->fallback, in, space) ?: |
| crypto_shash_export(&dctx->fallback, &state); |
| if (err) |
| goto out; |
| count -= space; |
| in += space; |
| } else { |
| memcpy(state.buf + leftover, in, count); |
| in = state.buf; |
| count += leftover; |
| state.count &= ~(SHA1_BLOCK_SIZE - 1); |
| } |
| } |
| |
| memcpy(result, &state.state, SHA256_DIGEST_SIZE); |
| |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ |
| : \ |
| : "c"((unsigned long)state.count + count), \ |
| "a"((unsigned long)state.count), \ |
| "S"(in), "D"(result)); |
| |
| padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); |
| |
| out: |
| return err; |
| } |
| |
| static int padlock_sha256_final(struct shash_desc *desc, u8 *out) |
| { |
| u8 buf[4]; |
| |
| return padlock_sha256_finup(desc, buf, 0, out); |
| } |
| |
| static int padlock_cra_init(struct crypto_tfm *tfm) |
| { |
| struct crypto_shash *hash = __crypto_shash_cast(tfm); |
| const char *fallback_driver_name = crypto_tfm_alg_name(tfm); |
| struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm); |
| struct crypto_shash *fallback_tfm; |
| int err = -ENOMEM; |
| |
| /* Allocate a fallback and abort if it failed. */ |
| fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, |
| CRYPTO_ALG_NEED_FALLBACK); |
| if (IS_ERR(fallback_tfm)) { |
| printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", |
| fallback_driver_name); |
| err = PTR_ERR(fallback_tfm); |
| goto out; |
| } |
| |
| ctx->fallback = fallback_tfm; |
| hash->descsize += crypto_shash_descsize(fallback_tfm); |
| return 0; |
| |
| out: |
| return err; |
| } |
| |
| static void padlock_cra_exit(struct crypto_tfm *tfm) |
| { |
| struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm); |
| |
| crypto_free_shash(ctx->fallback); |
| } |
| |
| static struct shash_alg sha1_alg = { |
| .digestsize = SHA1_DIGEST_SIZE, |
| .init = padlock_sha_init, |
| .update = padlock_sha_update, |
| .finup = padlock_sha1_finup, |
| .final = padlock_sha1_final, |
| .export = padlock_sha_export, |
| .import = padlock_sha_import, |
| .descsize = sizeof(struct padlock_sha_desc), |
| .statesize = sizeof(struct sha1_state), |
| .base = { |
| .cra_name = "sha1", |
| .cra_driver_name = "sha1-padlock", |
| .cra_priority = PADLOCK_CRA_PRIORITY, |
| .cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
| .cra_blocksize = SHA1_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct padlock_sha_ctx), |
| .cra_module = THIS_MODULE, |
| .cra_init = padlock_cra_init, |
| .cra_exit = padlock_cra_exit, |
| } |
| }; |
| |
| static struct shash_alg sha256_alg = { |
| .digestsize = SHA256_DIGEST_SIZE, |
| .init = padlock_sha_init, |
| .update = padlock_sha_update, |
| .finup = padlock_sha256_finup, |
| .final = padlock_sha256_final, |
| .export = padlock_sha_export, |
| .import = padlock_sha_import, |
| .descsize = sizeof(struct padlock_sha_desc), |
| .statesize = sizeof(struct sha256_state), |
| .base = { |
| .cra_name = "sha256", |
| .cra_driver_name = "sha256-padlock", |
| .cra_priority = PADLOCK_CRA_PRIORITY, |
| .cra_flags = CRYPTO_ALG_NEED_FALLBACK, |
| .cra_blocksize = SHA256_BLOCK_SIZE, |
| .cra_ctxsize = sizeof(struct padlock_sha_ctx), |
| .cra_module = THIS_MODULE, |
| .cra_init = padlock_cra_init, |
| .cra_exit = padlock_cra_exit, |
| } |
| }; |
| |
| /* Add two shash_alg instance for hardware-implemented * |
| * multiple-parts hash supported by VIA Nano Processor.*/ |
| static int padlock_sha1_init_nano(struct shash_desc *desc) |
| { |
| struct sha1_state *sctx = shash_desc_ctx(desc); |
| |
| *sctx = (struct sha1_state){ |
| .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, |
| }; |
| |
| return 0; |
| } |
| |
| static int padlock_sha1_update_nano(struct shash_desc *desc, |
| const u8 *data, unsigned int len) |
| { |
| struct sha1_state *sctx = shash_desc_ctx(desc); |
| unsigned int partial, done; |
| const u8 *src; |
| /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ |
| u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
| ((aligned(STACK_ALIGN))); |
| u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
| |
| partial = sctx->count & 0x3f; |
| sctx->count += len; |
| done = 0; |
| src = data; |
| memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE); |
| |
| if ((partial + len) >= SHA1_BLOCK_SIZE) { |
| |
| /* Append the bytes in state's buffer to a block to handle */ |
| if (partial) { |
| done = -partial; |
| memcpy(sctx->buffer + partial, data, |
| done + SHA1_BLOCK_SIZE); |
| src = sctx->buffer; |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" |
| : "+S"(src), "+D"(dst) \ |
| : "a"((long)-1), "c"((unsigned long)1)); |
| done += SHA1_BLOCK_SIZE; |
| src = data + done; |
| } |
| |
| /* Process the left bytes from the input data */ |
| if (len - done >= SHA1_BLOCK_SIZE) { |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" |
| : "+S"(src), "+D"(dst) |
| : "a"((long)-1), |
| "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE))); |
| done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE); |
| src = data + done; |
| } |
| partial = 0; |
| } |
| memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE); |
| memcpy(sctx->buffer + partial, src, len - done); |
| |
| return 0; |
| } |
| |
| static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out) |
| { |
| struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc); |
| unsigned int partial, padlen; |
| __be64 bits; |
| static const u8 padding[64] = { 0x80, }; |
| |
| bits = cpu_to_be64(state->count << 3); |
| |
| /* Pad out to 56 mod 64 */ |
| partial = state->count & 0x3f; |
| padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); |
| padlock_sha1_update_nano(desc, padding, padlen); |
| |
| /* Append length field bytes */ |
| padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits)); |
| |
| /* Swap to output */ |
| padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5); |
| |
| return 0; |
| } |
| |
| static int padlock_sha256_init_nano(struct shash_desc *desc) |
| { |
| struct sha256_state *sctx = shash_desc_ctx(desc); |
| |
| *sctx = (struct sha256_state){ |
| .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \ |
| SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7}, |
| }; |
| |
| return 0; |
| } |
| |
| static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data, |
| unsigned int len) |
| { |
| struct sha256_state *sctx = shash_desc_ctx(desc); |
| unsigned int partial, done; |
| const u8 *src; |
| /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/ |
| u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__ |
| ((aligned(STACK_ALIGN))); |
| u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT); |
| |
| partial = sctx->count & 0x3f; |
| sctx->count += len; |
| done = 0; |
| src = data; |
| memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE); |
| |
| if ((partial + len) >= SHA256_BLOCK_SIZE) { |
| |
| /* Append the bytes in state's buffer to a block to handle */ |
| if (partial) { |
| done = -partial; |
| memcpy(sctx->buf + partial, data, |
| done + SHA256_BLOCK_SIZE); |
| src = sctx->buf; |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" |
| : "+S"(src), "+D"(dst) |
| : "a"((long)-1), "c"((unsigned long)1)); |
| done += SHA256_BLOCK_SIZE; |
| src = data + done; |
| } |
| |
| /* Process the left bytes from input data*/ |
| if (len - done >= SHA256_BLOCK_SIZE) { |
| asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" |
| : "+S"(src), "+D"(dst) |
| : "a"((long)-1), |
| "c"((unsigned long)((len - done) / 64))); |
| done += ((len - done) - (len - done) % 64); |
| src = data + done; |
| } |
| partial = 0; |
| } |
| memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE); |
| memcpy(sctx->buf + partial, src, len - done); |
| |
| return 0; |
| } |
| |
| static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out) |
| { |
| struct sha256_state *state = |
| (struct sha256_state *)shash_desc_ctx(desc); |
| unsigned int partial, padlen; |
| __be64 bits; |
| static const u8 padding[64] = { 0x80, }; |
| |
| bits = cpu_to_be64(state->count << 3); |
| |
| /* Pad out to 56 mod 64 */ |
| partial = state->count & 0x3f; |
| padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial); |
| padlock_sha256_update_nano(desc, padding, padlen); |
| |
| /* Append length field bytes */ |
| padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits)); |
| |
| /* Swap to output */ |
| padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8); |
| |
| return 0; |
| } |
| |
| static int padlock_sha_export_nano(struct shash_desc *desc, |
| void *out) |
| { |
| int statesize = crypto_shash_statesize(desc->tfm); |
| void *sctx = shash_desc_ctx(desc); |
| |
| memcpy(out, sctx, statesize); |
| return 0; |
| } |
| |
| static int padlock_sha_import_nano(struct shash_desc *desc, |
| const void *in) |
| { |
| int statesize = crypto_shash_statesize(desc->tfm); |
| void *sctx = shash_desc_ctx(desc); |
| |
| memcpy(sctx, in, statesize); |
| return 0; |
| } |
| |
| static struct shash_alg sha1_alg_nano = { |
| .digestsize = SHA1_DIGEST_SIZE, |
| .init = padlock_sha1_init_nano, |
| .update = padlock_sha1_update_nano, |
| .final = padlock_sha1_final_nano, |
| .export = padlock_sha_export_nano, |
| .import = padlock_sha_import_nano, |
| .descsize = sizeof(struct sha1_state), |
| .statesize = sizeof(struct sha1_state), |
| .base = { |
| .cra_name = "sha1", |
| .cra_driver_name = "sha1-padlock-nano", |
| .cra_priority = PADLOCK_CRA_PRIORITY, |
| .cra_blocksize = SHA1_BLOCK_SIZE, |
| .cra_module = THIS_MODULE, |
| } |
| }; |
| |
| static struct shash_alg sha256_alg_nano = { |
| .digestsize = SHA256_DIGEST_SIZE, |
| .init = padlock_sha256_init_nano, |
| .update = padlock_sha256_update_nano, |
| .final = padlock_sha256_final_nano, |
| .export = padlock_sha_export_nano, |
| .import = padlock_sha_import_nano, |
| .descsize = sizeof(struct sha256_state), |
| .statesize = sizeof(struct sha256_state), |
| .base = { |
| .cra_name = "sha256", |
| .cra_driver_name = "sha256-padlock-nano", |
| .cra_priority = PADLOCK_CRA_PRIORITY, |
| .cra_blocksize = SHA256_BLOCK_SIZE, |
| .cra_module = THIS_MODULE, |
| } |
| }; |
| |
| static const struct x86_cpu_id padlock_sha_ids[] = { |
| X86_MATCH_FEATURE(X86_FEATURE_PHE, NULL), |
| {} |
| }; |
| MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids); |
| |
| static int __init padlock_init(void) |
| { |
| int rc = -ENODEV; |
| struct cpuinfo_x86 *c = &cpu_data(0); |
| struct shash_alg *sha1; |
| struct shash_alg *sha256; |
| |
| if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN)) |
| return -ENODEV; |
| |
| /* Register the newly added algorithm module if on * |
| * VIA Nano processor, or else just do as before */ |
| if (c->x86_model < 0x0f) { |
| sha1 = &sha1_alg; |
| sha256 = &sha256_alg; |
| } else { |
| sha1 = &sha1_alg_nano; |
| sha256 = &sha256_alg_nano; |
| } |
| |
| rc = crypto_register_shash(sha1); |
| if (rc) |
| goto out; |
| |
| rc = crypto_register_shash(sha256); |
| if (rc) |
| goto out_unreg1; |
| |
| printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n"); |
| |
| return 0; |
| |
| out_unreg1: |
| crypto_unregister_shash(sha1); |
| |
| out: |
| printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n"); |
| return rc; |
| } |
| |
| static void __exit padlock_fini(void) |
| { |
| struct cpuinfo_x86 *c = &cpu_data(0); |
| |
| if (c->x86_model >= 0x0f) { |
| crypto_unregister_shash(&sha1_alg_nano); |
| crypto_unregister_shash(&sha256_alg_nano); |
| } else { |
| crypto_unregister_shash(&sha1_alg); |
| crypto_unregister_shash(&sha256_alg); |
| } |
| } |
| |
| module_init(padlock_init); |
| module_exit(padlock_fini); |
| |
| MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support."); |
| MODULE_LICENSE("GPL"); |
| MODULE_AUTHOR("Michal Ludvig"); |
| |
| MODULE_ALIAS_CRYPTO("sha1-all"); |
| MODULE_ALIAS_CRYPTO("sha256-all"); |
| MODULE_ALIAS_CRYPTO("sha1-padlock"); |
| MODULE_ALIAS_CRYPTO("sha256-padlock"); |