| // SPDX-License-Identifier: GPL-2.0-only | |
| /* | |
| * K3 DTHE V2 crypto accelerator driver | |
| * | |
| * Copyright (C) Texas Instruments 2025 - https://www.ti.com | |
| * Author: T Pratham <t-pratham@ti.com> | |
| */ | |
| #include <crypto/aead.h> | |
| #include <crypto/aes.h> | |
| #include <crypto/algapi.h> | |
| #include <crypto/engine.h> | |
| #include <crypto/gcm.h> | |
| #include <crypto/internal/aead.h> | |
| #include <crypto/internal/skcipher.h> | |
| #include "dthev2-common.h" | |
| #include <linux/bitfield.h> | |
| #include <linux/delay.h> | |
| #include <linux/dmaengine.h> | |
| #include <linux/dma-mapping.h> | |
| #include <linux/io.h> | |
| #include <linux/iopoll.h> | |
| #include <linux/scatterlist.h> | |
| /* Registers */ | |
| // AES Engine | |
| #define DTHE_P_AES_BASE 0x7000 | |
| #define DTHE_P_AES_KEY1_0 0x0038 | |
| #define DTHE_P_AES_KEY1_1 0x003C | |
| #define DTHE_P_AES_KEY1_2 0x0030 | |
| #define DTHE_P_AES_KEY1_3 0x0034 | |
| #define DTHE_P_AES_KEY1_4 0x0028 | |
| #define DTHE_P_AES_KEY1_5 0x002C | |
| #define DTHE_P_AES_KEY1_6 0x0020 | |
| #define DTHE_P_AES_KEY1_7 0x0024 | |
| #define DTHE_P_AES_KEY2_0 0x0018 | |
| #define DTHE_P_AES_KEY2_1 0x001C | |
| #define DTHE_P_AES_KEY2_2 0x0010 | |
| #define DTHE_P_AES_KEY2_3 0x0014 | |
| #define DTHE_P_AES_KEY2_4 0x0008 | |
| #define DTHE_P_AES_KEY2_5 0x000C | |
| #define DTHE_P_AES_KEY2_6 0x0000 | |
| #define DTHE_P_AES_KEY2_7 0x0004 | |
| #define DTHE_P_AES_IV_IN_0 0x0040 | |
| #define DTHE_P_AES_IV_IN_1 0x0044 | |
| #define DTHE_P_AES_IV_IN_2 0x0048 | |
| #define DTHE_P_AES_IV_IN_3 0x004C | |
| #define DTHE_P_AES_CTRL 0x0050 | |
| #define DTHE_P_AES_C_LENGTH_0 0x0054 | |
| #define DTHE_P_AES_C_LENGTH_1 0x0058 | |
| #define DTHE_P_AES_AUTH_LENGTH 0x005C | |
| #define DTHE_P_AES_DATA_IN_OUT 0x0060 | |
| #define DTHE_P_AES_TAG_OUT 0x0070 | |
| #define DTHE_P_AES_SYSCONFIG 0x0084 | |
| #define DTHE_P_AES_IRQSTATUS 0x008C | |
| #define DTHE_P_AES_IRQENABLE 0x0090 | |
| /* Register write values and macros */ | |
| enum aes_ctrl_mode_masks { | |
| AES_CTRL_ECB_MASK = 0x00, | |
| AES_CTRL_CBC_MASK = BIT(5), | |
| AES_CTRL_CTR_MASK = BIT(6), | |
| AES_CTRL_XTS_MASK = BIT(12) | BIT(11), | |
| AES_CTRL_GCM_MASK = BIT(17) | BIT(16) | BIT(6), | |
| AES_CTRL_CCM_MASK = BIT(18) | BIT(6), | |
| }; | |
| #define DTHE_AES_CTRL_MODE_CLEAR_MASK ~GENMASK(28, 5) | |
| #define DTHE_AES_CTRL_DIR_ENC BIT(2) | |
| #define DTHE_AES_CTRL_KEYSIZE_16B BIT(3) | |
| #define DTHE_AES_CTRL_KEYSIZE_24B BIT(4) | |
| #define DTHE_AES_CTRL_KEYSIZE_32B (BIT(3) | BIT(4)) | |
| #define DTHE_AES_CTRL_CTR_WIDTH_128B (BIT(7) | BIT(8)) | |
| #define DTHE_AES_CCM_L_FROM_IV_MASK GENMASK(2, 0) | |
| #define DTHE_AES_CCM_M_BITS GENMASK(2, 0) | |
| #define DTHE_AES_CTRL_CCM_L_FIELD_MASK GENMASK(21, 19) | |
| #define DTHE_AES_CTRL_CCM_M_FIELD_MASK GENMASK(24, 22) | |
| #define DTHE_AES_CTRL_SAVE_CTX_SET BIT(29) | |
| #define DTHE_AES_CTRL_OUTPUT_READY BIT_MASK(0) | |
| #define DTHE_AES_CTRL_INPUT_READY BIT_MASK(1) | |
| #define DTHE_AES_CTRL_SAVED_CTX_READY BIT_MASK(30) | |
| #define DTHE_AES_CTRL_CTX_READY BIT_MASK(31) | |
| #define DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN GENMASK(6, 5) | |
| #define DTHE_AES_IRQENABLE_EN_ALL GENMASK(3, 0) | |
| /* Misc */ | |
| #define AES_IV_SIZE AES_BLOCK_SIZE | |
| #define AES_BLOCK_WORDS (AES_BLOCK_SIZE / sizeof(u32)) | |
| #define AES_IV_WORDS AES_BLOCK_WORDS | |
| #define DTHE_AES_GCM_AAD_MAXLEN (BIT_ULL(32) - 1) | |
| #define DTHE_AES_CCM_AAD_MAXLEN (BIT(16) - BIT(8)) | |
| #define DTHE_AES_CCM_CRYPT_MAXLEN (BIT_ULL(61) - 1) | |
| #define POLL_TIMEOUT_INTERVAL HZ | |
| static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| ctx->dev_data = dev_data; | |
| ctx->keylen = 0; | |
| return 0; | |
| } | |
| static int dthe_cipher_init_tfm_fallback(struct crypto_skcipher *tfm) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| const char *alg_name = crypto_tfm_alg_name(crypto_skcipher_tfm(tfm)); | |
| ctx->dev_data = dev_data; | |
| ctx->keylen = 0; | |
| ctx->skcipher_fb = crypto_alloc_sync_skcipher(alg_name, 0, | |
| CRYPTO_ALG_NEED_FALLBACK); | |
| if (IS_ERR(ctx->skcipher_fb)) { | |
| dev_err(dev_data->dev, "fallback driver %s couldn't be loaded\n", | |
| alg_name); | |
| return PTR_ERR(ctx->skcipher_fb); | |
| } | |
| return 0; | |
| } | |
| static void dthe_cipher_exit_tfm(struct crypto_skcipher *tfm) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| crypto_free_sync_skcipher(ctx->skcipher_fb); | |
| } | |
| static int dthe_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) | |
| return -EINVAL; | |
| ctx->keylen = keylen; | |
| memcpy(ctx->key, key, keylen); | |
| return 0; | |
| } | |
| static int dthe_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| ctx->aes_mode = DTHE_AES_ECB; | |
| return dthe_aes_setkey(tfm, key, keylen); | |
| } | |
| static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| ctx->aes_mode = DTHE_AES_CBC; | |
| return dthe_aes_setkey(tfm, key, keylen); | |
| } | |
| static int dthe_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| int ret = dthe_aes_setkey(tfm, key, keylen); | |
| if (ret) | |
| return ret; | |
| ctx->aes_mode = DTHE_AES_CTR; | |
| crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); | |
| crypto_sync_skcipher_set_flags(ctx->skcipher_fb, | |
| crypto_skcipher_get_flags(tfm) & | |
| CRYPTO_TFM_REQ_MASK); | |
| return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); | |
| } | |
| static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); | |
| if (keylen != 2 * AES_KEYSIZE_128 && | |
| keylen != 2 * AES_KEYSIZE_192 && | |
| keylen != 2 * AES_KEYSIZE_256) | |
| return -EINVAL; | |
| ctx->aes_mode = DTHE_AES_XTS; | |
| ctx->keylen = keylen / 2; | |
| memcpy(ctx->key, key, keylen); | |
| crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); | |
| crypto_sync_skcipher_set_flags(ctx->skcipher_fb, | |
| crypto_skcipher_get_flags(tfm) & | |
| CRYPTO_TFM_REQ_MASK); | |
| return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); | |
| } | |
| static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, | |
| struct dthe_aes_req_ctx *rctx, | |
| u32 *iv_in) | |
| { | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; | |
| u32 ctrl_val = 0; | |
| writel_relaxed(ctx->key[0], aes_base_reg + DTHE_P_AES_KEY1_0); | |
| writel_relaxed(ctx->key[1], aes_base_reg + DTHE_P_AES_KEY1_1); | |
| writel_relaxed(ctx->key[2], aes_base_reg + DTHE_P_AES_KEY1_2); | |
| writel_relaxed(ctx->key[3], aes_base_reg + DTHE_P_AES_KEY1_3); | |
| if (ctx->keylen > AES_KEYSIZE_128) { | |
| writel_relaxed(ctx->key[4], aes_base_reg + DTHE_P_AES_KEY1_4); | |
| writel_relaxed(ctx->key[5], aes_base_reg + DTHE_P_AES_KEY1_5); | |
| } | |
| if (ctx->keylen == AES_KEYSIZE_256) { | |
| writel_relaxed(ctx->key[6], aes_base_reg + DTHE_P_AES_KEY1_6); | |
| writel_relaxed(ctx->key[7], aes_base_reg + DTHE_P_AES_KEY1_7); | |
| } | |
| if (ctx->aes_mode == DTHE_AES_XTS) { | |
| size_t key2_offset = ctx->keylen / sizeof(u32); | |
| writel_relaxed(ctx->key[key2_offset + 0], aes_base_reg + DTHE_P_AES_KEY2_0); | |
| writel_relaxed(ctx->key[key2_offset + 1], aes_base_reg + DTHE_P_AES_KEY2_1); | |
| writel_relaxed(ctx->key[key2_offset + 2], aes_base_reg + DTHE_P_AES_KEY2_2); | |
| writel_relaxed(ctx->key[key2_offset + 3], aes_base_reg + DTHE_P_AES_KEY2_3); | |
| if (ctx->keylen > AES_KEYSIZE_128) { | |
| writel_relaxed(ctx->key[key2_offset + 4], aes_base_reg + DTHE_P_AES_KEY2_4); | |
| writel_relaxed(ctx->key[key2_offset + 5], aes_base_reg + DTHE_P_AES_KEY2_5); | |
| } | |
| if (ctx->keylen == AES_KEYSIZE_256) { | |
| writel_relaxed(ctx->key[key2_offset + 6], aes_base_reg + DTHE_P_AES_KEY2_6); | |
| writel_relaxed(ctx->key[key2_offset + 7], aes_base_reg + DTHE_P_AES_KEY2_7); | |
| } | |
| } | |
| if (rctx->enc) | |
| ctrl_val |= DTHE_AES_CTRL_DIR_ENC; | |
| if (ctx->keylen == AES_KEYSIZE_128) | |
| ctrl_val |= DTHE_AES_CTRL_KEYSIZE_16B; | |
| else if (ctx->keylen == AES_KEYSIZE_192) | |
| ctrl_val |= DTHE_AES_CTRL_KEYSIZE_24B; | |
| else | |
| ctrl_val |= DTHE_AES_CTRL_KEYSIZE_32B; | |
| // Write AES mode | |
| ctrl_val &= DTHE_AES_CTRL_MODE_CLEAR_MASK; | |
| switch (ctx->aes_mode) { | |
| case DTHE_AES_ECB: | |
| ctrl_val |= AES_CTRL_ECB_MASK; | |
| break; | |
| case DTHE_AES_CBC: | |
| ctrl_val |= AES_CTRL_CBC_MASK; | |
| break; | |
| case DTHE_AES_CTR: | |
| ctrl_val |= AES_CTRL_CTR_MASK; | |
| ctrl_val |= DTHE_AES_CTRL_CTR_WIDTH_128B; | |
| break; | |
| case DTHE_AES_XTS: | |
| ctrl_val |= AES_CTRL_XTS_MASK; | |
| break; | |
| case DTHE_AES_GCM: | |
| ctrl_val |= AES_CTRL_GCM_MASK; | |
| break; | |
| case DTHE_AES_CCM: | |
| ctrl_val |= AES_CTRL_CCM_MASK; | |
| ctrl_val |= FIELD_PREP(DTHE_AES_CTRL_CCM_L_FIELD_MASK, | |
| (iv_in[0] & DTHE_AES_CCM_L_FROM_IV_MASK)); | |
| ctrl_val |= FIELD_PREP(DTHE_AES_CTRL_CCM_M_FIELD_MASK, | |
| ((ctx->authsize - 2) >> 1) & DTHE_AES_CCM_M_BITS); | |
| break; | |
| } | |
| if (iv_in) { | |
| ctrl_val |= DTHE_AES_CTRL_SAVE_CTX_SET; | |
| for (int i = 0; i < AES_IV_WORDS; ++i) | |
| writel_relaxed(iv_in[i], | |
| aes_base_reg + DTHE_P_AES_IV_IN_0 + (DTHE_REG_SIZE * i)); | |
| } | |
| writel_relaxed(ctrl_val, aes_base_reg + DTHE_P_AES_CTRL); | |
| } | |
| static int dthe_aes_do_fallback(struct skcipher_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); | |
| struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); | |
| SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb); | |
| skcipher_request_set_callback(subreq, skcipher_request_flags(req), | |
| req->base.complete, req->base.data); | |
| skcipher_request_set_crypt(subreq, req->src, req->dst, | |
| req->cryptlen, req->iv); | |
| return rctx->enc ? crypto_skcipher_encrypt(subreq) : | |
| crypto_skcipher_decrypt(subreq); | |
| } | |
| static void dthe_aes_dma_in_callback(void *data) | |
| { | |
| struct skcipher_request *req = (struct skcipher_request *)data; | |
| struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); | |
| complete(&rctx->aes_compl); | |
| } | |
| static int dthe_aes_run(struct crypto_engine *engine, void *areq) | |
| { | |
| struct skcipher_request *req = container_of(areq, struct skcipher_request, base); | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); | |
| unsigned int len = req->cryptlen; | |
| struct scatterlist *src = req->src; | |
| struct scatterlist *dst = req->dst; | |
| int src_nents = sg_nents_for_len(src, len); | |
| int dst_nents = sg_nents_for_len(dst, len); | |
| int src_mapped_nents; | |
| int dst_mapped_nents; | |
| bool diff_dst; | |
| enum dma_data_direction src_dir, dst_dir; | |
| struct device *tx_dev, *rx_dev; | |
| struct dma_async_tx_descriptor *desc_in, *desc_out; | |
| int ret; | |
| void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; | |
| u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); | |
| u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); | |
| aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; | |
| writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); | |
| aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; | |
| writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); | |
| if (src == dst) { | |
| diff_dst = false; | |
| src_dir = DMA_BIDIRECTIONAL; | |
| dst_dir = DMA_BIDIRECTIONAL; | |
| } else { | |
| diff_dst = true; | |
| src_dir = DMA_TO_DEVICE; | |
| dst_dir = DMA_FROM_DEVICE; | |
| } | |
| /* | |
| * CTR mode can operate on any input length, but the hardware | |
| * requires input length to be a multiple of the block size. | |
| * We need to handle the padding in the driver. | |
| */ | |
| if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) { | |
| unsigned int pad_size = AES_BLOCK_SIZE - (req->cryptlen % AES_BLOCK_SIZE); | |
| u8 *pad_buf = rctx->padding; | |
| struct scatterlist *sg; | |
| len += pad_size; | |
| src_nents++; | |
| dst_nents++; | |
| src = kmalloc_objs(*src, src_nents, GFP_ATOMIC); | |
| if (!src) { | |
| ret = -ENOMEM; | |
| goto aes_ctr_src_alloc_err; | |
| } | |
| sg_init_table(src, src_nents); | |
| sg = dthe_copy_sg(src, req->src, req->cryptlen); | |
| memzero_explicit(pad_buf, AES_BLOCK_SIZE); | |
| sg_set_buf(sg, pad_buf, pad_size); | |
| if (diff_dst) { | |
| dst = kmalloc_objs(*dst, dst_nents, GFP_ATOMIC); | |
| if (!dst) { | |
| ret = -ENOMEM; | |
| goto aes_ctr_dst_alloc_err; | |
| } | |
| sg_init_table(dst, dst_nents); | |
| sg = dthe_copy_sg(dst, req->dst, req->cryptlen); | |
| sg_set_buf(sg, pad_buf, pad_size); | |
| } else { | |
| dst = src; | |
| } | |
| } | |
| tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); | |
| rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); | |
| src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); | |
| if (src_mapped_nents == 0) { | |
| ret = -EINVAL; | |
| goto aes_map_src_err; | |
| } | |
| if (!diff_dst) { | |
| dst_mapped_nents = src_mapped_nents; | |
| } else { | |
| dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); | |
| if (dst_mapped_nents == 0) { | |
| ret = -EINVAL; | |
| goto aes_map_dst_err; | |
| } | |
| } | |
| desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, dst_mapped_nents, | |
| DMA_DEV_TO_MEM, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
| if (!desc_in) { | |
| dev_err(dev_data->dev, "IN prep_slave_sg() failed\n"); | |
| ret = -EINVAL; | |
| goto aes_prep_err; | |
| } | |
| desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, src_mapped_nents, | |
| DMA_MEM_TO_DEV, DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
| if (!desc_out) { | |
| dev_err(dev_data->dev, "OUT prep_slave_sg() failed\n"); | |
| ret = -EINVAL; | |
| goto aes_prep_err; | |
| } | |
| desc_in->callback = dthe_aes_dma_in_callback; | |
| desc_in->callback_param = req; | |
| init_completion(&rctx->aes_compl); | |
| if (ctx->aes_mode == DTHE_AES_ECB) | |
| dthe_aes_set_ctrl_key(ctx, rctx, NULL); | |
| else | |
| dthe_aes_set_ctrl_key(ctx, rctx, (u32 *)req->iv); | |
| writel_relaxed(lower_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_0); | |
| writel_relaxed(upper_32_bits(len), aes_base_reg + DTHE_P_AES_C_LENGTH_1); | |
| dmaengine_submit(desc_in); | |
| dmaengine_submit(desc_out); | |
| dma_async_issue_pending(dev_data->dma_aes_rx); | |
| dma_async_issue_pending(dev_data->dma_aes_tx); | |
| // Need to do a timeout to ensure finalise gets called if DMA callback fails for any reason | |
| ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); | |
| if (!ret) { | |
| ret = -ETIMEDOUT; | |
| dmaengine_terminate_sync(dev_data->dma_aes_rx); | |
| dmaengine_terminate_sync(dev_data->dma_aes_tx); | |
| for (int i = 0; i < AES_BLOCK_WORDS; ++i) | |
| readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + (DTHE_REG_SIZE * i)); | |
| } else { | |
| ret = 0; | |
| } | |
| // For modes other than ECB, read IV_OUT | |
| if (ctx->aes_mode != DTHE_AES_ECB) { | |
| u32 *iv_out = (u32 *)req->iv; | |
| for (int i = 0; i < AES_IV_WORDS; ++i) | |
| iv_out[i] = readl_relaxed(aes_base_reg + | |
| DTHE_P_AES_IV_IN_0 + | |
| (DTHE_REG_SIZE * i)); | |
| } | |
| aes_prep_err: | |
| if (dst_dir != DMA_BIDIRECTIONAL) | |
| dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); | |
| aes_map_dst_err: | |
| dma_unmap_sg(tx_dev, src, src_nents, src_dir); | |
| aes_map_src_err: | |
| if (ctx->aes_mode == DTHE_AES_CTR && req->cryptlen % AES_BLOCK_SIZE) { | |
| memzero_explicit(rctx->padding, AES_BLOCK_SIZE); | |
| if (diff_dst) | |
| kfree(dst); | |
| aes_ctr_dst_alloc_err: | |
| kfree(src); | |
| aes_ctr_src_alloc_err: | |
| /* | |
| * Fallback to software if ENOMEM | |
| */ | |
| if (ret == -ENOMEM) | |
| ret = dthe_aes_do_fallback(req); | |
| } | |
| local_bh_disable(); | |
| crypto_finalize_skcipher_request(dev_data->engine, req, ret); | |
| local_bh_enable(); | |
| return 0; | |
| } | |
| static int dthe_aes_crypt(struct skcipher_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| struct crypto_engine *engine; | |
| /* | |
| * If data is not a multiple of AES_BLOCK_SIZE: | |
| * - need to return -EINVAL for ECB, CBC as they are block ciphers | |
| * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS | |
| * - do nothing for CTR | |
| */ | |
| if (req->cryptlen % AES_BLOCK_SIZE) { | |
| if (ctx->aes_mode == DTHE_AES_XTS) | |
| return dthe_aes_do_fallback(req); | |
| if (ctx->aes_mode != DTHE_AES_CTR) | |
| return -EINVAL; | |
| } | |
| /* | |
| * If data length input is zero, no need to do any operation. | |
| * Except for XTS mode, where data length should be non-zero. | |
| */ | |
| if (req->cryptlen == 0) { | |
| if (ctx->aes_mode == DTHE_AES_XTS) | |
| return -EINVAL; | |
| return 0; | |
| } | |
| engine = dev_data->engine; | |
| return crypto_transfer_skcipher_request_to_engine(engine, req); | |
| } | |
| static int dthe_aes_encrypt(struct skcipher_request *req) | |
| { | |
| struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); | |
| rctx->enc = 1; | |
| return dthe_aes_crypt(req); | |
| } | |
| static int dthe_aes_decrypt(struct skcipher_request *req) | |
| { | |
| struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); | |
| rctx->enc = 0; | |
| return dthe_aes_crypt(req); | |
| } | |
| static int dthe_aead_init_tfm(struct crypto_aead *tfm) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| ctx->dev_data = dev_data; | |
| const char *alg_name = crypto_tfm_alg_name(crypto_aead_tfm(tfm)); | |
| ctx->aead_fb = crypto_alloc_sync_aead(alg_name, 0, | |
| CRYPTO_ALG_NEED_FALLBACK); | |
| if (IS_ERR(ctx->aead_fb)) { | |
| dev_err(dev_data->dev, "fallback driver %s couldn't be loaded\n", | |
| alg_name); | |
| return PTR_ERR(ctx->aead_fb); | |
| } | |
| return 0; | |
| } | |
| static void dthe_aead_exit_tfm(struct crypto_aead *tfm) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| crypto_free_sync_aead(ctx->aead_fb); | |
| } | |
| /** | |
| * dthe_aead_prep_aad - Prepare AAD scatterlist from input request | |
| * @sg: Input scatterlist containing AAD | |
| * @assoclen: Length of AAD | |
| * @pad_buf: Buffer to hold AAD padding if needed | |
| * | |
| * Description: | |
| * Creates a scatterlist containing only the AAD portion with padding | |
| * to align to AES_BLOCK_SIZE. This simplifies DMA handling by allowing | |
| * AAD to be sent separately via TX-only DMA. | |
| * | |
| * Return: | |
| * Pointer to the AAD scatterlist, or ERR_PTR(error) on failure. | |
| * The calling function needs to free the returned scatterlist when done. | |
| **/ | |
| static struct scatterlist *dthe_aead_prep_aad(struct scatterlist *sg, | |
| unsigned int assoclen, | |
| u8 *pad_buf) | |
| { | |
| struct scatterlist *aad_sg; | |
| struct scatterlist *to_sg; | |
| int aad_nents; | |
| if (assoclen == 0) | |
| return NULL; | |
| aad_nents = sg_nents_for_len(sg, assoclen); | |
| if (assoclen % AES_BLOCK_SIZE) | |
| aad_nents++; | |
| aad_sg = kmalloc_objs(struct scatterlist, aad_nents, GFP_ATOMIC); | |
| if (!aad_sg) | |
| return ERR_PTR(-ENOMEM); | |
| sg_init_table(aad_sg, aad_nents); | |
| to_sg = dthe_copy_sg(aad_sg, sg, assoclen); | |
| if (assoclen % AES_BLOCK_SIZE) { | |
| unsigned int pad_len = AES_BLOCK_SIZE - (assoclen % AES_BLOCK_SIZE); | |
| memset(pad_buf, 0, pad_len); | |
| sg_set_buf(to_sg, pad_buf, pad_len); | |
| } | |
| return aad_sg; | |
| } | |
| /** | |
| * dthe_aead_prep_crypt - Prepare crypt scatterlist from req->src/req->dst | |
| * @sg: Input req->src/req->dst scatterlist | |
| * @assoclen: Length of AAD (to skip) | |
| * @cryptlen: Length of ciphertext/plaintext (minus the size of TAG in decryption) | |
| * @pad_buf: Zeroed buffer to hold crypt padding if needed | |
| * | |
| * Description: | |
| * Creates a scatterlist containing only the ciphertext/plaintext portion | |
| * (skipping AAD) with padding to align to AES_BLOCK_SIZE. | |
| * | |
| * Return: | |
| * Pointer to the ciphertext scatterlist, or ERR_PTR(error) on failure. | |
| * The calling function needs to free the returned scatterlist when done. | |
| **/ | |
| static struct scatterlist *dthe_aead_prep_crypt(struct scatterlist *sg, | |
| unsigned int assoclen, | |
| unsigned int cryptlen, | |
| u8 *pad_buf) | |
| { | |
| struct scatterlist *out_sg[1]; | |
| struct scatterlist *crypt_sg; | |
| struct scatterlist *to_sg; | |
| size_t split_sizes[1] = {cryptlen}; | |
| int out_mapped_nents[1]; | |
| int crypt_nents; | |
| int err; | |
| if (cryptlen == 0) | |
| return NULL; | |
| /* Skip AAD, extract ciphertext portion */ | |
| err = sg_split(sg, 0, assoclen, 1, split_sizes, out_sg, out_mapped_nents, GFP_ATOMIC); | |
| if (err) | |
| goto dthe_aead_prep_crypt_split_err; | |
| crypt_nents = sg_nents_for_len(out_sg[0], cryptlen); | |
| if (cryptlen % AES_BLOCK_SIZE) | |
| crypt_nents++; | |
| crypt_sg = kmalloc_objs(struct scatterlist, crypt_nents, GFP_ATOMIC); | |
| if (!crypt_sg) { | |
| err = -ENOMEM; | |
| goto dthe_aead_prep_crypt_mem_err; | |
| } | |
| sg_init_table(crypt_sg, crypt_nents); | |
| to_sg = dthe_copy_sg(crypt_sg, out_sg[0], cryptlen); | |
| if (cryptlen % AES_BLOCK_SIZE) { | |
| unsigned int pad_len = AES_BLOCK_SIZE - (cryptlen % AES_BLOCK_SIZE); | |
| sg_set_buf(to_sg, pad_buf, pad_len); | |
| } | |
| dthe_aead_prep_crypt_mem_err: | |
| kfree(out_sg[0]); | |
| dthe_aead_prep_crypt_split_err: | |
| if (err) | |
| return ERR_PTR(err); | |
| return crypt_sg; | |
| } | |
| static int dthe_aead_read_tag(struct dthe_tfm_ctx *ctx, u32 *tag) | |
| { | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; | |
| u32 val; | |
| int ret; | |
| ret = readl_relaxed_poll_timeout(aes_base_reg + DTHE_P_AES_CTRL, val, | |
| (val & DTHE_AES_CTRL_SAVED_CTX_READY), | |
| 0, POLL_TIMEOUT_INTERVAL); | |
| if (ret) | |
| return ret; | |
| for (int i = 0; i < AES_BLOCK_WORDS; ++i) | |
| tag[i] = readl_relaxed(aes_base_reg + | |
| DTHE_P_AES_TAG_OUT + | |
| DTHE_REG_SIZE * i); | |
| return 0; | |
| } | |
| static int dthe_aead_enc_get_tag(struct aead_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
| u32 tag[AES_BLOCK_WORDS]; | |
| int nents; | |
| int ret; | |
| ret = dthe_aead_read_tag(ctx, tag); | |
| if (ret) | |
| return ret; | |
| nents = sg_nents_for_len(req->dst, req->cryptlen + req->assoclen + ctx->authsize); | |
| sg_pcopy_from_buffer(req->dst, nents, tag, ctx->authsize, | |
| req->assoclen + req->cryptlen); | |
| return 0; | |
| } | |
| static int dthe_aead_dec_verify_tag(struct aead_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
| u32 tag_out[AES_BLOCK_WORDS]; | |
| u32 tag_in[AES_BLOCK_WORDS]; | |
| int nents; | |
| int ret; | |
| ret = dthe_aead_read_tag(ctx, tag_out); | |
| if (ret) | |
| return ret; | |
| nents = sg_nents_for_len(req->src, req->assoclen + req->cryptlen); | |
| sg_pcopy_to_buffer(req->src, nents, tag_in, ctx->authsize, | |
| req->assoclen + req->cryptlen - ctx->authsize); | |
| if (crypto_memneq(tag_in, tag_out, ctx->authsize)) | |
| return -EBADMSG; | |
| else | |
| return 0; | |
| } | |
| static int dthe_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256) | |
| return -EINVAL; | |
| crypto_sync_aead_clear_flags(ctx->aead_fb, CRYPTO_TFM_REQ_MASK); | |
| crypto_sync_aead_set_flags(ctx->aead_fb, | |
| crypto_aead_get_flags(tfm) & | |
| CRYPTO_TFM_REQ_MASK); | |
| return crypto_sync_aead_setkey(ctx->aead_fb, key, keylen); | |
| } | |
| static int dthe_gcm_aes_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| int ret; | |
| ret = dthe_aead_setkey(tfm, key, keylen); | |
| if (ret) | |
| return ret; | |
| ctx->aes_mode = DTHE_AES_GCM; | |
| ctx->keylen = keylen; | |
| memcpy(ctx->key, key, keylen); | |
| return ret; | |
| } | |
| static int dthe_ccm_aes_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| int ret; | |
| ret = dthe_aead_setkey(tfm, key, keylen); | |
| if (ret) | |
| return ret; | |
| ctx->aes_mode = DTHE_AES_CCM; | |
| ctx->keylen = keylen; | |
| memcpy(ctx->key, key, keylen); | |
| return ret; | |
| } | |
| static int dthe_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(tfm); | |
| /* Invalid auth size will be handled by crypto_aead_setauthsize() */ | |
| ctx->authsize = authsize; | |
| return crypto_sync_aead_setauthsize(ctx->aead_fb, authsize); | |
| } | |
| static int dthe_aead_do_fallback(struct aead_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| SYNC_AEAD_REQUEST_ON_STACK(subreq, ctx->aead_fb); | |
| aead_request_set_callback(subreq, req->base.flags, | |
| req->base.complete, req->base.data); | |
| aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, req->iv); | |
| aead_request_set_ad(subreq, req->assoclen); | |
| return rctx->enc ? crypto_aead_encrypt(subreq) : | |
| crypto_aead_decrypt(subreq); | |
| } | |
| static void dthe_aead_dma_in_callback(void *data) | |
| { | |
| struct aead_request *req = (struct aead_request *)data; | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| complete(&rctx->aes_compl); | |
| } | |
| static int dthe_aead_run(struct crypto_engine *engine, void *areq) | |
| { | |
| struct aead_request *req = container_of(areq, struct aead_request, base); | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| unsigned int cryptlen = req->cryptlen; | |
| unsigned int assoclen = req->assoclen; | |
| unsigned int authsize = ctx->authsize; | |
| unsigned int unpadded_cryptlen; | |
| struct scatterlist *src = NULL; | |
| struct scatterlist *dst = NULL; | |
| struct scatterlist *aad_sg = NULL; | |
| u32 iv_in[AES_IV_WORDS]; | |
| int aad_nents = 0; | |
| int src_nents = 0; | |
| int dst_nents = 0; | |
| int aad_mapped_nents = 0; | |
| int src_mapped_nents = 0; | |
| int dst_mapped_nents = 0; | |
| u8 *src_assoc_padbuf = rctx->padding; | |
| u8 *src_crypt_padbuf = rctx->padding + AES_BLOCK_SIZE; | |
| u8 *dst_crypt_padbuf = rctx->padding + AES_BLOCK_SIZE; | |
| bool diff_dst; | |
| enum dma_data_direction aad_dir, src_dir, dst_dir; | |
| struct device *tx_dev, *rx_dev; | |
| struct dma_async_tx_descriptor *desc_in, *desc_out, *desc_aad_out; | |
| int ret; | |
| int err; | |
| void __iomem *aes_base_reg = dev_data->regs + DTHE_P_AES_BASE; | |
| u32 aes_irqenable_val = readl_relaxed(aes_base_reg + DTHE_P_AES_IRQENABLE); | |
| u32 aes_sysconfig_val = readl_relaxed(aes_base_reg + DTHE_P_AES_SYSCONFIG); | |
| aes_sysconfig_val |= DTHE_AES_SYSCONFIG_DMA_DATA_IN_OUT_EN; | |
| writel_relaxed(aes_sysconfig_val, aes_base_reg + DTHE_P_AES_SYSCONFIG); | |
| aes_irqenable_val |= DTHE_AES_IRQENABLE_EN_ALL; | |
| writel_relaxed(aes_irqenable_val, aes_base_reg + DTHE_P_AES_IRQENABLE); | |
| /* In decryption, the last authsize bytes are the TAG */ | |
| if (!rctx->enc) | |
| cryptlen -= authsize; | |
| unpadded_cryptlen = cryptlen; | |
| memset(src_assoc_padbuf, 0, AES_BLOCK_SIZE); | |
| memset(src_crypt_padbuf, 0, AES_BLOCK_SIZE); | |
| memset(dst_crypt_padbuf, 0, AES_BLOCK_SIZE); | |
| tx_dev = dmaengine_get_dma_device(dev_data->dma_aes_tx); | |
| rx_dev = dmaengine_get_dma_device(dev_data->dma_aes_rx); | |
| if (req->src == req->dst) { | |
| diff_dst = false; | |
| src_dir = DMA_BIDIRECTIONAL; | |
| dst_dir = DMA_BIDIRECTIONAL; | |
| } else { | |
| diff_dst = true; | |
| src_dir = DMA_TO_DEVICE; | |
| dst_dir = DMA_FROM_DEVICE; | |
| } | |
| aad_dir = DMA_TO_DEVICE; | |
| /* Prep AAD scatterlist (always from req->src) */ | |
| aad_sg = dthe_aead_prep_aad(req->src, req->assoclen, src_assoc_padbuf); | |
| if (IS_ERR(aad_sg)) { | |
| ret = PTR_ERR(aad_sg); | |
| goto aead_prep_aad_err; | |
| } | |
| /* Prep ciphertext src scatterlist */ | |
| src = dthe_aead_prep_crypt(req->src, req->assoclen, cryptlen, src_crypt_padbuf); | |
| if (IS_ERR(src)) { | |
| ret = PTR_ERR(src); | |
| goto aead_prep_src_err; | |
| } | |
| /* Prep ciphertext dst scatterlist (only if separate dst) */ | |
| if (diff_dst) { | |
| dst = dthe_aead_prep_crypt(req->dst, req->assoclen, unpadded_cryptlen, | |
| dst_crypt_padbuf); | |
| if (IS_ERR(dst)) { | |
| ret = PTR_ERR(dst); | |
| goto aead_prep_dst_err; | |
| } | |
| } else { | |
| dst = src; | |
| } | |
| /* Calculate padded lengths for nents calculations */ | |
| if (req->assoclen % AES_BLOCK_SIZE) | |
| assoclen += AES_BLOCK_SIZE - (req->assoclen % AES_BLOCK_SIZE); | |
| if (cryptlen % AES_BLOCK_SIZE) | |
| cryptlen += AES_BLOCK_SIZE - (cryptlen % AES_BLOCK_SIZE); | |
| if (assoclen != 0) { | |
| /* Map AAD for TX only */ | |
| aad_nents = sg_nents_for_len(aad_sg, assoclen); | |
| aad_mapped_nents = dma_map_sg(tx_dev, aad_sg, aad_nents, aad_dir); | |
| if (aad_mapped_nents == 0) { | |
| dev_err(dev_data->dev, "Failed to map AAD for TX\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_map_aad_err; | |
| } | |
| /* Prepare DMA descriptors for AAD TX */ | |
| desc_aad_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, aad_sg, | |
| aad_mapped_nents, DMA_MEM_TO_DEV, | |
| DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
| if (!desc_aad_out) { | |
| dev_err(dev_data->dev, "AAD TX prep_slave_sg() failed\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_prep_aad_err; | |
| } | |
| } | |
| if (cryptlen != 0) { | |
| /* Map ciphertext src for TX (BIDIRECTIONAL if in-place) */ | |
| src_nents = sg_nents_for_len(src, cryptlen); | |
| src_mapped_nents = dma_map_sg(tx_dev, src, src_nents, src_dir); | |
| if (src_mapped_nents == 0) { | |
| dev_err(dev_data->dev, "Failed to map ciphertext src for TX\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_prep_aad_err; | |
| } | |
| /* Prepare DMA descriptors for ciphertext TX */ | |
| desc_out = dmaengine_prep_slave_sg(dev_data->dma_aes_tx, src, | |
| src_mapped_nents, DMA_MEM_TO_DEV, | |
| DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
| if (!desc_out) { | |
| dev_err(dev_data->dev, "Ciphertext TX prep_slave_sg() failed\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_prep_src_err; | |
| } | |
| /* Map ciphertext dst for RX (only if separate dst) */ | |
| if (diff_dst) { | |
| dst_nents = sg_nents_for_len(dst, cryptlen); | |
| dst_mapped_nents = dma_map_sg(rx_dev, dst, dst_nents, dst_dir); | |
| if (dst_mapped_nents == 0) { | |
| dev_err(dev_data->dev, "Failed to map ciphertext dst for RX\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_prep_src_err; | |
| } | |
| } else { | |
| dst_nents = src_nents; | |
| dst_mapped_nents = src_mapped_nents; | |
| } | |
| /* Prepare DMA descriptor for ciphertext RX */ | |
| desc_in = dmaengine_prep_slave_sg(dev_data->dma_aes_rx, dst, | |
| dst_mapped_nents, DMA_DEV_TO_MEM, | |
| DMA_PREP_INTERRUPT | DMA_CTRL_ACK); | |
| if (!desc_in) { | |
| dev_err(dev_data->dev, "Ciphertext RX prep_slave_sg() failed\n"); | |
| ret = -EINVAL; | |
| goto aead_dma_prep_dst_err; | |
| } | |
| desc_in->callback = dthe_aead_dma_in_callback; | |
| desc_in->callback_param = req; | |
| } else if (assoclen != 0) { | |
| /* AAD-only operation */ | |
| desc_aad_out->callback = dthe_aead_dma_in_callback; | |
| desc_aad_out->callback_param = req; | |
| } | |
| init_completion(&rctx->aes_compl); | |
| /* | |
| * HACK: There is an unknown hw issue where if the previous operation had alen = 0 and | |
| * plen != 0, the current operation's tag calculation is incorrect in the case where | |
| * plen = 0 and alen != 0 currently. This is a workaround for now which somehow works; | |
| * by resetting the context by writing a 1 to the C_LENGTH_0 and AUTH_LENGTH registers. | |
| */ | |
| if (cryptlen == 0) { | |
| writel_relaxed(1, aes_base_reg + DTHE_P_AES_C_LENGTH_0); | |
| writel_relaxed(1, aes_base_reg + DTHE_P_AES_AUTH_LENGTH); | |
| } | |
| if (ctx->aes_mode == DTHE_AES_GCM) { | |
| if (req->iv) { | |
| memcpy(iv_in, req->iv, GCM_AES_IV_SIZE); | |
| } else { | |
| iv_in[0] = 0; | |
| iv_in[1] = 0; | |
| iv_in[2] = 0; | |
| } | |
| iv_in[3] = 0x01000000; | |
| } else { | |
| memcpy(iv_in, req->iv, AES_IV_SIZE); | |
| } | |
| /* Clear key2 to reset previous GHASH intermediate data */ | |
| for (int i = 0; i < AES_KEYSIZE_256 / sizeof(u32); ++i) | |
| writel_relaxed(0, aes_base_reg + DTHE_P_AES_KEY2_6 + DTHE_REG_SIZE * i); | |
| dthe_aes_set_ctrl_key(ctx, rctx, iv_in); | |
| writel_relaxed(lower_32_bits(unpadded_cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_0); | |
| writel_relaxed(upper_32_bits(unpadded_cryptlen), aes_base_reg + DTHE_P_AES_C_LENGTH_1); | |
| writel_relaxed(req->assoclen, aes_base_reg + DTHE_P_AES_AUTH_LENGTH); | |
| /* Submit DMA descriptors: AAD TX, ciphertext TX, ciphertext RX */ | |
| if (assoclen != 0) | |
| dmaengine_submit(desc_aad_out); | |
| if (cryptlen != 0) { | |
| dmaengine_submit(desc_out); | |
| dmaengine_submit(desc_in); | |
| } | |
| if (cryptlen != 0) | |
| dma_async_issue_pending(dev_data->dma_aes_rx); | |
| dma_async_issue_pending(dev_data->dma_aes_tx); | |
| /* Need to do timeout to ensure finalise gets called if DMA callback fails for any reason */ | |
| ret = wait_for_completion_timeout(&rctx->aes_compl, msecs_to_jiffies(DTHE_DMA_TIMEOUT_MS)); | |
| if (!ret) { | |
| ret = -ETIMEDOUT; | |
| if (cryptlen != 0) | |
| dmaengine_terminate_sync(dev_data->dma_aes_rx); | |
| dmaengine_terminate_sync(dev_data->dma_aes_tx); | |
| for (int i = 0; i < AES_BLOCK_WORDS; ++i) | |
| readl_relaxed(aes_base_reg + DTHE_P_AES_DATA_IN_OUT + DTHE_REG_SIZE * i); | |
| } else { | |
| ret = 0; | |
| } | |
| if (cryptlen != 0) | |
| dma_sync_sg_for_cpu(rx_dev, dst, dst_nents, dst_dir); | |
| if (rctx->enc) | |
| err = dthe_aead_enc_get_tag(req); | |
| else | |
| err = dthe_aead_dec_verify_tag(req); | |
| ret = (ret) ? ret : err; | |
| aead_dma_prep_dst_err: | |
| if (diff_dst && cryptlen != 0) | |
| dma_unmap_sg(rx_dev, dst, dst_nents, dst_dir); | |
| aead_dma_prep_src_err: | |
| if (cryptlen != 0) | |
| dma_unmap_sg(tx_dev, src, src_nents, src_dir); | |
| aead_dma_prep_aad_err: | |
| if (assoclen != 0) | |
| dma_unmap_sg(tx_dev, aad_sg, aad_nents, aad_dir); | |
| aead_dma_map_aad_err: | |
| if (diff_dst && cryptlen != 0) | |
| kfree(dst); | |
| aead_prep_dst_err: | |
| if (cryptlen != 0) | |
| kfree(src); | |
| aead_prep_src_err: | |
| if (assoclen != 0) | |
| kfree(aad_sg); | |
| aead_prep_aad_err: | |
| memzero_explicit(rctx->padding, 2 * AES_BLOCK_SIZE); | |
| if (ret) | |
| ret = dthe_aead_do_fallback(req); | |
| local_bh_disable(); | |
| crypto_finalize_aead_request(engine, req, ret); | |
| local_bh_enable(); | |
| return 0; | |
| } | |
| static int dthe_aead_crypt(struct aead_request *req) | |
| { | |
| struct dthe_tfm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| struct dthe_data *dev_data = dthe_get_dev(ctx); | |
| struct crypto_engine *engine; | |
| unsigned int cryptlen = req->cryptlen; | |
| bool is_zero_ctr = true; | |
| /* In decryption, last authsize bytes are the TAG */ | |
| if (!rctx->enc) | |
| cryptlen -= ctx->authsize; | |
| if (ctx->aes_mode == DTHE_AES_CCM) { | |
| /* | |
| * For CCM Mode, the 128-bit IV contains the following: | |
| * | 0 .. 2 | 3 .. 7 | 8 .. (127-8*L) | (128-8*L) .. 127 | | |
| * | L-1 | Zero | Nonce | Counter | | |
| * L needs to be between 2-8 (inclusive), i.e. 1 <= (L-1) <= 7 | |
| * and the next 5 bits need to be zeroes. Else return -EINVAL | |
| */ | |
| u8 *iv = req->iv; | |
| u8 L = iv[0]; | |
| /* variable L stores L-1 here */ | |
| if (L < 1 || L > 7) | |
| return -EINVAL; | |
| /* | |
| * DTHEv2 HW can only work with zero initial counter in CCM mode. | |
| * Check if the initial counter value is zero or not | |
| */ | |
| for (int i = 0; i < L + 1; ++i) { | |
| if (iv[AES_IV_SIZE - 1 - i] != 0) { | |
| is_zero_ctr = false; | |
| break; | |
| } | |
| } | |
| } | |
| /* | |
| * Need to fallback to software in the following cases due to HW restrictions: | |
| * - Both AAD and plaintext/ciphertext are zero length | |
| * - For AES-GCM, AAD length is more than 2^32 - 1 bytes | |
| * - For AES-CCM, AAD length is more than 2^16 - 2^8 bytes | |
| * - For AES-CCM, plaintext/ciphertext length is more than 2^61 - 1 bytes | |
| * - For AES-CCM, AAD length is non-zero but plaintext/ciphertext length is zero | |
| * - For AES-CCM, the initial counter (last L+1 bytes of IV) is not all zeroes | |
| * | |
| * PS: req->cryptlen is currently unsigned int type, which causes the second and fourth | |
| * cases above tautologically false. If req->cryptlen is to be changed to a 64-bit | |
| * type, the check for these would also need to be added below. | |
| */ | |
| if ((req->assoclen == 0 && cryptlen == 0) || | |
| (ctx->aes_mode == DTHE_AES_CCM && req->assoclen > DTHE_AES_CCM_AAD_MAXLEN) || | |
| (ctx->aes_mode == DTHE_AES_CCM && cryptlen == 0) || | |
| (ctx->aes_mode == DTHE_AES_CCM && !is_zero_ctr)) | |
| return dthe_aead_do_fallback(req); | |
| engine = dev_data->engine; | |
| return crypto_transfer_aead_request_to_engine(engine, req); | |
| } | |
| static int dthe_aead_encrypt(struct aead_request *req) | |
| { | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| rctx->enc = 1; | |
| return dthe_aead_crypt(req); | |
| } | |
| static int dthe_aead_decrypt(struct aead_request *req) | |
| { | |
| struct dthe_aes_req_ctx *rctx = aead_request_ctx(req); | |
| rctx->enc = 0; | |
| return dthe_aead_crypt(req); | |
| } | |
| static struct skcipher_engine_alg cipher_algs[] = { | |
| { | |
| .base.init = dthe_cipher_init_tfm, | |
| .base.setkey = dthe_aes_ecb_setkey, | |
| .base.encrypt = dthe_aes_encrypt, | |
| .base.decrypt = dthe_aes_decrypt, | |
| .base.min_keysize = AES_MIN_KEY_SIZE, | |
| .base.max_keysize = AES_MAX_KEY_SIZE, | |
| .base.base = { | |
| .cra_name = "ecb(aes)", | |
| .cra_driver_name = "ecb-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY, | |
| .cra_alignmask = AES_BLOCK_SIZE - 1, | |
| .cra_blocksize = AES_BLOCK_SIZE, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aes_run, | |
| }, /* ECB AES */ | |
| { | |
| .base.init = dthe_cipher_init_tfm, | |
| .base.setkey = dthe_aes_cbc_setkey, | |
| .base.encrypt = dthe_aes_encrypt, | |
| .base.decrypt = dthe_aes_decrypt, | |
| .base.min_keysize = AES_MIN_KEY_SIZE, | |
| .base.max_keysize = AES_MAX_KEY_SIZE, | |
| .base.ivsize = AES_IV_SIZE, | |
| .base.base = { | |
| .cra_name = "cbc(aes)", | |
| .cra_driver_name = "cbc-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY, | |
| .cra_alignmask = AES_BLOCK_SIZE - 1, | |
| .cra_blocksize = AES_BLOCK_SIZE, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aes_run, | |
| }, /* CBC AES */ | |
| { | |
| .base.init = dthe_cipher_init_tfm_fallback, | |
| .base.exit = dthe_cipher_exit_tfm, | |
| .base.setkey = dthe_aes_ctr_setkey, | |
| .base.encrypt = dthe_aes_encrypt, | |
| .base.decrypt = dthe_aes_decrypt, | |
| .base.min_keysize = AES_MIN_KEY_SIZE, | |
| .base.max_keysize = AES_MAX_KEY_SIZE, | |
| .base.ivsize = AES_IV_SIZE, | |
| .base.chunksize = AES_BLOCK_SIZE, | |
| .base.base = { | |
| .cra_name = "ctr(aes)", | |
| .cra_driver_name = "ctr-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY | | |
| CRYPTO_ALG_NEED_FALLBACK, | |
| .cra_blocksize = 1, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aes_run, | |
| }, /* CTR AES */ | |
| { | |
| .base.init = dthe_cipher_init_tfm_fallback, | |
| .base.exit = dthe_cipher_exit_tfm, | |
| .base.setkey = dthe_aes_xts_setkey, | |
| .base.encrypt = dthe_aes_encrypt, | |
| .base.decrypt = dthe_aes_decrypt, | |
| .base.min_keysize = AES_MIN_KEY_SIZE * 2, | |
| .base.max_keysize = AES_MAX_KEY_SIZE * 2, | |
| .base.ivsize = AES_IV_SIZE, | |
| .base.base = { | |
| .cra_name = "xts(aes)", | |
| .cra_driver_name = "xts-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY | | |
| CRYPTO_ALG_NEED_FALLBACK, | |
| .cra_alignmask = AES_BLOCK_SIZE - 1, | |
| .cra_blocksize = AES_BLOCK_SIZE, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aes_run, | |
| }, /* XTS AES */ | |
| }; | |
| static struct aead_engine_alg aead_algs[] = { | |
| { | |
| .base.init = dthe_aead_init_tfm, | |
| .base.exit = dthe_aead_exit_tfm, | |
| .base.setkey = dthe_gcm_aes_setkey, | |
| .base.setauthsize = dthe_aead_setauthsize, | |
| .base.maxauthsize = AES_BLOCK_SIZE, | |
| .base.encrypt = dthe_aead_encrypt, | |
| .base.decrypt = dthe_aead_decrypt, | |
| .base.chunksize = AES_BLOCK_SIZE, | |
| .base.ivsize = GCM_AES_IV_SIZE, | |
| .base.base = { | |
| .cra_name = "gcm(aes)", | |
| .cra_driver_name = "gcm-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_AEAD | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_NEED_FALLBACK, | |
| .cra_blocksize = 1, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aead_run, | |
| }, /* GCM AES */ | |
| { | |
| .base.init = dthe_aead_init_tfm, | |
| .base.exit = dthe_aead_exit_tfm, | |
| .base.setkey = dthe_ccm_aes_setkey, | |
| .base.setauthsize = dthe_aead_setauthsize, | |
| .base.maxauthsize = AES_BLOCK_SIZE, | |
| .base.encrypt = dthe_aead_encrypt, | |
| .base.decrypt = dthe_aead_decrypt, | |
| .base.chunksize = AES_BLOCK_SIZE, | |
| .base.ivsize = AES_IV_SIZE, | |
| .base.base = { | |
| .cra_name = "ccm(aes)", | |
| .cra_driver_name = "ccm-aes-dthev2", | |
| .cra_priority = 299, | |
| .cra_flags = CRYPTO_ALG_TYPE_AEAD | | |
| CRYPTO_ALG_KERN_DRIVER_ONLY | | |
| CRYPTO_ALG_ASYNC | | |
| CRYPTO_ALG_NEED_FALLBACK, | |
| .cra_blocksize = 1, | |
| .cra_ctxsize = sizeof(struct dthe_tfm_ctx), | |
| .cra_reqsize = sizeof(struct dthe_aes_req_ctx), | |
| .cra_module = THIS_MODULE, | |
| }, | |
| .op.do_one_request = dthe_aead_run, | |
| }, /* CCM AES */ | |
| }; | |
| int dthe_register_aes_algs(void) | |
| { | |
| int ret = 0; | |
| ret = crypto_engine_register_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); | |
| if (ret) | |
| return ret; | |
| ret = crypto_engine_register_aeads(aead_algs, ARRAY_SIZE(aead_algs)); | |
| if (ret) | |
| crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); | |
| return ret; | |
| } | |
| void dthe_unregister_aes_algs(void) | |
| { | |
| crypto_engine_unregister_skciphers(cipher_algs, ARRAY_SIZE(cipher_algs)); | |
| crypto_engine_unregister_aeads(aead_algs, ARRAY_SIZE(aead_algs)); | |
| } |