From ea52e07a9554e72ac8a6ff26eb529f14b3e66248 Mon Sep 17 00:00:00 2001 From: Berne Hebark Date: Wed, 25 May 2011 14:27:51 +0200 Subject: crypto: ux500: hash: DMA support - Support for DMA. - Direct to CPU mode for data size < 4 byte data. - Workaround to handle data sizes not modulo wordsize. - Error message/check for HMAC empty message with keysize > 0. - Error message/check for HMAC DMA for u5500, since not working. - Additional, update cryp driver dma code according to this patch and make minor adjustments to comply with mainline code and design are covered by AP370178. ST-Ericsson ID: 280691 ST-Ericsson Linux next: Not tested, ER 320876 ST-Ericsson FOSS-OUT ID: Trivial Change-Id: I14d64d1577f007969b372ed4ef04556eca8bc0d6 Signed-off-by: Berne Hebark Reviewed-on: http://gerrit.lud.stericsson.com/gerrit/33983 Reviewed-by: Per FORLIN Reviewed-by: QATOOLS Reviewed-by: QABUILD --- arch/arm/mach-ux500/include/mach/crypto-ux500.h | 4 +- drivers/crypto/ux500/hash/hash_alg.h | 36 +- drivers/crypto/ux500/hash/hash_core.c | 602 +++++++++++++++++------- 3 files changed, 470 insertions(+), 172 deletions(-) diff --git a/arch/arm/mach-ux500/include/mach/crypto-ux500.h b/arch/arm/mach-ux500/include/mach/crypto-ux500.h index 9d1e1c52c13..80c4620d633 100644 --- a/arch/arm/mach-ux500/include/mach/crypto-ux500.h +++ b/arch/arm/mach-ux500/include/mach/crypto-ux500.h @@ -5,6 +5,7 @@ * License terms: GNU General Public License (GPL) version 2 */ #ifndef _CRYPTO_UX500_H +#include #include struct cryp_platform_data { @@ -13,7 +14,8 @@ struct cryp_platform_data { }; struct hash_platform_data { - struct stedma40_chan_cfg mem_to_engine; + void *mem_to_engine; + bool (*dma_filter)(struct dma_chan *chan, void *filter_param); }; #endif diff --git a/drivers/crypto/ux500/hash/hash_alg.h b/drivers/crypto/ux500/hash/hash_alg.h index 299f0bacc2c..61db5b511b6 100644 --- a/drivers/crypto/ux500/hash/hash_alg.h +++ b/drivers/crypto/ux500/hash/hash_alg.h @@ -11,6 +11,7 @@ #include #define HASH_BLOCK_SIZE 64 +#define HASH_DMA_ALIGN_SIZE 4 /* Maximum value of the length's high word */ #define HASH_HIGH_WORD_MAX_VAL 0xFFFFFFFFUL @@ -125,6 +126,12 @@ 0x01, HASH_STR_DCAL_POS, \ HASH_STR_DCAL_MASK) +/* Hardware access method */ +enum hash_mode { + HASH_MODE_CPU, + HASH_MODE_DMA +}; + /** * struct uint64 - Structure to handle 64 bits integers. * @high_word: Most significant bits. @@ -286,6 +293,26 @@ struct hash_config { int oper_mode; }; +/** + * struct hash_dma - Structure used for dma. + * @mask: DMA capabilities bitmap mask. + * @complete: Used to maintain state for a "completion". + * @chan_mem2hash: DMA channel. + * @cfg_mem2hash: DMA channel configuration. + * @sg_len: Scatterlist length. + * @sg: Scatterlist. + * @nents: Number of sg entries. + */ +struct hash_dma { + dma_cap_mask_t mask; + struct completion complete; + struct dma_chan *chan_mem2hash; + void *cfg_mem2hash; + int sg_len; + struct scatterlist *sg; + int nents; +}; + /** * struct hash_ctx - The context used for hash calculations. * @key: The key used in the operation. @@ -293,8 +320,10 @@ struct hash_config { * @updated: Indicates if hardware is initialized for new operations. * @state: The state of the current calculations. * @config: The current configuration. - * @digestsize The size of current digest. - * @device Pointer to the device structure. + * @digestsize: The size of current digest. + * @device: Pointer to the device structure. + * @dma_mode: Used in special cases (workaround), e.g. need to change to + * cpu mode, if not supported/working in dma mode. */ struct hash_ctx { u8 *key; @@ -304,6 +333,7 @@ struct hash_ctx { struct hash_config config; int digestsize; struct hash_device_data *device; + bool dma_mode; }; /** @@ -318,6 +348,7 @@ struct hash_ctx { * @regulator: Pointer to the device's power control. * @clk: Pointer to the device's clock control. * @restore_dev_state: TRUE = saved state, FALSE = no saved state. + * @dma: Structure used for dma. */ struct hash_device_data { struct hash_register __iomem *base; @@ -330,6 +361,7 @@ struct hash_device_data { struct ux500_regulator *regulator; struct clk *clk; bool restore_dev_state; + struct hash_dma dma; }; int hash_check_hw(struct hash_device_data *device_data); diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index 08a89eeb601..b2a58dccf76 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -29,12 +30,17 @@ #include #include +#include #include #include "hash_alg.h" #define DEV_DBG_NAME "hashX hashX:" +static int hash_mode; +module_param(hash_mode, int, 0); +MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1"); + /** * Pre-calculated empty message digests. */ @@ -113,6 +119,101 @@ static void release_hash_device(struct hash_device_data *device_data) up(&driver_data.device_allocation); } +static void hash_dma_setup_channel(struct hash_device_data *device_data, + struct device *dev) +{ + struct hash_platform_data *platform_data = dev->platform_data; + dma_cap_zero(device_data->dma.mask); + dma_cap_set(DMA_SLAVE, device_data->dma.mask); + + device_data->dma.cfg_mem2hash = platform_data->mem_to_engine; + device_data->dma.chan_mem2hash = + dma_request_channel(device_data->dma.mask, + platform_data->dma_filter, + device_data->dma.cfg_mem2hash); + + init_completion(&device_data->dma.complete); +} + +static void hash_dma_callback(void *data) +{ + struct hash_ctx *ctx = (struct hash_ctx *) data; + + complete(&ctx->device->dma.complete); +} + +static int hash_set_dma_transfer(struct hash_ctx *ctx, struct scatterlist *sg, + int len, enum dma_data_direction direction) +{ + struct dma_async_tx_descriptor *desc = NULL; + struct dma_chan *channel = NULL; + dma_cookie_t cookie; + + if (direction != DMA_TO_DEVICE) { + dev_err(ctx->device->dev, "[%s] Invalid DMA direction", + __func__); + return -EFAULT; + } + + sg->length = ALIGN(sg->length, HASH_DMA_ALIGN_SIZE); + + channel = ctx->device->dma.chan_mem2hash; + ctx->device->dma.sg = sg; + ctx->device->dma.sg_len = dma_map_sg(channel->device->dev, + ctx->device->dma.sg, ctx->device->dma.nents, + direction); + + if (!ctx->device->dma.sg_len) { + dev_err(ctx->device->dev, + "[%s]: Could not map the sg list (TO_DEVICE)", + __func__); + return -EFAULT; + } + + dev_dbg(ctx->device->dev, "[%s]: Setting up DMA for buffer " + "(TO_DEVICE)", __func__); + desc = channel->device->device_prep_slave_sg(channel, + ctx->device->dma.sg, ctx->device->dma.sg_len, + direction, DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + if (!desc) { + dev_err(ctx->device->dev, + "[%s]: device_prep_slave_sg() failed!", __func__); + return -EFAULT; + } + + desc->callback = hash_dma_callback; + desc->callback_param = ctx; + + cookie = desc->tx_submit(desc); + dma_async_issue_pending(channel); + + return 0; +} + +static void hash_dma_done(struct hash_ctx *ctx) +{ + struct dma_chan *chan; + + chan = ctx->device->dma.chan_mem2hash; + chan->device->device_control(chan, DMA_TERMINATE_ALL, 0); + dma_unmap_sg(chan->device->dev, ctx->device->dma.sg, + ctx->device->dma.sg_len, DMA_TO_DEVICE); + +} + +static int hash_dma_write(struct hash_ctx *ctx, + struct scatterlist *sg, int len) +{ + int error = hash_set_dma_transfer(ctx, sg, len, DMA_TO_DEVICE); + if (error) { + dev_dbg(ctx->device->dev, "[%s]: hash_set_dma_transfer() " + "failed", __func__); + return error; + } + + return len; +} + /** * get_empty_message_digest - Returns a pre-calculated digest for * the empty message. @@ -197,8 +298,6 @@ static int hash_disable_power( int ret = 0; struct device *dev = device_data->dev; - dev_dbg(dev, "[%s]", __func__); - spin_lock(&device_data->power_state_lock); if (!device_data->power_state) goto out; @@ -236,7 +335,6 @@ static int hash_enable_power( { int ret = 0; struct device *dev = device_data->dev; - dev_dbg(dev, "[%s]", __func__); spin_lock(&device_data->power_state_lock); if (!device_data->power_state) { @@ -287,8 +385,6 @@ static int hash_get_device_data(struct hash_ctx *ctx, struct klist_node *device_node; struct hash_device_data *local_device_data = NULL; - pr_debug(DEV_DBG_NAME " [%s]", __func__); - /* Wait until a device is available */ ret = down_interruptible(&driver_data.device_allocation); if (ret) @@ -390,8 +486,6 @@ static int init_hash_hw(struct hash_device_data *device_data, { int ret = 0; - dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32)ctx); - ret = hash_setconfiguration(device_data, &ctx->config); if (ret) { dev_err(device_data->dev, "[%s] hash_setconfiguration() " @@ -407,6 +501,61 @@ static int init_hash_hw(struct hash_device_data *device_data, return ret; } +/** + * hash_get_nents - Return number of entries (nents) in scatterlist (sg). + * + * @sg: Scatterlist. + * @size: Size in bytes. + * @aligned: True if sg data aligned to work in DMA mode. + * + * Reentrancy: Non Re-entrant + */ +static int hash_get_nents(struct scatterlist *sg, int size, bool *aligned) +{ + int nents = 0; + bool aligned_data = true; + + while (size > 0 && sg) { + nents++; + size -= sg->length; + + /* hash_set_dma_transfer will align last nent */ + if (aligned && !IS_ALIGNED(sg->offset, HASH_DMA_ALIGN_SIZE) || + (!IS_ALIGNED(sg->length, HASH_DMA_ALIGN_SIZE) && + size > 0)) + aligned_data = false; + + sg = sg_next(sg); + } + + if (aligned) + *aligned = aligned_data; + + if (size != 0) + return -EFAULT; + + return nents; +} + +/** + * hash_dma_valid_data - checks for dma valid sg data. + * @sg: Scatterlist. + * @datasize: Datasize in bytes. + * + * NOTE! This function checks for dma valid sg data, since dma + * only accept datasizes of even wordsize. + */ +static bool hash_dma_valid_data(struct scatterlist *sg, int datasize) +{ + bool aligned; + + /* Need to include at least one nent, else error */ + if (hash_get_nents(sg, datasize, &aligned) < 1) + return false; + + return aligned; +} + /** * hash_init - Common hash init function for SHA1/SHA2 (SHA256). * @req: The hash request for the job. @@ -418,13 +567,39 @@ static int hash_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s] data size: %d", __func__, req->nbytes); - if (!ctx->key) ctx->keylen = 0; memset(&ctx->state, 0, sizeof(struct hash_state)); ctx->updated = 0; + if (hash_mode == HASH_MODE_DMA) { + if ((ctx->config.oper_mode == HASH_OPER_MODE_HMAC) && + cpu_is_u5500()) { + pr_debug(DEV_DBG_NAME " [%s] HMAC and DMA not working " + "on u5500, directing to CPU mode.", + __func__); + ctx->dma_mode = false; /* Don't use DMA in this case */ + goto out; + } + + if (req->nbytes < HASH_DMA_ALIGN_SIZE) { + ctx->dma_mode = false; /* Don't use DMA in this case */ + + pr_debug(DEV_DBG_NAME " [%s] DMA mode, but direct " + "to CPU mode for data size < %d", + __func__, HASH_DMA_ALIGN_SIZE); + } else { + if (hash_dma_valid_data(req->src, req->nbytes)) { + ctx->dma_mode = true; + } else { + ctx->dma_mode = false; + pr_debug(DEV_DBG_NAME " [%s] DMA mode, but " + "direct to CPU mode for " + "non-aligned data", __func__); + } + } + } +out: return 0; } @@ -474,9 +649,6 @@ static void hash_processblock( static void hash_messagepad(struct hash_device_data *device_data, const u32 *message, u8 index_bytes) { - dev_dbg(device_data->dev, "[%s] (bytes in final msg=%d))", - __func__, index_bytes); - /* * Clear hash str register, only clear NBLW * since DCAL will be reset by hardware. @@ -561,7 +733,6 @@ int hash_setconfiguration(struct hash_device_data *device_data, struct hash_config *config) { int ret = 0; - dev_dbg(device_data->dev, "[%s] ", __func__); if (config->algorithm != HASH_ALGO_SHA1 && config->algorithm != HASH_ALGO_SHA256) @@ -573,13 +744,6 @@ int hash_setconfiguration(struct hash_device_data *device_data, */ HASH_SET_DATA_FORMAT(config->data_format); - /* - * Empty message bit. This bit is needed when the hash input data - * contain the empty message. Always set in current impl. but with - * no impact on data different than empty message. - */ - HASH_SET_BITS(&device_data->base->cr, HASH_CR_EMPTYMSG_MASK); - /* * ALGO bit. Set to 0b1 for SHA-1 and 0b0 for SHA-256 */ @@ -652,7 +816,6 @@ void hash_begin(struct hash_device_data *device_data, struct hash_ctx *ctx) { /* HW and SW initializations */ /* Note: there is no need to initialize buffer and digest members */ - dev_dbg(device_data->dev, "[%s] ", __func__); while (device_data->base->str & HASH_STR_DCAL_MASK) cpu_relax(); @@ -688,6 +851,7 @@ int hash_process_data( msg_length = 0; } else { if (ctx->updated) { + ret = hash_resume_state(device_data, &ctx->state); if (ret) { @@ -696,7 +860,6 @@ int hash_process_data( " failed!", __func__); goto out; } - } else { ret = init_hash_hw(device_data, ctx); if (ret) { @@ -732,6 +895,7 @@ int hash_process_data( } hash_incrementlength(ctx, HASH_BLOCK_SIZE); data_buffer += (HASH_BLOCK_SIZE - *index); + msg_length -= (HASH_BLOCK_SIZE - *index); *index = 0; @@ -750,6 +914,236 @@ out: return ret; } +/** + * hash_dma_final - The hash dma final function for SHA1/SHA256. + * @req: The hash request for the job. + */ +static int hash_dma_final(struct ahash_request *req) +{ + int ret = 0; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_device_data *device_data; + u8 digest[SHA256_DIGEST_SIZE]; + int bytes_written = 0; + + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx); + + /* Enable device power (and clock) */ + ret = hash_enable_power(device_data, false); + if (ret) { + dev_err(device_data->dev, "[%s]: " + "hash_enable_power() failed!", __func__); + goto out; + } + + if (ctx->updated) { + ret = hash_resume_state(device_data, &ctx->state); + + if (ret) { + dev_err(device_data->dev, "[%s] hash_resume_state() " + "failed!", __func__); + goto out_power; + } + + } + + if (!ctx->updated) { + ret = hash_setconfiguration(device_data, &ctx->config); + if (ret) { + dev_err(device_data->dev, "[%s] " + "hash_setconfiguration() failed!", + __func__); + goto out_power; + } + + /* Enable DMA input */ + if (hash_mode != HASH_MODE_DMA || !ctx->dma_mode) { + HASH_CLEAR_BITS(&device_data->base->cr, + HASH_CR_DMAE_MASK); + } else { + HASH_SET_BITS(&device_data->base->cr, + HASH_CR_DMAE_MASK); + HASH_SET_BITS(&device_data->base->cr, + HASH_CR_PRIVN_MASK); + } + + HASH_INITIALIZE; + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC) + hash_hw_write_key(device_data, ctx->key, ctx->keylen); + + /* Number of bits in last word = (nbytes * 8) % 32 */ + HASH_SET_NBLW((req->nbytes * 8) % 32); + ctx->updated = 1; + } + + /* Store the nents in the dma struct. */ + ctx->device->dma.nents = hash_get_nents(req->src, req->nbytes, NULL); + if (!ctx->device->dma.nents) { + dev_err(device_data->dev, "[%s] " + "ctx->device->dma.nents = 0", __func__); + goto out_power; + } + + bytes_written = hash_dma_write(ctx, req->src, req->nbytes); + if (bytes_written != req->nbytes) { + dev_err(device_data->dev, "[%s] " + "hash_dma_write() failed!", __func__); + goto out_power; + } + + wait_for_completion(&ctx->device->dma.complete); + hash_dma_done(ctx); + + while (device_data->base->str & HASH_STR_DCAL_MASK) + cpu_relax(); + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) { + unsigned int keylen = ctx->keylen; + u8 *key = ctx->key; + + dev_dbg(device_data->dev, "[%s] keylen: %d", __func__, + ctx->keylen); + hash_hw_write_key(device_data, key, keylen); + } + + hash_get_digest(device_data, digest, ctx->config.algorithm); + memcpy(req->result, digest, ctx->digestsize); + +out_power: + /* Disable power (and clock) */ + if (hash_disable_power(device_data, false)) + dev_err(device_data->dev, "[%s] hash_disable_power() failed!", + __func__); + +out: + release_hash_device(device_data); + + /** + * Allocated in setkey, and only used in HMAC. + */ + kfree(ctx->key); + + return ret; +} + +/** + * hash_hw_final - The final hash calculation function + * @req: The hash request for the job. + */ +int hash_hw_final(struct ahash_request *req) +{ + int ret = 0; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); + struct hash_device_data *device_data; + u8 digest[SHA256_DIGEST_SIZE]; + + ret = hash_get_device_data(ctx, &device_data); + if (ret) + return ret; + + dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx); + + /* Enable device power (and clock) */ + ret = hash_enable_power(device_data, false); + if (ret) { + dev_err(device_data->dev, "[%s]: " + "hash_enable_power() failed!", __func__); + goto out; + } + + if (ctx->updated) { + ret = hash_resume_state(device_data, &ctx->state); + + if (ret) { + dev_err(device_data->dev, "[%s] hash_resume_state() " + "failed!", __func__); + goto out_power; + } + } else if (req->nbytes == 0 && ctx->keylen == 0) { + u8 zero_hash[SHA256_DIGEST_SIZE]; + u32 zero_hash_size = 0; + bool zero_digest = false; + /** + * Use a pre-calculated empty message digest + * (workaround since hw return zeroes, hw bug!?) + */ + ret = get_empty_message_digest(device_data, &zero_hash[0], + &zero_hash_size, &zero_digest); + if (!ret && likely(zero_hash_size == ctx->digestsize) && + zero_digest) { + memcpy(req->result, &zero_hash[0], ctx->digestsize); + goto out_power; + } else if (!ret && !zero_digest) { + dev_dbg(device_data->dev, "[%s] HMAC zero msg with " + "key, continue...", __func__); + } else { + dev_err(device_data->dev, "[%s] ret=%d, or wrong " + "digest size? %s", __func__, ret, + (zero_hash_size == ctx->digestsize) ? + "true" : "false"); + /* Return error */ + goto out_power; + } + } else if (req->nbytes == 0 && ctx->keylen > 0) { + dev_err(device_data->dev, "[%s] Empty message with " + "keylength > 0, NOT supported.", __func__); + goto out_power; + } + + if (!ctx->updated) { + ret = init_hash_hw(device_data, ctx); + if (ret) { + dev_err(device_data->dev, "[%s] init_hash_hw() " + "failed!", __func__); + goto out_power; + } + } + + if (ctx->state.index) { + hash_messagepad(device_data, ctx->state.buffer, + ctx->state.index); + } else { + HASH_SET_DCAL; + while (device_data->base->str & HASH_STR_DCAL_MASK) + cpu_relax(); + } + + if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) { + unsigned int keylen = ctx->keylen; + u8 *key = ctx->key; + + dev_dbg(device_data->dev, "[%s] keylen: %d", __func__, + ctx->keylen); + hash_hw_write_key(device_data, key, keylen); + } + + hash_get_digest(device_data, digest, ctx->config.algorithm); + memcpy(req->result, digest, ctx->digestsize); + +out_power: + /* Disable power (and clock) */ + if (hash_disable_power(device_data, false)) + dev_err(device_data->dev, "[%s] hash_disable_power() failed!", + __func__); + +out: + release_hash_device(device_data); + + /** + * Allocated in setkey, and only used in HMAC. + */ + kfree(ctx->key); + + return ret; +} + /** * hash_hw_update - Updates current HASH computation hashing another part of * the message. @@ -770,8 +1164,6 @@ int hash_hw_update(struct ahash_request *req) struct crypto_hash_walk walk; int msg_length = crypto_hash_walk_first(req, &walk); - pr_debug(DEV_DBG_NAME " [%s] datalength: %d", __func__, msg_length); - /* Empty message ("") is correct indata */ if (msg_length == 0) return ret; @@ -818,9 +1210,9 @@ int hash_hw_update(struct ahash_request *req) } ctx->state.index = index; - dev_dbg(device_data->dev, "[%s] indata length=%d, " - "bin=%d))", __func__, ctx->state.index, ctx->state.bit_index); + "bin=%d))", __func__, ctx->state.index, ctx->state.bit_index); + out_power: /* Disable power (and clock) */ if (hash_disable_power(device_data, false)) @@ -846,9 +1238,6 @@ int hash_resume_state(struct hash_device_data *device_data, s32 count; int hash_mode = HASH_OPER_MODE_HASH; - dev_dbg(device_data->dev, "[%s] (state(0x%x)))", - __func__, (u32) device_state); - if (NULL == device_state) { dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!", __func__); @@ -909,9 +1298,6 @@ int hash_save_state(struct hash_device_data *device_data, u32 count; int hash_mode = HASH_OPER_MODE_HASH; - dev_dbg(device_data->dev, "[%s] state(0x%x)))", - __func__, (u32) device_state); - if (NULL == device_state) { dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!", __func__); @@ -961,8 +1347,6 @@ int hash_check_hw(struct hash_device_data *device_data) { int ret = 0; - dev_dbg(device_data->dev, "[%s] ", __func__); - if (NULL == device_data) { ret = -EPERM; dev_err(device_data->dev, "[%s] HASH_INVALID_PARAMETER!", @@ -1041,17 +1425,18 @@ void hash_get_digest(struct hash_device_data *device_data, static int ahash_update(struct ahash_request *req) { int ret = 0; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s] ", __func__); + if (hash_mode != HASH_MODE_DMA || !ctx->dma_mode) + ret = hash_hw_update(req); + /* Skip update for DMA, all data will be passed to DMA in final */ - ret = hash_hw_update(req); if (ret) { pr_err(DEV_DBG_NAME " [%s] hash_hw_update() failed!", __func__); - goto out; } -out: return ret; } @@ -1064,103 +1449,18 @@ static int ahash_final(struct ahash_request *req) int ret = 0; struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - struct hash_device_data *device_data; - u8 digest[SHA256_DIGEST_SIZE]; - - pr_debug(DEV_DBG_NAME " [%s] ", __func__); - ret = hash_get_device_data(ctx, &device_data); - if (ret) - return ret; + pr_debug(DEV_DBG_NAME " [%s] data size: %d", __func__, req->nbytes); - dev_dbg(device_data->dev, "[%s] (ctx=0x%x)!", __func__, (u32) ctx); + if ((hash_mode == HASH_MODE_DMA) && ctx->dma_mode) + ret = hash_dma_final(req); + else + ret = hash_hw_final(req); - /* Enable device power (and clock) */ - ret = hash_enable_power(device_data, false); if (ret) { - dev_err(device_data->dev, "[%s]: " - "hash_enable_power() failed!", __func__); - goto out; - } - - if (ctx->updated) { - ret = hash_resume_state(device_data, &ctx->state); - - if (ret) { - dev_err(device_data->dev, "[%s] hash_resume_state() " - "failed!", __func__); - goto out_power; - } - } else if (req->nbytes == 0 && ctx->keylen == 0) { - u8 zero_hash[SHA256_DIGEST_SIZE]; - u32 zero_hash_size = 0; - bool zero_digest = false; - /** - * Use a pre-calculated empty message digest - * (workaround since hw return zeroes, hw bug!?) - */ - ret = get_empty_message_digest(device_data, &zero_hash[0], - &zero_hash_size, &zero_digest); - if (!ret && likely(zero_hash_size == ctx->digestsize) && - zero_digest) { - memcpy(req->result, &zero_hash[0], ctx->digestsize); - goto out_power; - } else if (!ret && !zero_digest) { - dev_dbg(device_data->dev, "[%s] HMAC zero msg with " - "key, continue...", __func__); - } else { - dev_err(device_data->dev, "[%s] ret=%d, or wrong " - "digest size? %s", __func__, ret, - (zero_hash_size == ctx->digestsize) ? - "true" : "false"); - /* Return error */ - goto out_power; - } - } - - if (!ctx->updated) { - ret = init_hash_hw(device_data, ctx); - if (ret) { - dev_err(device_data->dev, "[%s] init_hash_hw() " - "failed!", __func__); - goto out_power; - } - } - - if (ctx->state.index) { - hash_messagepad(device_data, ctx->state.buffer, - ctx->state.index); - } else { - HASH_SET_DCAL; - while (device_data->base->str & HASH_STR_DCAL_MASK) - cpu_relax(); - } - - if (ctx->config.oper_mode == HASH_OPER_MODE_HMAC && ctx->key) { - unsigned int keylen = ctx->keylen; - u8 *key = ctx->key; - - dev_dbg(device_data->dev, "[%s] keylen: %d", __func__, - ctx->keylen); - hash_hw_write_key(device_data, key, keylen); - } - - hash_get_digest(device_data, digest, ctx->config.algorithm); - memcpy(req->result, digest, ctx->digestsize); - -out_power: - /* Disable power (and clock) */ - if (hash_disable_power(device_data, false)) - dev_err(device_data->dev, "[%s] hash_disable_power() failed!", + pr_err(DEV_DBG_NAME " [%s] hash_hw/dma_final() failed", __func__); - -out: - release_hash_device(device_data); - - /** - * Allocated in setkey, and only used in HMAC. - */ - kfree(ctx->key); + } return ret; } @@ -1171,8 +1471,6 @@ static int hash_setkey(struct crypto_ahash *tfm, int ret = 0; struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s] keylen: %d", __func__, keylen); - /** * Freed in final. */ @@ -1194,8 +1492,6 @@ static int ahash_sha1_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s]: (ctx=0x%x)!", __func__, (u32) ctx); - ctx->config.data_format = HASH_DATA_8_BITS; ctx->config.algorithm = HASH_ALGO_SHA1; ctx->config.oper_mode = HASH_OPER_MODE_HASH; @@ -1209,8 +1505,6 @@ static int ahash_sha256_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s]: (ctx=0x%x)!", __func__, (u32) ctx); - ctx->config.data_format = HASH_DATA_8_BITS; ctx->config.algorithm = HASH_ALGO_SHA256; ctx->config.oper_mode = HASH_OPER_MODE_HASH; @@ -1223,8 +1517,6 @@ static int ahash_sha1_digest(struct ahash_request *req) { int ret2, ret1; - pr_debug(DEV_DBG_NAME " [%s]", __func__); - ret1 = ahash_sha1_init(req); if (ret1) goto out; @@ -1240,8 +1532,6 @@ static int ahash_sha256_digest(struct ahash_request *req) { int ret2, ret1; - pr_debug(DEV_DBG_NAME " [%s]", __func__); - ret1 = ahash_sha256_init(req); if (ret1) goto out; @@ -1258,8 +1548,6 @@ static int hmac_sha1_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s]: (ctx=0x%x)!", __func__, (u32) ctx); - ctx->config.data_format = HASH_DATA_8_BITS; ctx->config.algorithm = HASH_ALGO_SHA1; ctx->config.oper_mode = HASH_OPER_MODE_HMAC; @@ -1273,8 +1561,6 @@ static int hmac_sha256_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct hash_ctx *ctx = crypto_ahash_ctx(tfm); - pr_debug(DEV_DBG_NAME " [%s]: (ctx=0x%x)!", __func__, (u32) ctx); - ctx->config.data_format = HASH_DATA_8_BITS; ctx->config.algorithm = HASH_ALGO_SHA256; ctx->config.oper_mode = HASH_OPER_MODE_HMAC; @@ -1287,8 +1573,6 @@ static int hmac_sha1_digest(struct ahash_request *req) { int ret2, ret1; - pr_debug(DEV_DBG_NAME " [%s]", __func__); - ret1 = hmac_sha1_init(req); if (ret1) goto out; @@ -1304,8 +1588,6 @@ static int hmac_sha256_digest(struct ahash_request *req) { int ret2, ret1; - pr_debug(DEV_DBG_NAME " [%s]", __func__); - ret1 = hmac_sha256_init(req); if (ret1) goto out; @@ -1320,16 +1602,12 @@ out: static int hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - pr_debug(DEV_DBG_NAME " [%s]", __func__); - return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA1); } static int hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - pr_debug(DEV_DBG_NAME " [%s]", __func__); - return hash_setkey(tfm, key, keylen, HASH_ALGO_SHA256); } @@ -1425,8 +1703,6 @@ static int ahash_algs_register_all(struct hash_device_data *device_data) int i; int count; - dev_dbg(device_data->dev, "[%s]", __func__); - for (i = 0; i < ARRAY_SIZE(ux500_ahash_algs); i++) { ret = crypto_register_ahash(ux500_ahash_algs[i]); if (ret) { @@ -1451,8 +1727,6 @@ static void ahash_algs_unregister_all(struct hash_device_data *device_data) { int i; - dev_dbg(device_data->dev, "[%s]", __func__); - for (i = 0; i < ARRAY_SIZE(ux500_ahash_algs); i++) crypto_unregister_ahash(ux500_ahash_algs[i]); } @@ -1468,7 +1742,6 @@ static int ux500_hash_probe(struct platform_device *pdev) struct hash_device_data *device_data; struct device *dev = &pdev->dev; - dev_dbg(dev, "[%s] (pdev=0x%x)", __func__, (u32) pdev); device_data = kzalloc(sizeof(struct hash_device_data), GFP_ATOMIC); if (!device_data) { dev_dbg(dev, "[%s] kzalloc() failed!", __func__); @@ -1505,7 +1778,6 @@ static int ux500_hash_probe(struct platform_device *pdev) /* Enable power for HASH1 hardware block */ device_data->regulator = ux500_regulator_get(dev); - if (IS_ERR(device_data->regulator)) { dev_err(dev, "[%s] regulator_get() failed!", __func__); ret = PTR_ERR(device_data->regulator); @@ -1534,6 +1806,9 @@ static int ux500_hash_probe(struct platform_device *pdev) goto out_power; } + if (hash_mode == HASH_MODE_DMA) + hash_dma_setup_channel(device_data, dev); + platform_set_drvdata(pdev, device_data); /* Put the new device into the device list... */ @@ -1585,8 +1860,6 @@ static int ux500_hash_remove(struct platform_device *pdev) struct hash_device_data *device_data; struct device *dev = &pdev->dev; - dev_dbg(dev, "[%s] (pdev=0x%x)", __func__, (u32) pdev); - device_data = platform_get_drvdata(pdev); if (!device_data) { dev_err(dev, "[%s]: platform_get_drvdata() failed!", @@ -1646,8 +1919,6 @@ static void ux500_hash_shutdown(struct platform_device *pdev) struct resource *res = NULL; struct hash_device_data *device_data; - dev_dbg(&pdev->dev, "[%s]", __func__); - device_data = platform_get_drvdata(pdev); if (!device_data) { dev_err(&pdev->dev, "[%s] platform_get_drvdata() failed!", @@ -1701,8 +1972,6 @@ static int ux500_hash_suspend(struct platform_device *pdev, pm_message_t state) struct hash_device_data *device_data; struct hash_ctx *temp_ctx = NULL; - dev_dbg(&pdev->dev, "[%s]", __func__); - device_data = platform_get_drvdata(pdev); if (!device_data) { dev_err(&pdev->dev, "[%s] platform_get_drvdata() failed!", @@ -1740,8 +2009,6 @@ static int ux500_hash_resume(struct platform_device *pdev) struct hash_device_data *device_data; struct hash_ctx *temp_ctx = NULL; - dev_dbg(&pdev->dev, "[%s]", __func__); - device_data = platform_get_drvdata(pdev); if (!device_data) { dev_err(&pdev->dev, "[%s] platform_get_drvdata() failed!", @@ -1783,7 +2050,6 @@ static struct platform_driver hash_driver = { */ static int __init ux500_hash_mod_init(void) { - pr_debug(DEV_DBG_NAME " [%s] is called!", __func__); klist_init(&driver_data.device_list, NULL, NULL); /* Initialize the semaphore to 0 devices (locked state) */ sema_init(&driver_data.device_allocation, 0); @@ -1796,8 +2062,6 @@ static int __init ux500_hash_mod_init(void) */ static void __exit ux500_hash_mod_fini(void) { - pr_debug(DEV_DBG_NAME " [%s] is called!", __func__); - platform_driver_unregister(&hash_driver); return; } -- cgit v1.2.3