From ac58ec89186c7270936a6d0e122a33b06e78b78e Mon Sep 17 00:00:00 2001 From: Joakim Bech Date: Mon, 30 May 2011 08:58:22 +0200 Subject: ux500: cryp: Use relaxed versions of writel/readl - Use readl_relaxed instead of readl. - Use writel_relaxed instead of writel. - Use writesl where a stream of data is written to the crypto hardware. - Use readsl where a stream of data is read from the crypto hardware. The above changes will increase performance when doing cryptographic operations. We noticed about 30%-50% better throughput when running the cryptographic test module tcrypt. ST-Ericsson ID: 343394 ST-Ericsson Linux next: Not tested, ER 320876 ST-Ericsson FOSS-OUT ID: Trivial Change-Id: I32d9229f93a8b86257d5a9ab9729cfc4337caa52 Signed-off-by: Joakim Bech Reviewed-on: http://gerrit.lud.stericsson.com/gerrit/24275 Reviewed-by: QATEST Reviewed-by: Berne HEBARK Reviewed-by: Linus WALLEIJ --- drivers/crypto/ux500/cryp/cryp.c | 109 ++++++++++++++++++---------------- drivers/crypto/ux500/cryp/cryp_core.c | 23 ++++--- drivers/crypto/ux500/cryp/cryp_irq.c | 10 ++-- drivers/crypto/ux500/cryp/cryp_p.h | 9 +-- 4 files changed, 79 insertions(+), 72 deletions(-) diff --git a/drivers/crypto/ux500/cryp/cryp.c b/drivers/crypto/ux500/cryp/cryp.c index 30318817b18..ae4fe318528 100644 --- a/drivers/crypto/ux500/cryp/cryp.c +++ b/drivers/crypto/ux500/cryp/cryp.c @@ -34,14 +34,22 @@ int cryp_check(struct cryp_device_data *device_data) return -EINVAL; /* Check Peripheral and Pcell Id Register for CRYP */ - if ((CRYP_PERIPHERAL_ID0 == readl(&device_data->base->periphId0)) - && (CRYP_PERIPHERAL_ID1 == readl(&device_data->base->periphId1)) - && (CRYP_PERIPHERAL_ID2 == readl(&device_data->base->periphId2)) - && (CRYP_PERIPHERAL_ID3 == readl(&device_data->base->periphId3)) - && (CRYP_PCELL_ID0 == readl(&device_data->base->pcellId0)) - && (CRYP_PCELL_ID1 == readl(&device_data->base->pcellId1)) - && (CRYP_PCELL_ID2 == readl(&device_data->base->pcellId2)) - && (CRYP_PCELL_ID3 == readl(&device_data->base->pcellId3))) { + if ((CRYP_PERIPHERAL_ID0 == + readl_relaxed(&device_data->base->periphId0)) + && (CRYP_PERIPHERAL_ID1 == + readl_relaxed(&device_data->base->periphId1)) + && (CRYP_PERIPHERAL_ID2 == + readl_relaxed(&device_data->base->periphId2)) + && (CRYP_PERIPHERAL_ID3 == + readl_relaxed(&device_data->base->periphId3)) + && (CRYP_PCELL_ID0 == + readl_relaxed(&device_data->base->pcellId0)) + && (CRYP_PCELL_ID1 == + readl_relaxed(&device_data->base->pcellId1)) + && (CRYP_PCELL_ID2 == + readl_relaxed(&device_data->base->pcellId2)) + && (CRYP_PCELL_ID3 == + readl_relaxed(&device_data->base->pcellId3))) { return 0; } @@ -83,7 +91,8 @@ void cryp_flush_inoutfifo(struct cryp_device_data *device_data) * register when starting a new calculation, which means Input FIFO is * not full and input FIFO is empty. */ - while (readl(&device_data->base->sr) != CRYP_SR_INFIFO_READY_MASK) + while (readl_relaxed(&device_data->base->sr) != + CRYP_SR_INFIFO_READY_MASK) cpu_relax(); } @@ -130,7 +139,7 @@ int cryp_set_configuration(struct cryp_device_data *device_data, (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS) | (KSE_ENABLED << CRYP_CR_KSE_POS)); - writel(cr_for_kse, &device_data->base->cr); + writel_relaxed(cr_for_kse, &device_data->base->cr); cryp_wait_until_done(device_data); } @@ -201,27 +210,27 @@ int cryp_configure_key_values(struct cryp_device_data *device_data, switch (key_reg_index) { case CRYP_KEY_REG_1: - writel(key_value.key_value_left, + writel_relaxed(key_value.key_value_left, &device_data->base->key_1_l); - writel(key_value.key_value_right, + writel_relaxed(key_value.key_value_right, &device_data->base->key_1_r); break; case CRYP_KEY_REG_2: - writel(key_value.key_value_left, + writel_relaxed(key_value.key_value_left, &device_data->base->key_2_l); - writel(key_value.key_value_right, + writel_relaxed(key_value.key_value_right, &device_data->base->key_2_r); break; case CRYP_KEY_REG_3: - writel(key_value.key_value_left, + writel_relaxed(key_value.key_value_left, &device_data->base->key_3_l); - writel(key_value.key_value_right, + writel_relaxed(key_value.key_value_right, &device_data->base->key_3_r); break; case CRYP_KEY_REG_4: - writel(key_value.key_value_left, + writel_relaxed(key_value.key_value_left, &device_data->base->key_4_l); - writel(key_value.key_value_right, + writel_relaxed(key_value.key_value_right, &device_data->base->key_4_r); break; default: @@ -249,15 +258,15 @@ int cryp_configure_init_vector(struct cryp_device_data *device_data, switch (init_vector_index) { case CRYP_INIT_VECTOR_INDEX_0: - writel(init_vector_value.init_value_left, + writel_relaxed(init_vector_value.init_value_left, &device_data->base->init_vect_0_l); - writel(init_vector_value.init_value_right, + writel_relaxed(init_vector_value.init_value_right, &device_data->base->init_vect_0_r); break; case CRYP_INIT_VECTOR_INDEX_1: - writel(init_vector_value.init_value_left, + writel_relaxed(init_vector_value.init_value_left, &device_data->base->init_vect_1_l); - writel(init_vector_value.init_value_right, + writel_relaxed(init_vector_value.init_value_right, &device_data->base->init_vect_1_r); break; default: @@ -293,26 +302,26 @@ void cryp_save_device_context(struct cryp_device_data *device_data, cryp_configure_for_dma(device_data, CRYP_DMA_DISABLE_BOTH); if (CRYP_TEST_BITS(&src_reg->sr, CRYP_SR_IFEM_MASK) == 0) - ctx->din = readl(&src_reg->din); + ctx->din = readl_relaxed(&src_reg->din); - ctx->cr = readl(&src_reg->cr) & CRYP_CR_CONTEXT_SAVE_MASK; + ctx->cr = readl_relaxed(&src_reg->cr) & CRYP_CR_CONTEXT_SAVE_MASK; switch (config->keysize) { case CRYP_KEY_SIZE_256: - ctx->key_4_l = readl(&src_reg->key_4_l); - ctx->key_4_r = readl(&src_reg->key_4_r); + ctx->key_4_l = readl_relaxed(&src_reg->key_4_l); + ctx->key_4_r = readl_relaxed(&src_reg->key_4_r); case CRYP_KEY_SIZE_192: - ctx->key_3_l = readl(&src_reg->key_3_l); - ctx->key_3_r = readl(&src_reg->key_3_r); + ctx->key_3_l = readl_relaxed(&src_reg->key_3_l); + ctx->key_3_r = readl_relaxed(&src_reg->key_3_r); case CRYP_KEY_SIZE_128: - ctx->key_2_l = readl(&src_reg->key_2_l); - ctx->key_2_r = readl(&src_reg->key_2_r); + ctx->key_2_l = readl_relaxed(&src_reg->key_2_l); + ctx->key_2_r = readl_relaxed(&src_reg->key_2_r); default: - ctx->key_1_l = readl(&src_reg->key_1_l); - ctx->key_1_r = readl(&src_reg->key_1_r); + ctx->key_1_l = readl_relaxed(&src_reg->key_1_l); + ctx->key_1_r = readl_relaxed(&src_reg->key_1_r); } /* Save IV for CBC mode for both AES and DES. */ @@ -320,10 +329,10 @@ void cryp_save_device_context(struct cryp_device_data *device_data, if (algomode == CRYP_ALGO_TDES_CBC || algomode == CRYP_ALGO_DES_CBC || algomode == CRYP_ALGO_AES_CBC) { - ctx->init_vect_0_l = readl(&src_reg->init_vect_0_l); - ctx->init_vect_0_r = readl(&src_reg->init_vect_0_r); - ctx->init_vect_1_l = readl(&src_reg->init_vect_1_l); - ctx->init_vect_1_r = readl(&src_reg->init_vect_1_r); + ctx->init_vect_0_l = readl_relaxed(&src_reg->init_vect_0_l); + ctx->init_vect_0_r = readl_relaxed(&src_reg->init_vect_0_r); + ctx->init_vect_1_l = readl_relaxed(&src_reg->init_vect_1_l); + ctx->init_vect_1_r = readl_relaxed(&src_reg->init_vect_1_r); } } @@ -346,30 +355,30 @@ void cryp_restore_device_context(struct cryp_device_data *device_data, */ switch (config->keysize) { case CRYP_KEY_SIZE_256: - writel(ctx->key_4_l, ®->key_4_l); - writel(ctx->key_4_r, ®->key_4_r); + writel_relaxed(ctx->key_4_l, ®->key_4_l); + writel_relaxed(ctx->key_4_r, ®->key_4_r); case CRYP_KEY_SIZE_192: - writel(ctx->key_3_l, ®->key_3_l); - writel(ctx->key_3_r, ®->key_3_r); + writel_relaxed(ctx->key_3_l, ®->key_3_l); + writel_relaxed(ctx->key_3_r, ®->key_3_r); case CRYP_KEY_SIZE_128: - writel(ctx->key_2_l, ®->key_2_l); - writel(ctx->key_2_r, ®->key_2_r); + writel_relaxed(ctx->key_2_l, ®->key_2_l); + writel_relaxed(ctx->key_2_r, ®->key_2_r); default: - writel(ctx->key_1_l, ®->key_1_l); - writel(ctx->key_1_r, ®->key_1_r); + writel_relaxed(ctx->key_1_l, ®->key_1_l); + writel_relaxed(ctx->key_1_r, ®->key_1_r); } /* Restore IV for CBC mode for AES and DES. */ if (config->algomode == CRYP_ALGO_TDES_CBC || config->algomode == CRYP_ALGO_DES_CBC || config->algomode == CRYP_ALGO_AES_CBC) { - writel(ctx->init_vect_0_l, ®->init_vect_0_l); - writel(ctx->init_vect_0_r, ®->init_vect_0_r); - writel(ctx->init_vect_1_l, ®->init_vect_1_l); - writel(ctx->init_vect_1_r, ®->init_vect_1_r); + writel_relaxed(ctx->init_vect_0_l, ®->init_vect_0_l); + writel_relaxed(ctx->init_vect_0_r, ®->init_vect_0_r); + writel_relaxed(ctx->init_vect_1_l, ®->init_vect_1_l); + writel_relaxed(ctx->init_vect_1_r, ®->init_vect_1_r); } } @@ -381,7 +390,7 @@ void cryp_restore_device_context(struct cryp_device_data *device_data, */ int cryp_write_indata(struct cryp_device_data *device_data, u32 write_data) { - writel(write_data, &device_data->base->din); + writel_relaxed(write_data, &device_data->base->din); return 0; } @@ -394,7 +403,7 @@ int cryp_write_indata(struct cryp_device_data *device_data, u32 write_data) */ int cryp_read_outdata(struct cryp_device_data *device_data, u32 *read_data) { - *read_data = readl(&device_data->base->dout); + *read_data = readl_relaxed(&device_data->base->dout); return 0; } diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index 0276501831a..da168cb57f6 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -376,11 +376,11 @@ static int cryp_setup_context(struct cryp_ctx *ctx, switch (cryp_mode) { case CRYP_MODE_INTERRUPT: - writel(CRYP_IMSC_DEFAULT, &device_data->base->imsc); + writel_relaxed(CRYP_IMSC_DEFAULT, &device_data->base->imsc); break; case CRYP_MODE_DMA: - writel(CRYP_DMACR_DEFAULT, &device_data->base->dmacr); + writel_relaxed(CRYP_DMACR_DEFAULT, &device_data->base->dmacr); break; default: @@ -416,7 +416,8 @@ static int cryp_setup_context(struct cryp_ctx *ctx, } else control_register = ctx->dev_ctx.cr; - writel(control_register | (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS), + writel(control_register | + (CRYP_CRYPEN_ENABLE << CRYP_CR_CRYPEN_POS), &device_data->base->cr); return 0; @@ -634,23 +635,19 @@ static int cryp_dma_read(struct cryp_ctx *ctx, struct scatterlist *sg, int len) static void cryp_polling_mode(struct cryp_ctx *ctx, struct cryp_device_data *device_data) { - int i; + int len = ctx->blocksize / BYTES_PER_WORD; int remaining_length = ctx->datalen; u32 *indata = (u32 *)ctx->indata; u32 *outdata = (u32 *)ctx->outdata; while (remaining_length > 0) { - for (i = 0; i < ctx->blocksize / BYTES_PER_WORD; i++) { - writel(*indata, &device_data->base->din); - ++indata; - remaining_length -= BYTES_PER_WORD; - } + writesl(&device_data->base->din, indata, len); + indata += len; + remaining_length -= (len * BYTES_PER_WORD); cryp_wait_until_done(device_data); - for (i = 0; i < ctx->blocksize / BYTES_PER_WORD; i++) { - *outdata = readl(&device_data->base->dout); - ++outdata; - } + readsl(&device_data->base->dout, outdata, len); + outdata += len; cryp_wait_until_done(device_data); } } diff --git a/drivers/crypto/ux500/cryp/cryp_irq.c b/drivers/crypto/ux500/cryp/cryp_irq.c index 8814acc05d7..08d291cdbe6 100644 --- a/drivers/crypto/ux500/cryp/cryp_irq.c +++ b/drivers/crypto/ux500/cryp/cryp_irq.c @@ -23,9 +23,9 @@ void cryp_enable_irq_src(struct cryp_device_data *device_data, u32 irq_src) dev_dbg(device_data->dev, "[%s]", __func__); - i = readl(&device_data->base->imsc); + i = readl_relaxed(&device_data->base->imsc); i = i | irq_src; - writel(i, &device_data->base->imsc); + writel_relaxed(i, &device_data->base->imsc); } void cryp_disable_irq_src(struct cryp_device_data *device_data, u32 irq_src) @@ -34,12 +34,12 @@ void cryp_disable_irq_src(struct cryp_device_data *device_data, u32 irq_src) dev_dbg(device_data->dev, "[%s]", __func__); - i = readl(&device_data->base->imsc); + i = readl_relaxed(&device_data->base->imsc); i = i & ~irq_src; - writel(i, &device_data->base->imsc); + writel_relaxed(i, &device_data->base->imsc); } bool cryp_pending_irq_src(struct cryp_device_data *device_data, u32 irq_src) { - return (readl(&device_data->base->mis) & irq_src) > 0; + return (readl_relaxed(&device_data->base->mis) & irq_src) > 0; } diff --git a/drivers/crypto/ux500/cryp/cryp_p.h b/drivers/crypto/ux500/cryp/cryp_p.h index 5171ee10f71..4b615a33fe9 100644 --- a/drivers/crypto/ux500/cryp/cryp_p.h +++ b/drivers/crypto/ux500/cryp/cryp_p.h @@ -21,16 +21,17 @@ * Generic Macros */ #define CRYP_SET_BITS(reg_name, mask) \ - writel((readl(reg_name) | mask), reg_name) + writel_relaxed((readl_relaxed(reg_name) | mask), reg_name) #define CRYP_WRITE_BIT(reg_name, val, mask) \ - writel(((readl(reg_name) & ~(mask)) | ((val) & (mask))), reg_name) + writel_relaxed(((readl_relaxed(reg_name) & ~(mask)) |\ + ((val) & (mask))), reg_name) #define CRYP_TEST_BITS(reg_name, val) \ - (readl(reg_name) & (val)) + (readl_relaxed(reg_name) & (val)) #define CRYP_PUT_BITS(reg, val, shift, mask) \ - writel(((readl(reg) & ~(mask)) | \ + writel_relaxed(((readl_relaxed(reg) & ~(mask)) | \ (((u32)val << shift) & (mask))), reg) /** -- cgit v1.2.3