marvell/linux/drivers/crypto/asr/te200/asr-sha.c

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/err.h>
#include <linux/clk-provider.h>
#include <linux/clk.h>
#include <linux/io.h>
#include <linux/hw_random.h>
#include <linux/platform_device.h>
#include <linux/of_device.h>
#include <linux/device.h>
#include <linux/init.h>
#include <crypto/hmac.h>
#include <crypto/sha.h>
#include "asr-te200.h"
#include "asr-sha.h"

// #define ASR_TE200_SHA_TEST

static struct asr_te200_sha *asr_sha_local = NULL;
static struct mutex hash_lock = __MUTEX_INITIALIZER(hash_lock);

static inline u32 asr_sha_read(struct asr_te200_sha *dd, u32 offset)
{
	u32 value = readl_relaxed(dd->io_base + offset);

	return value;
}

static inline void asr_sha_write(struct asr_te200_sha *dd,
					u32 offset, u32 value)
{
	writel_relaxed(value, dd->io_base + offset);
}

/* ------- te200 sha hardware operation -------- */
static int hash_clock_switch(struct asr_te200_sha *dd, int enable)
{
	uint32_t value;

	value = asr_sha_read(dd, TE200_CLOCK_CTRL);
	if (enable) {
		value |= HASH_CLK_EN;
	} else {
		value &= ~HASH_CLK_EN;
	}

	asr_sha_write(dd, TE200_CLOCK_CTRL, value);

	return 0;
}

static int hash_start_run(struct asr_te200_sha *dd)
{
	uint32_t value;
	value = asr_sha_read(dd, TE200_SHASH_CTRL);
	value |= HASH_RUN;
	asr_sha_write(dd, TE200_SHASH_CTRL, value);
	return 0; 
}

static int hash_wait_intr(struct asr_te200_sha *dd)
{
	int ret = 0;
	uint32_t value;
	uint32_t time_start;
	uint32_t clk_val;
	clk_val = asr_sha_read(dd, TE200_CLOCK_CTRL);

	time_start = jiffies;
	value = asr_sha_read(dd, TE200_SHASH_INTR_STAT);

	while (1) {
		value = asr_sha_read(dd, TE200_SHASH_INTR_STAT);

		if (value & HASH_INVALID_CMD) {
			dev_err(dd->dev, "invallid cmd\n");
			ret = -1;
			break;
		}

		if (value & HASH_BUS_ERROR) {
			dev_err(dd->dev, "bus err\n");
			ret = -1;
			break;
		}

		if ((jiffies - time_start) > 500) {
			dev_err(dd->dev, "wait intr timeout !\n");
			ret = -1;
			break;
		}

		if (value & HASH_CMD_INTR) {
			break;
		}
	}

	value = asr_sha_read(dd, TE200_SHASH_INTR_STAT);
	value |= HASH_CMD_INTR;
	asr_sha_write(dd, TE200_SHASH_INTR_STAT, value);
	return ret;
}

static inline void sha_cache_operation(void *addr, int size)
{
	__cpuc_flush_dcache_area(addr, size);
}

static int _hash_op_init(struct asr_sha_reqctx *reqctx, int alg, uint8_t *ext_iv)
{
	int ret;
	uint32_t cmd = 0;
	uint32_t ext_iv_phys;
	struct asr_te200_sha *dd = reqctx->dd;
	te200_hash_context_t *ctx = &reqctx->hash_ctx;

	hash_clock_switch(dd, 1);

	if (ext_iv) {
		cmd |= HASH_INIT_CMD | HASH_SET_EXT_IV | HASH_PARAM_IS_ADDR | HASH_INTER_TRIGGERD;
		/* Set initial length */
		if (ctx->total_bits_num != 0)
			cmd |= 0x4;
	} else {
		cmd |= HASH_INIT_CMD | HASH_PARAM_IS_ADDR | HASH_INTER_TRIGGERD;
	}

	switch (alg) {
	case HASH_SHA1:
		cmd &= HASH_MODE_SHA1;
		break;
	case HASH_SHA224:
		cmd |= HASH_MODE_SHA224;
		break;
	case HASH_SHA256:
		cmd |= HASH_MODE_SHA256;
		break;
	default:
		hash_clock_switch(dd, 0);
		return -EINVAL;
	}

	asr_sha_write(dd, TE200_SHASH_QUEUE, cmd);
	if (ext_iv) {
		ext_iv_phys = (uint32_t)virt_to_phys((void *)ext_iv);
		sha_cache_operation((void *)ext_iv, 32);
		asr_sha_write(dd, TE200_SHASH_QUEUE, ext_iv_phys);
		/* Set HASH total bits length, split 64 bits into two parts, 32 bits for
			* each */
		if (ctx->total_bits_num != 0) {
			asr_sha_write(dd, TE200_SHASH_QUEUE, (ctx->total_bits_num & 0xFFFFFFFF));
			asr_sha_write(dd, TE200_SHASH_QUEUE, (ctx->total_bits_num >> 0x20));
		}
	}

	hash_start_run(dd);
	ret = hash_wait_intr(dd);
	reqctx->hash_ctx.finish_flag = 1;

	hash_clock_switch(dd, 0);
	return ret;
}

static int _hash_op_proc(struct asr_sha_reqctx *reqctx, const uint8_t *src, size_t size)
{
	int ret = 0;
	uint32_t cmd = 0;
	uint32_t src_phys;
	struct asr_te200_sha *dd = reqctx->dd;
	te200_hash_context_t *ctx = &reqctx->hash_ctx;
	size_t input_data_len = 0;
	uint32_t old_extra_len = ctx->count;

	hash_clock_switch(dd, 1);

	/* Extra data bytes number */
	ctx->count = (size + old_extra_len) % HASH_BUF_LEN;
	if (size + old_extra_len >= HASH_BUF_LEN) {
		/* First handle old extra data, then the new input data */
		if (old_extra_len != 0) {
			src_phys = (uint32_t)virt_to_phys((void *)ctx->extra_data);
			sha_cache_operation((void *)ctx->extra_data, old_extra_len);

			cmd = HASH_PROCESS_CMD | HASH_INTER_TRIGGERD;
			asr_sha_write(dd, TE200_SHASH_QUEUE, cmd);

			asr_sha_write(dd, TE200_SHASH_QUEUE, src_phys);
			asr_sha_write(dd, TE200_SHASH_QUEUE, old_extra_len);

			hash_start_run(dd);
			ret = hash_wait_intr(dd);
			if (ret)
				goto err;
			ctx->total_bits_num += old_extra_len * 8;
		}

		cmd = HASH_PROCESS_CMD | HASH_INTER_TRIGGERD;
		input_data_len = size - ctx->count;

		src_phys = virt_to_phys((void *)src);
		sha_cache_operation((void *)src, input_data_len);
		asr_sha_write(dd, TE200_SHASH_QUEUE, cmd);
		asr_sha_write(dd, TE200_SHASH_QUEUE, (uint32_t)src_phys);
		asr_sha_write(dd, TE200_SHASH_QUEUE, input_data_len);

		hash_start_run(dd);
		ret = hash_wait_intr(dd);
		if (ret)
			goto err;

		/* Total data bits number */
		ctx->total_bits_num += input_data_len * 8;
		/* Save new extra data */
		memset(ctx->extra_data, 0, sizeof( ctx->extra_data ));
		memcpy(ctx->extra_data, (src + size - ctx->count), ctx->count);
	} else {
		/* If ilen + old_extra_len < HASH_BUF_LEN */
		/* Save input data and return. */
		memcpy(ctx->extra_data + old_extra_len, src, size);		
	}
	ret = 0;

err:
	hash_clock_switch(dd, 0);
	return ret;
}

static int _hash_op_finish(struct asr_sha_reqctx *reqctx,
						  uint8_t *out, uint32_t out_size, int padding)
{
	int ret = 0;
	uint32_t cmd = 0;
	uint32_t out_phys;
	struct asr_te200_sha *dd = reqctx->dd;
	te200_hash_context_t *ctx = &reqctx->hash_ctx;
	uint32_t extra_data_phys;

	/* filter uninitialized finish request */
	if ( !reqctx->hash_ctx.finish_flag ) {
		return ret;
	}

	hash_clock_switch(dd, 1);

	if (padding == 0) {
		cmd = HASH_FINISH_CMD | HASH_INTER_TRIGGERD;
		ctx->hash_temp_valid = 1;
		ctx->finish_flag     = 0;
	} else {
		/* If extra data count is not zero, execute HASH process command first */
		if (ctx->count != 0) {
			cmd = HASH_PROCESS_CMD | HASH_INTER_TRIGGERD;
			asr_sha_write(dd, TE200_SHASH_QUEUE, cmd);

			extra_data_phys = (uint32_t)virt_to_phys((void *)ctx->extra_data);
			sha_cache_operation((void *)ctx->extra_data, ctx->count);
			asr_sha_write(dd, TE200_SHASH_QUEUE, extra_data_phys);
			asr_sha_write(dd, TE200_SHASH_QUEUE, ctx->count);
			hash_start_run(dd);
			ret = hash_wait_intr(dd);
			if (ret)
				goto err;
		}
		cmd = HASH_FINISH_CMD | HASH_PADDING | HASH_INTER_TRIGGERD;
	}

	out_phys = virt_to_phys((void *)out);
	sha_cache_operation((void *)out, out_size);

	asr_sha_write(dd, TE200_SHASH_QUEUE, cmd);

	asr_sha_write(dd, TE200_SHASH_QUEUE, (uint32_t)out_phys);

	hash_start_run(dd);
	ret = hash_wait_intr(dd);
	if (ret)
		goto err;

	ret = 0;
err:
	hash_clock_switch(dd, 0);
	return ret;
}

static struct asr_sha_reqctx *_g_sha_ctx = NULL;
#define GET_HASH_LEN( reqctx )                         \
	( ( reqctx->hash_ctx.alg == HASH_SHA1 )                  \
		  ? 20                                 		\
		  : ( reqctx->hash_ctx.alg == HASH_SHA224 )          \
				? 28                         		\
				: ( reqctx->hash_ctx.alg == HASH_SHA256 )    \
					  ? 32 : 0)

static int hash_op_init(struct asr_sha_reqctx *reqctx, int alg)
{
	int ret = 0;
	unsigned char garbage[64] = {0};
	uint32_t hash_temp_len;

	mutex_lock(&hash_lock);

	if (_g_sha_ctx != reqctx) {
		/* First finish old session (_g_sha_ctx), then load new session(ctx) */
		if (_g_sha_ctx != NULL) {
			hash_temp_len = GET_HASH_LEN(_g_sha_ctx);
			if (hash_temp_len == 0) {
				ret = -1;
				goto exit;
			}
			ret = _hash_op_finish(_g_sha_ctx, _g_sha_ctx->hash_ctx.hash_temp, hash_temp_len, 0 );
			_g_sha_ctx = NULL;
			if (ret) {
				printk("swap out previously context failed");
				goto exit;
			}
		}
	} else {
		/*
		 * This session re-start, flush garbage data. before execute
		 * finish command must check if it's finish flag is set,
		 * if not no need to excecute finish command
		 */
		if ( _g_sha_ctx != NULL ) {
			hash_temp_len = GET_HASH_LEN(_g_sha_ctx);
			if (hash_temp_len == 0) {
				ret = -1;
				goto exit;
			}
			ret = _hash_op_finish( _g_sha_ctx, garbage, hash_temp_len, 1 );
			_g_sha_ctx = NULL;
			if (ret) {
				printk("hash finish error during switching context!");
				goto exit;
			}
		}		
	}

	memset(&reqctx->hash_ctx, 0, sizeof(reqctx->hash_ctx));
	reqctx->hash_ctx.alg = alg;
	ret = _hash_op_init(reqctx, alg, NULL);
	if (ret) {
		printk( " execute hash init failed when te200 hash init" );
		goto exit;
	}

	_g_sha_ctx = reqctx;
	ret = 0;

exit:
	mutex_unlock(&hash_lock);
	return ret;
}

static int hash_op_proc(struct asr_sha_reqctx *reqctx, const uint8_t *src, size_t size)
{
	int ret = 0;
	uint32_t hash_temp_len;

	mutex_lock(&hash_lock);

	if (reqctx == NULL) {
		ret = -1;
		goto exit;
	}

	/* Multi-session */
	if ( _g_sha_ctx != reqctx ) {
		/* First finish old session (_g_sha_ctx), then load new session(ctx) */
		if (_g_sha_ctx != NULL) {
			hash_temp_len = GET_HASH_LEN(_g_sha_ctx);
			if (hash_temp_len == 0) {
				ret = -1;
				goto exit;
			}
			ret = _hash_op_finish( _g_sha_ctx, _g_sha_ctx->hash_ctx.hash_temp, hash_temp_len, 0 );
			_g_sha_ctx = NULL;
			if (ret) {
				printk("hash finish error during switching context!");
				goto exit;
			}
		}

		/* Re-initialize */
		/* Execute te200 HASH_init command, load hash intermediate data */
		hash_temp_len = GET_HASH_LEN( reqctx );
		if ( reqctx->hash_ctx.hash_temp_valid == 1 ) {
			ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, reqctx->hash_ctx.hash_temp);
		} else {
			ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, NULL);
		}
		if ( ret != 0 ) {
			printk("execute hash init failed when update, reason: %x", ret);
			goto exit;
		}
		_g_sha_ctx = reqctx;
	}

	/* Execute te200 HASH_process command */
	ret = _hash_op_proc(reqctx, src, size);
	if ( ret != 0 ) {
		printk("execute hash process failed when update, reason: %x", ret);
		goto exit;
	}

	ret = 0;

exit:
	mutex_unlock(&hash_lock);
	return ret;
}

static int hash_op_finish(struct asr_sha_reqctx *reqctx, uint8_t *out, uint32_t out_size)
{
	int ret = 0;
	uint32_t hash_temp_len;
	
	mutex_lock(&hash_lock);

	if ((reqctx == NULL) || (NULL == out)) {
		printk( "context might probably not initialised!!" );
		ret = -1;
		goto exit;
	}

	if ( _g_sha_ctx == reqctx ) {
		/* even though invoke hash_finish_req right after _hash_op_init
		   should get a default hash ouput*/

		if ( !reqctx->hash_ctx.finish_flag ) {
			if ( reqctx->hash_ctx.hash_temp_valid == 1 ) {
				ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, reqctx->hash_ctx.hash_temp);
			} else {
				ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, NULL);
			}
			if ( ret != 0 ) {
				printk("execute hash init failed when finish, reason: %x", ret);
				goto exit;
			}
		}

		ret = _hash_op_finish( reqctx, out, out_size, 1 );
	} else {
		/*  when finished the session must check it's finish flag first, if not
		* set don't need to finish it  */
		if ( _g_sha_ctx != NULL ) {
			/* Save current session, then load new session */
			hash_temp_len = GET_HASH_LEN(_g_sha_ctx);
			if (hash_temp_len == 0) {
				ret = -1;
				goto exit;
			}
			ret = _hash_op_finish( _g_sha_ctx, _g_sha_ctx->hash_ctx.hash_temp, hash_temp_len, 0 );
			_g_sha_ctx = NULL;
			if ( ret != 0 ) {
				printk("hash finish error during switching context!");
				goto exit;
			}
		}

		if ( reqctx->hash_ctx.hash_temp_valid == 1 ) {
			ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, reqctx->hash_ctx.hash_temp);
		} else {
			ret = _hash_op_init(reqctx, reqctx->hash_ctx.alg, NULL);
		}
		if ( ret != 0 ) {
			printk("execute hash init failed when finish, reason: %x", ret);
			goto exit;
		}

		_g_sha_ctx = reqctx;
		ret = _hash_op_finish( reqctx, out, out_size, 1 );
	}

	_g_sha_ctx = NULL;

	ret = 0;

exit:
	mutex_unlock(&hash_lock);
	return ret;	
}

int asr_te200_hash_init(struct asr_sha_reqctx *reqctx, int alg)
{
	reqctx->dd = asr_sha_local;

	if (!reqctx->dd) {
		return -1;
	}
	return hash_op_init(reqctx, alg);
}

int asr_te200_hash_proc(struct asr_sha_reqctx *reqctx, const uint8_t *src, size_t size)
{
	int ret;
	uint8_t *psrc;
	reqctx->dd = asr_sha_local;
	
	if (!reqctx->dd) {
		return -1;
	}

	psrc = kmalloc(size, GFP_KERNEL);
	if (!psrc) {
		return -1;
	}
	memcpy(psrc, (void *)src, size);

	ret = hash_op_proc(reqctx, psrc, size);
	kfree(psrc);

	return ret;
}

int asr_te200_hash_finish(struct asr_sha_reqctx *reqctx, uint8_t *out, uint32_t out_size)
{
	int ret;
	/* Avoid cache caherence problems caused by out variables being optimized */
	uint8_t hash[64] __aligned(64) = {0};
	reqctx->dd = asr_sha_local;

	if (!reqctx->dd) {
		return -1;
	}
	ret = hash_op_finish(reqctx, hash, out_size);
	memcpy(out, hash, out_size);
	
	return ret;

}
/* ------- end -------- */

static size_t asr_sha_append_sg(struct asr_sha_reqctx *ctx)
{
	size_t count;

	while ((ctx->bufcnt < ctx->buflen) && ctx->total) {
		count = min(ctx->sg->length - ctx->offset, ctx->total);
		count = min(count, ctx->buflen - ctx->bufcnt);

		if (count <= 0) {
			/*
			* Check if count <= 0 because the buffer is full or
			* because the sg length is 0. In the latest case,
			* check if there is another sg in the list, a 0 length
			* sg doesn't necessarily mean the end of the sg list.
			*/
			if ((ctx->sg->length == 0) && !sg_is_last(ctx->sg)) {
				ctx->sg = sg_next(ctx->sg);
				continue;
			} else {
				break;
			}
		}

		scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, ctx->sg,
			ctx->offset, count, 0);

		ctx->bufcnt += count;
		ctx->offset += count;
		ctx->total -= count;

		if (ctx->offset == ctx->sg->length) {
			ctx->sg = sg_next(ctx->sg);
			if (ctx->sg)
				ctx->offset = 0;
			else
				ctx->total = 0;
		}
	}

	return 0;
}

static int asr_sha_done(struct asr_te200_sha *dd);

static int asr_sha_handle_queue(struct asr_te200_sha *dd,
				  struct ahash_request *req)
{
	struct crypto_async_request *async_req, *backlog;
	struct asr_sha_ctx *ctx;
	unsigned long flags;
	bool start_async;
	int err = 0, ret = 0;

	spin_lock_irqsave(&dd->lock, flags);
	if (req)
		ret = ahash_enqueue_request(&dd->queue, req);

	if (SHA_FLAGS_BUSY & dd->flags) {
		spin_unlock_irqrestore(&dd->lock, flags);
		return ret;
	}

	backlog = crypto_get_backlog(&dd->queue);
	async_req = crypto_dequeue_request(&dd->queue);
	if (async_req)
		dd->flags |= SHA_FLAGS_BUSY;

	spin_unlock_irqrestore(&dd->lock, flags);

	if (!async_req) {
		return ret;
	}

	if (backlog)
		backlog->complete(backlog, -EINPROGRESS);

	ctx = crypto_tfm_ctx(async_req->tfm);

	dd->req = ahash_request_cast(async_req);
	start_async = (dd->req != req);
	dd->is_async = start_async;
	dd->force_complete = false;

	/* WARNING: ctx->start() MAY change dd->is_async. */
	err = ctx->start(dd);
	return (start_async) ? ret : err;
}

static int asr_sha_enqueue(struct ahash_request *req, unsigned int op)
{
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);
	struct asr_te200_sha *dd = ctx->dd;

	ctx->op = op;

	return asr_sha_handle_queue(dd, req);
}

static void asr_sha_copy_ready_hash(struct ahash_request *req)
{
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	if (!req->result)
		return;

	switch (ctx->flags & SHA_FLAGS_ALGO_MASK) {
	case SHA_FLAGS_SHA1:
		memcpy(req->result, ctx->digest, SHA1_DIGEST_SIZE);
		break;
	case SHA_FLAGS_SHA224:
		memcpy(req->result, ctx->digest, SHA224_DIGEST_SIZE);
		break;
	case SHA_FLAGS_SHA256:
		memcpy(req->result, ctx->digest, SHA256_DIGEST_SIZE);
		break;
	default:
		return;
	}
}

static inline int asr_sha_complete(struct asr_te200_sha *dd, int err)
{
	struct ahash_request *req = dd->req;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	dd->flags &= ~(SHA_FLAGS_BUSY);
	ctx->flags &= ~(SHA_FLAGS_FINAL);

	if ((dd->is_async || dd->force_complete) && req->base.complete)
		req->base.complete(&req->base, err);

	/* handle new request */
	tasklet_schedule(&dd->queue_task);

	return err;
}

static int asr_sha_buff_init(struct asr_te200_sha *dd, uint32_t len)
{
	struct ahash_request *req = dd->req;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	ctx->buffer = (void *)__get_free_pages(GFP_KERNEL, get_order(len));
	if (!ctx->buffer) {
		dev_err(dd->dev, "unable to alloc pages.\n");
		return -ENOMEM;
	}

	ctx->buflen = PAGE_SIZE << get_order(len);

	return 0;
}

static void asr_sha_buff_cleanup(struct asr_te200_sha *dd, uint32_t len)
{
	struct ahash_request *req = dd->req;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	free_pages((unsigned long)ctx->buffer, get_order(len));
	ctx->buflen = 0;
}

static int sha_init_req(struct asr_sha_reqctx *ctx)
{
	int ret = 0;

	/* hardware: hash init */
	ret = hash_op_init(ctx, ctx->alg);
	if (ret)
		return -EINVAL;
	return 0;
}

static int sha_update_req(struct asr_sha_reqctx *ctx)
{
	int ret = 0;
	int bufcnt;
	uint32_t buflen = ctx->total;

	ret = asr_sha_buff_init(ctx->dd, ctx->total);
	if (ret)
		return -ENOMEM;
	
	asr_sha_append_sg(ctx);
	bufcnt = ctx->bufcnt;
	ctx->bufcnt = 0;

	/* hashware: hash process */
	ret = hash_op_proc(ctx, ctx->buffer, bufcnt);
	if (ret)
		ret = -EINVAL;

	asr_sha_buff_cleanup(ctx->dd, buflen);
	return ret;
}

static void sha_finish_req(struct asr_sha_reqctx *ctx, int *err)
{
	uint8_t *hash = (uint8_t *)ctx->digest;
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(ctx->dd->req);
	uint32_t hash_size = crypto_ahash_digestsize(tfm);
	
	if (!(*err) && (ctx->flags & SHA_FLAGS_FINAL)) {
		*err = hash_op_finish(ctx, (uint8_t *)hash, hash_size);
		asr_sha_copy_ready_hash(ctx->dd->req);
		ctx->flags &= (~SHA_FLAGS_FINAL);
	} else {
		ctx->flags |= SHA_FLAGS_ERROR;
	}
}

static void sha_next_req(struct asr_sha_reqctx *ctx, int *err)
{
	if (likely(!(*err) && (SHA_FLAGS_FINAL & ctx->flags)))
		sha_finish_req(ctx, err);

	(void)asr_sha_complete(ctx->dd, *err);
}

static int asr_sha_start(struct asr_te200_sha *dd)
{
	int err = 0;
	struct ahash_request *req = dd->req;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);
	struct asr_te200_dev *te200_dd = dev_get_drvdata(dd->dev);
	struct asr_te200_ops *te200_ops = te200_dd->te200_ops;

	te200_ops->dev_get(te200_dd);

	dd->resume = asr_sha_done; 

	if ((ctx->flags & SHA_FLAGS_INIT)) {
		err = sha_init_req(ctx);
		ctx->flags &= (~SHA_FLAGS_INIT);
		if (err) {
			te200_ops->dev_put(te200_dd);
			return err;
		}
	}

	if (ctx->op == SHA_OP_UPDATE) {
		err = sha_update_req(ctx);
		if (!err && (ctx->flags & SHA_FLAGS_FINUP))
			/* no final() after finup() */
			sha_finish_req(ctx, &err);
	} else if (ctx->op == SHA_OP_FINAL) {
		sha_finish_req(ctx, &err);
	}

	if (unlikely(err != -EINPROGRESS)) {
		/* Task will not finish it, so do it here */
		sha_next_req(ctx, &err);
	}

	te200_ops->dev_put(te200_dd);
	return err;
}

static int asr_sha_cra_init(struct crypto_tfm *tfm)
{
	struct asr_sha_ctx *ctx = crypto_tfm_ctx(tfm);
	crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
				 sizeof(struct asr_sha_reqctx));
	ctx->start = asr_sha_start;

	return 0;
}

static void asr_sha_cra_exit(struct crypto_tfm *tfm)
{
	struct asr_sha_ctx *ctx = crypto_tfm_ctx(tfm);
	memset(ctx, 0, sizeof(*ctx));
}

static inline void asr_sha_get(struct asr_te200_sha *dd)
{
	mutex_lock(&dd->sha_lock);
}

static inline void asr_sha_put(struct asr_te200_sha *dd)
{
	if(mutex_is_locked(&dd->sha_lock))
		mutex_unlock(&dd->sha_lock);
}

static int asr_sha_init(struct ahash_request *req)
{
	struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);
	struct asr_te200_sha *dd = asr_sha_local;

	asr_sha_get(dd);

	ctx->dd = dd;
	ctx->flags = 0;
	ctx->alg = 0;

	switch (crypto_ahash_digestsize(tfm)) {
	case SHA1_DIGEST_SIZE:
		ctx->flags |= SHA_FLAGS_SHA1;
		ctx->alg = HASH_SHA1;
		break;
	case SHA224_DIGEST_SIZE:
		ctx->flags |= SHA_FLAGS_SHA224;
		ctx->alg = HASH_SHA224;
		break;
	case SHA256_DIGEST_SIZE:
		ctx->flags |= SHA_FLAGS_SHA256;
		ctx->alg = HASH_SHA256;
		break;
	default:
		asr_sha_put(dd);
		return -EINVAL;
	}

	ctx->bufcnt = 0;

	ctx->flags |= SHA_FLAGS_INIT;
	
	asr_sha_put(dd);
	return 0;
}

static int asr_sha_update(struct ahash_request *req)
{
	int ret = 0;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	asr_sha_get(ctx->dd);

	ctx->total = req->nbytes;
	ctx->sg = req->src;
	ctx->offset = 0;

	ret = asr_sha_enqueue(req, SHA_OP_UPDATE);

	asr_sha_put(ctx->dd);
	return ret;
}

static int asr_sha_final(struct ahash_request *req)
{
	int ret = 0;
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	asr_sha_get(ctx->dd);

	ctx->flags |= SHA_FLAGS_FINAL;
	if (ctx->flags & SHA_FLAGS_ERROR) {
		asr_sha_put(ctx->dd);
		return 0; /* uncompleted hash is not needed */
	}
	ret = asr_sha_enqueue(req, SHA_OP_FINAL);

	asr_sha_put(ctx->dd);
	return ret;
}

static int asr_sha_finup(struct ahash_request *req)
{
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);
	int err1, err2;

	ctx->flags |= SHA_FLAGS_FINUP;

	err1 = asr_sha_update(req);
	if (err1 == -EINPROGRESS ||
		(err1 == -EBUSY && (ahash_request_flags(req) &
				CRYPTO_TFM_REQ_MAY_BACKLOG))) {
		asr_sha_put(ctx->dd);
		return err1;
	}
	/*
	 * final() has to be always called to cleanup resources
	 * even if udpate() failed, except EINPROGRESS
	 */
	err2 = asr_sha_final(req);

	return err1 ?: err2;
}

static int asr_sha_digest(struct ahash_request *req)
{
	return asr_sha_init(req) ?: asr_sha_finup(req);
}

static int asr_sha_export(struct ahash_request *req, void *out)
{
	const struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	memcpy(out, ctx, sizeof(*ctx));
	return 0;
}

static int asr_sha_import(struct ahash_request *req, const void *in)
{
	struct asr_sha_reqctx *ctx = ahash_request_ctx(req);

	memcpy(ctx, in, sizeof(*ctx));
	return 0;
}

static struct ahash_alg sha_algs[] = {
	/* sha1 */
	{
		.init		= asr_sha_init,
		.update		= asr_sha_update,
		.final		= asr_sha_final,
		.finup		= asr_sha_finup,
		.digest		= asr_sha_digest,
		.export		= asr_sha_export,
		.import		= asr_sha_import,
		.halg = {
			.digestsize	= SHA1_DIGEST_SIZE,
			.statesize	= sizeof(struct asr_sha_reqctx),
			.base	= {
				.cra_name		= "sha1",
				.cra_driver_name	= "asr-sha1",
				.cra_priority		= 300,
				.cra_flags		= CRYPTO_ALG_ASYNC,
				.cra_blocksize		= SHA1_BLOCK_SIZE,
				.cra_ctxsize		= sizeof(struct asr_sha_ctx),
				.cra_alignmask		= 0,
				.cra_module		= THIS_MODULE,
				.cra_init		= asr_sha_cra_init,
				.cra_exit		= asr_sha_cra_exit,
			}
		}
	},
	/* sha224 */
	{
		.init		= asr_sha_init,
		.update		= asr_sha_update,
		.final		= asr_sha_final,
		.finup		= asr_sha_finup,
		.digest		= asr_sha_digest,
		.export		= asr_sha_export,
		.import		= asr_sha_import,
		.halg = {
			.digestsize	= SHA224_DIGEST_SIZE,
			.statesize	= sizeof(struct asr_sha_reqctx),
			.base	= {
				.cra_name		= "sha224",
				.cra_driver_name	= "asr-sha224",
				.cra_priority		= 300,
				.cra_flags		= CRYPTO_ALG_ASYNC,
				.cra_blocksize		= SHA224_BLOCK_SIZE,
				.cra_ctxsize		= sizeof(struct asr_sha_ctx),
				.cra_alignmask		= 0,
				.cra_module		= THIS_MODULE,
				.cra_init		= asr_sha_cra_init,
				.cra_exit		= asr_sha_cra_exit,
			}
		}
	},
	/* sha256 */
	{
		.init		= asr_sha_init,
		.update		= asr_sha_update,
		.final		= asr_sha_final,
		.finup		= asr_sha_finup,
		.digest		= asr_sha_digest,
		.export		= asr_sha_export,
		.import		= asr_sha_import,
		.halg = {
			.digestsize	= SHA256_DIGEST_SIZE,
			.statesize	= sizeof(struct asr_sha_reqctx),
			.base	= {
				.cra_name		= "sha256",
				.cra_driver_name	= "asr-sha256",
				.cra_priority		= 300,
				.cra_flags		= CRYPTO_ALG_ASYNC,
				.cra_blocksize		= SHA256_BLOCK_SIZE,
				.cra_ctxsize		= sizeof(struct asr_sha_ctx),
				.cra_alignmask		= 0,
				.cra_module		= THIS_MODULE,
				.cra_init		= asr_sha_cra_init,
				.cra_exit		= asr_sha_cra_exit,
			}
		}
	},
};

static void asr_sha_queue_task(unsigned long data)
{
	struct asr_te200_sha *dd = (struct asr_te200_sha *)data;

	asr_sha_handle_queue(dd, NULL);
}

static int asr_sha_done(struct asr_te200_sha *dd)
{
	int err = 0;

	if (SHA_FLAGS_OUTPUT_READY & dd->flags) {
		dd->flags &= ~SHA_FLAGS_OUTPUT_READY;
	}
	
	return err;
}

static void asr_sha_done_task(unsigned long data)
{
	struct asr_te200_sha *dd = (struct asr_te200_sha *)data;

	dd->is_async = true;
	(void)dd->resume(dd);
}

#ifdef ASR_TE200_SHA_TEST
	static int te200_sha_test(struct asr_te200_sha *dd);
#endif

int asr_te200_sha_register(struct asr_te200_dev *te200_dd)
{
	int err, i, j;
	struct device_node *np = NULL;
	struct asr_te200_sha *sha_dd;

	sha_dd = &te200_dd->asr_sha;

	sha_dd->dev = te200_dd->dev;
	sha_dd->io_base = te200_dd->io_base;
	sha_dd->phys_base = te200_dd->phys_base;

	np = sha_dd->dev->of_node;

	asr_sha_local = sha_dd;

	spin_lock_init(&sha_dd->lock);
	mutex_init(&sha_dd->sha_lock);
	tasklet_init(&sha_dd->done_task, asr_sha_done_task,
					(unsigned long)sha_dd);
	tasklet_init(&sha_dd->queue_task, asr_sha_queue_task,
					(unsigned long)sha_dd);
	crypto_init_queue(&sha_dd->queue, ASR_SHA_QUEUE_LENGTH);
	

	for (i = 0; i < ARRAY_SIZE(sha_algs); i++) {
		err = crypto_register_ahash(&sha_algs[i]);
		if (err)
			goto err_sha_algs;
	}

#ifdef ASR_TE200_SHA_TEST
	te200_sha_test(sha_dd);
#endif

	return 0;

err_sha_algs:
	for (j = 0; j < i; j++)
		crypto_unregister_ahash(&sha_algs[j]);

	return err;
}
EXPORT_SYMBOL_GPL(asr_te200_sha_register);

int asr_te200_sha_unregister(struct asr_te200_dev *te200_dd)
{
	int i;
	struct asr_te200_sha *sha_dd = &te200_dd->asr_sha;

	for (i = 0; i < ARRAY_SIZE(sha_algs); i++)
		crypto_unregister_ahash(&sha_algs[i]);

	tasklet_kill(&sha_dd->queue_task);
	tasklet_kill(&sha_dd->done_task);

	return 0;
}

#ifdef ASR_TE200_SHA_TEST
static int te200_sha_test(struct asr_te200_sha *dd)
{
	int ret = 0;
	
	const struct {
		const char *msg;
		uint8_t hash[20];
	} sha1_tests[] = {
		{
			"abc", 
			{   0xa9, 0x99, 0x3e, 0x36, 0x47, 0x06, 
				0x81, 0x6a, 0xba, 0x3e, 0x25, 0x71, 
				0x78, 0x50, 0xc2, 0x6c, 0x9c, 0xd0,
				0xd8, 0x9d 
			}
		},
		{
			"asjhsdjljfdsdjjkdfwyqeuwouzxkmcxjkmwqds"
			"jklfdfjlkdfkfsfkjlfskjdflioherfjjfdjkfd"
			"nkfdfdojjodfjdfjflj;sljjlfkklnfnkgbhhoi"
			"gfhigfopojpfjojpoffkjlfskjdflioherfjjfd"
			"jkfdnkfdfdojjodfjdfjfljnfnkgbhhoigfhigf"
			"oponfnkgbhhoigfhigfopojpfjoewiroiowiods"
			"djkisijdknknkskdnknflnnesniewinoinknmdn"
			"kknknsdnjjfsnnkfnkknslnklknfnknkflksnlk"
			"lskldklklklnmlflmlmlfmlfml",
			{
				0xc4, 0x53, 0xca, 0x24, 0xfa, 0xe5,
				0x39, 0x53, 0x08, 0x8c, 0x57, 0x1a, 
				0x96, 0xe9, 0x64, 0x7f, 0xd5, 0xf9, 
				0x13, 0x91
			}
		},
		{
			"asjhsdjljfdsdjjkdfwyqeuwouzxkmcxjkmwqdsjklfdfjlkdfkfs"
			"fkjlfskjdflioherfjjfdjkfdnkfdfdojjodfjdfjflj;sljjlfkkl"
			"nfnkgbhhoigfhigfopojpfjojpoffkjlfskjdflioherfjjfdjkfdn"
			"kfdfdojjodfjdfjfljnfnkgbhhoigfhigfoponfnkgbhhoigfhigfo"
			"pojpfjoewiroiowiodsdjkisijdknknkskdnknflnnesniewinoinkn"
			"mdnkknknsdnjjfsnnkfnkknslnklknfnknkflksnlklskldklklklnm"
			"lflmlmlfmlfml",
			{
				0xc4, 0x53, 0xca, 0x24, 0xfa, 0xe5,
				0x39, 0x53, 0x08, 0x8c, 0x57, 0x1a, 
				0x96, 0xe9, 0x64, 0x7f, 0xd5, 0xf9, 
				0x13, 0x91
			}
		}
	};

	struct asr_sha_reqctx ctx1;
	struct asr_sha_reqctx ctx2;
	struct asr_sha_reqctx ctx3;

	unsigned char out_sha1_1[20] = {0};
	unsigned char out_sha1_2[20] = {0};
	unsigned char out_sha1_3[20] = {0};

	ctx1.dd = dd;
	ctx2.dd = dd;
	ctx3.dd = dd;

	ret = hash_op_init(&ctx1, HASH_SHA1);
	if (ret)
		return ret;
	ret = hash_op_proc(&ctx1, (uint8_t *)sha1_tests[0].msg, strlen(sha1_tests[0].msg));
	if (ret)
		return ret;
	ret = hash_op_init(&ctx2, HASH_SHA1);
	if (ret)
		return ret;
	ret = hash_op_proc(&ctx2, (uint8_t *)sha1_tests[1].msg, 10);
	if (ret)
		return ret;
	ret = hash_op_finish(&ctx1, out_sha1_1, sizeof(out_sha1_1));
	if (ret)
		return ret;
	ret = hash_op_init(&ctx3, HASH_SHA1);
	if (ret)
		return ret;
	ret = hash_op_proc(&ctx2, (uint8_t *)sha1_tests[1].msg+10, strlen(sha1_tests[1].msg)-10);
	if (ret)
		return ret;
	ret = hash_op_proc(&ctx3, (uint8_t *)sha1_tests[2].msg, 23);
	if (ret)
		return ret;
	ret = hash_op_finish(&ctx2, out_sha1_2, sizeof(out_sha1_2));
	if (ret)
		return ret;
	ret = hash_op_proc(&ctx3, (uint8_t *)sha1_tests[2].msg+23, strlen(sha1_tests[2].msg)-23);
	if (ret)
		return ret;
	ret = hash_op_finish(&ctx3, out_sha1_3, sizeof(out_sha1_3));
	if (ret)
		return ret;

	if (memcmp(out_sha1_1, sha1_tests[0].hash, sizeof(out_sha1_1))) {
		printk("sha1 test 0 failed");
	} else {
		printk("sha1 test 0 pass");
	}
	if (memcmp(out_sha1_2, sha1_tests[1].hash, sizeof(out_sha1_2))) {
		printk("sha1 test 1 failed");
	} else {
		printk("sha1 test 1 pass");
	}
	if (memcmp(out_sha1_3, sha1_tests[2].hash, sizeof(out_sha1_3))) {
		printk("sha1 test 2 failed");
	} else {
		printk("sha1 test 2 pass");
	}

	return 0;
}
#endif

EXPORT_SYMBOL_GPL(asr_te200_sha_unregister);