[T106][ZXW-22]7520V3SCV2.01.01.02P42U09_VEC_V0.8_AP_VEC origin source commit
Change-Id: Ic6e05d89ecd62fc34f82b23dcf306c93764aec4b
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/Kconfig b/ap/os/linux/linux-3.4.x/drivers/crypto/Kconfig
new file mode 100644
index 0000000..dd414d9
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/Kconfig
@@ -0,0 +1,299 @@
+
+menuconfig CRYPTO_HW
+ bool "Hardware crypto devices"
+ default y
+ ---help---
+ Say Y here to get to see options for hardware crypto devices and
+ processors. This option alone does not add any kernel code.
+
+ If you say N, all options in this submenu will be skipped and disabled.
+
+if CRYPTO_HW
+
+config CRYPTO_DEV_PADLOCK
+ tristate "Support for VIA PadLock ACE"
+ depends on X86 && !UML
+ help
+ Some VIA processors come with an integrated crypto engine
+ (so called VIA PadLock ACE, Advanced Cryptography Engine)
+ that provides instructions for very fast cryptographic
+ operations with supported algorithms.
+
+ The instructions are used only when the CPU supports them.
+ Otherwise software encryption is used.
+
+config CRYPTO_DEV_PADLOCK_AES
+ tristate "PadLock driver for AES algorithm"
+ depends on CRYPTO_DEV_PADLOCK
+ select CRYPTO_BLKCIPHER
+ select CRYPTO_AES
+ help
+ Use VIA PadLock for AES algorithm.
+
+ Available in VIA C3 and newer CPUs.
+
+ If unsure say M. The compiled module will be
+ called padlock-aes.
+
+config CRYPTO_DEV_PADLOCK_SHA
+ tristate "PadLock driver for SHA1 and SHA256 algorithms"
+ depends on CRYPTO_DEV_PADLOCK
+ select CRYPTO_HASH
+ select CRYPTO_SHA1
+ select CRYPTO_SHA256
+ help
+ Use VIA PadLock for SHA1/SHA256 algorithms.
+
+ Available in VIA C7 and newer processors.
+
+ If unsure say M. The compiled module will be
+ called padlock-sha.
+
+config CRYPTO_DEV_GEODE
+ tristate "Support for the Geode LX AES engine"
+ depends on X86_32 && PCI
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ help
+ Say 'Y' here to use the AMD Geode LX processor on-board AES
+ engine for the CryptoAPI AES algorithm.
+
+ To compile this driver as a module, choose M here: the module
+ will be called geode-aes.
+
+config ZCRYPT
+ tristate "Support for PCI-attached cryptographic adapters"
+ depends on S390
+ select HW_RANDOM
+ help
+ Select this option if you want to use a PCI-attached cryptographic
+ adapter like:
+ + PCI Cryptographic Accelerator (PCICA)
+ + PCI Cryptographic Coprocessor (PCICC)
+ + PCI-X Cryptographic Coprocessor (PCIXCC)
+ + Crypto Express2 Coprocessor (CEX2C)
+ + Crypto Express2 Accelerator (CEX2A)
+ + Crypto Express3 Coprocessor (CEX3C)
+ + Crypto Express3 Accelerator (CEX3A)
+
+config CRYPTO_SHA1_S390
+ tristate "SHA1 digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ SHA-1 secure hash standard (FIPS 180-1/DFIPS 180-2).
+
+ It is available as of z990.
+
+config CRYPTO_SHA256_S390
+ tristate "SHA256 digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ SHA256 secure hash standard (DFIPS 180-2).
+
+ It is available as of z9.
+
+config CRYPTO_SHA512_S390
+ tristate "SHA384 and SHA512 digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ SHA512 secure hash standard.
+
+ It is available as of z10.
+
+config CRYPTO_DES_S390
+ tristate "DES and Triple DES cipher algorithms"
+ depends on S390
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ help
+ This is the s390 hardware accelerated implementation of the
+ DES cipher algorithm (FIPS 46-2), and Triple DES EDE (FIPS 46-3).
+
+ As of z990 the ECB and CBC mode are hardware accelerated.
+ As of z196 the CTR mode is hardware accelerated.
+
+config CRYPTO_AES_S390
+ tristate "AES cipher algorithms"
+ depends on S390
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ help
+ This is the s390 hardware accelerated implementation of the
+ AES cipher algorithms (FIPS-197).
+
+ As of z9 the ECB and CBC modes are hardware accelerated
+ for 128 bit keys.
+ As of z10 the ECB and CBC modes are hardware accelerated
+ for all AES key sizes.
+ As of z196 the CTR mode is hardware accelerated for all AES
+ key sizes and XTS mode is hardware accelerated for 256 and
+ 512 bit keys.
+
+config S390_PRNG
+ tristate "Pseudo random number generator device driver"
+ depends on S390
+ default "m"
+ help
+ Select this option if you want to use the s390 pseudo random number
+ generator. The PRNG is part of the cryptographic processor functions
+ and uses triple-DES to generate secure random numbers like the
+ ANSI X9.17 standard. User-space programs access the
+ pseudo-random-number device through the char device /dev/prandom.
+
+ It is available as of z9.
+
+config CRYPTO_GHASH_S390
+ tristate "GHASH digest algorithm"
+ depends on S390
+ select CRYPTO_HASH
+ help
+ This is the s390 hardware accelerated implementation of the
+ GHASH message digest algorithm for GCM (Galois/Counter Mode).
+
+ It is available as of z196.
+
+config CRYPTO_DEV_MV_CESA
+ tristate "Marvell's Cryptographic Engine"
+ depends on PLAT_ORION
+ select CRYPTO_ALGAPI
+ select CRYPTO_AES
+ select CRYPTO_BLKCIPHER2
+ select CRYPTO_HASH
+ help
+ This driver allows you to utilize the Cryptographic Engines and
+ Security Accelerator (CESA) which can be found on the Marvell Orion
+ and Kirkwood SoCs, such as QNAP's TS-209.
+
+ Currently the driver supports AES in ECB and CBC mode without DMA.
+
+config CRYPTO_DEV_NIAGARA2
+ tristate "Niagara2 Stream Processing Unit driver"
+ select CRYPTO_DES
+ select CRYPTO_ALGAPI
+ depends on SPARC64
+ help
+ Each core of a Niagara2 processor contains a Stream
+ Processing Unit, which itself contains several cryptographic
+ sub-units. One set provides the Modular Arithmetic Unit,
+ used for SSL offload. The other set provides the Cipher
+ Group, which can perform encryption, decryption, hashing,
+ checksumming, and raw copies.
+
+config CRYPTO_DEV_HIFN_795X
+ tristate "Driver HIFN 795x crypto accelerator chips"
+ select CRYPTO_DES
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ select HW_RANDOM if CRYPTO_DEV_HIFN_795X_RNG
+ depends on PCI
+ depends on !ARCH_DMA_ADDR_T_64BIT
+ help
+ This option allows you to have support for HIFN 795x crypto adapters.
+
+config CRYPTO_DEV_HIFN_795X_RNG
+ bool "HIFN 795x random number generator"
+ depends on CRYPTO_DEV_HIFN_795X
+ help
+ Select this option if you want to enable the random number generator
+ on the HIFN 795x crypto adapters.
+
+source drivers/crypto/caam/Kconfig
+
+config CRYPTO_DEV_TALITOS
+ tristate "Talitos Freescale Security Engine (SEC)"
+ select CRYPTO_ALGAPI
+ select CRYPTO_AUTHENC
+ select HW_RANDOM
+ depends on FSL_SOC
+ help
+ Say 'Y' here to use the Freescale Security Engine (SEC)
+ to offload cryptographic algorithm computation.
+
+ The Freescale SEC is present on PowerQUICC 'E' processors, such
+ as the MPC8349E and MPC8548E.
+
+ To compile this driver as a module, choose M here: the module
+ will be called talitos.
+
+config CRYPTO_DEV_IXP4XX
+ tristate "Driver for IXP4xx crypto hardware acceleration"
+ depends on ARCH_IXP4XX
+ select CRYPTO_DES
+ select CRYPTO_ALGAPI
+ select CRYPTO_AUTHENC
+ select CRYPTO_BLKCIPHER
+ help
+ Driver for the IXP4xx NPE crypto engine.
+
+config CRYPTO_DEV_PPC4XX
+ tristate "Driver AMCC PPC4xx crypto accelerator"
+ depends on PPC && 4xx
+ select CRYPTO_HASH
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ help
+ This option allows you to have support for AMCC crypto acceleration.
+
+config CRYPTO_DEV_OMAP_SHAM
+ tristate "Support for OMAP SHA1/MD5 hw accelerator"
+ depends on ARCH_OMAP2 || ARCH_OMAP3
+ select CRYPTO_SHA1
+ select CRYPTO_MD5
+ help
+ OMAP processors have SHA1/MD5 hw accelerator. Select this if you
+ want to use the OMAP module for SHA1/MD5 algorithms.
+
+config CRYPTO_DEV_OMAP_AES
+ tristate "Support for OMAP AES hw engine"
+ depends on ARCH_OMAP2 || ARCH_OMAP3
+ select CRYPTO_AES
+ help
+ OMAP processors have AES module accelerator. Select this if you
+ want to use the OMAP module for AES algorithms.
+
+config CRYPTO_DEV_PICOXCELL
+ tristate "Support for picoXcell IPSEC and Layer2 crypto engines"
+ depends on ARCH_PICOXCELL && HAVE_CLK
+ select CRYPTO_AES
+ select CRYPTO_AUTHENC
+ select CRYPTO_ALGAPI
+ select CRYPTO_DES
+ select CRYPTO_CBC
+ select CRYPTO_ECB
+ select CRYPTO_SEQIV
+ help
+ This option enables support for the hardware offload engines in the
+ Picochip picoXcell SoC devices. Select this for IPSEC ESP offload
+ and for 3gpp Layer 2 ciphering support.
+
+ Saying m here will build a module named pipcoxcell_crypto.
+
+config CRYPTO_DEV_S5P
+ tristate "Support for Samsung S5PV210 crypto accelerator"
+ depends on ARCH_S5PV210
+ select CRYPTO_AES
+ select CRYPTO_ALGAPI
+ select CRYPTO_BLKCIPHER
+ help
+ This option allows you to have support for S5P crypto acceleration.
+ Select this to offload Samsung S5PV210 or S5PC110 from AES
+ algorithms execution.
+
+config CRYPTO_DEV_TEGRA_AES
+ tristate "Support for TEGRA AES hw engine"
+ depends on ARCH_TEGRA
+ select CRYPTO_AES
+ help
+ TEGRA processors have AES module accelerator. Select this if you
+ want to use the TEGRA module for AES algorithms.
+
+ To compile this driver as a module, choose M here: the module
+ will be called tegra-aes.
+
+endif # CRYPTO_HW
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/Makefile b/ap/os/linux/linux-3.4.x/drivers/crypto/Makefile
new file mode 100644
index 0000000..f3e64ea
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/Makefile
@@ -0,0 +1,16 @@
+obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
+obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
+obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o
+obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o
+n2_crypto-y := n2_core.o n2_asm.o
+obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o
+obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o
+obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam/
+obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o
+obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/
+obj-$(CONFIG_CRYPTO_DEV_OMAP_SHAM) += omap-sham.o
+obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes.o
+obj-$(CONFIG_CRYPTO_DEV_PICOXCELL) += picoxcell_crypto.o
+obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o
+obj-$(CONFIG_CRYPTO_DEV_TEGRA_AES) += tegra-aes.o
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/Makefile b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/Makefile
new file mode 100644
index 0000000..5c0c62b
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/Makefile
@@ -0,0 +1,2 @@
+obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += crypto4xx.o
+crypto4xx-y := crypto4xx_core.o crypto4xx_alg.o crypto4xx_sa.o
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_alg.c b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_alg.c
new file mode 100644
index 0000000..a33243c
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_alg.c
@@ -0,0 +1,294 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * This file implements the Linux crypto algorithms.
+ */
+
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock_types.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <linux/hash.h>
+#include <crypto/internal/hash.h>
+#include <linux/dma-mapping.h>
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include "crypto4xx_reg_def.h"
+#include "crypto4xx_sa.h"
+#include "crypto4xx_core.h"
+
+void set_dynamic_sa_command_0(struct dynamic_sa_ctl *sa, u32 save_h,
+ u32 save_iv, u32 ld_h, u32 ld_iv, u32 hdr_proc,
+ u32 h, u32 c, u32 pad_type, u32 op_grp, u32 op,
+ u32 dir)
+{
+ sa->sa_command_0.w = 0;
+ sa->sa_command_0.bf.save_hash_state = save_h;
+ sa->sa_command_0.bf.save_iv = save_iv;
+ sa->sa_command_0.bf.load_hash_state = ld_h;
+ sa->sa_command_0.bf.load_iv = ld_iv;
+ sa->sa_command_0.bf.hdr_proc = hdr_proc;
+ sa->sa_command_0.bf.hash_alg = h;
+ sa->sa_command_0.bf.cipher_alg = c;
+ sa->sa_command_0.bf.pad_type = pad_type & 3;
+ sa->sa_command_0.bf.extend_pad = pad_type >> 2;
+ sa->sa_command_0.bf.op_group = op_grp;
+ sa->sa_command_0.bf.opcode = op;
+ sa->sa_command_0.bf.dir = dir;
+}
+
+void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, u32 hmac_mc,
+ u32 cfb, u32 esn, u32 sn_mask, u32 mute,
+ u32 cp_pad, u32 cp_pay, u32 cp_hdr)
+{
+ sa->sa_command_1.w = 0;
+ sa->sa_command_1.bf.crypto_mode31 = (cm & 4) >> 2;
+ sa->sa_command_1.bf.crypto_mode9_8 = cm & 3;
+ sa->sa_command_1.bf.feedback_mode = cfb,
+ sa->sa_command_1.bf.sa_rev = 1;
+ sa->sa_command_1.bf.extended_seq_num = esn;
+ sa->sa_command_1.bf.seq_num_mask = sn_mask;
+ sa->sa_command_1.bf.mutable_bit_proc = mute;
+ sa->sa_command_1.bf.copy_pad = cp_pad;
+ sa->sa_command_1.bf.copy_payload = cp_pay;
+ sa->sa_command_1.bf.copy_hdr = cp_hdr;
+}
+
+int crypto4xx_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+ ctx->direction = DIR_OUTBOUND;
+ ctx->hash_final = 0;
+ ctx->is_hash = 0;
+ ctx->pd_ctl = 0x1;
+
+ return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
+ req->nbytes, req->info,
+ get_dynamic_sa_iv_size(ctx));
+}
+
+int crypto4xx_decrypt(struct ablkcipher_request *req)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+ ctx->direction = DIR_INBOUND;
+ ctx->hash_final = 0;
+ ctx->is_hash = 0;
+ ctx->pd_ctl = 1;
+
+ return crypto4xx_build_pd(&req->base, ctx, req->src, req->dst,
+ req->nbytes, req->info,
+ get_dynamic_sa_iv_size(ctx));
+}
+
+/**
+ * AES Functions
+ */
+static int crypto4xx_setkey_aes(struct crypto_ablkcipher *cipher,
+ const u8 *key,
+ unsigned int keylen,
+ unsigned char cm,
+ u8 fb)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct dynamic_sa_ctl *sa;
+ int rc;
+
+ if (keylen != AES_KEYSIZE_256 &&
+ keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_128) {
+ crypto_ablkcipher_set_flags(cipher,
+ CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ /* Create SA */
+ if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
+ crypto4xx_free_sa(ctx);
+
+ rc = crypto4xx_alloc_sa(ctx, SA_AES128_LEN + (keylen-16) / 4);
+ if (rc)
+ return rc;
+
+ if (ctx->state_record_dma_addr == 0) {
+ rc = crypto4xx_alloc_state_record(ctx);
+ if (rc) {
+ crypto4xx_free_sa(ctx);
+ return rc;
+ }
+ }
+ /* Setup SA */
+ sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+ ctx->hash_final = 0;
+
+ set_dynamic_sa_command_0(sa, SA_NOT_SAVE_HASH, SA_NOT_SAVE_IV,
+ SA_LOAD_HASH_FROM_SA, SA_LOAD_IV_FROM_STATE,
+ SA_NO_HEADER_PROC, SA_HASH_ALG_NULL,
+ SA_CIPHER_ALG_AES, SA_PAD_TYPE_ZERO,
+ SA_OP_GROUP_BASIC, SA_OPCODE_DECRYPT,
+ DIR_INBOUND);
+
+ set_dynamic_sa_command_1(sa, cm, SA_HASH_MODE_HASH,
+ fb, SA_EXTENDED_SN_OFF,
+ SA_SEQ_MASK_OFF, SA_MC_ENABLE,
+ SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
+ SA_NOT_COPY_HDR);
+ crypto4xx_memcpy_le(ctx->sa_in + get_dynamic_sa_offset_key_field(ctx),
+ key, keylen);
+ sa->sa_contents = SA_AES_CONTENTS | (keylen << 2);
+ sa->sa_command_1.bf.key_len = keylen >> 3;
+ ctx->is_hash = 0;
+ ctx->direction = DIR_INBOUND;
+ memcpy(ctx->sa_in + get_dynamic_sa_offset_state_ptr_field(ctx),
+ (void *)&ctx->state_record_dma_addr, 4);
+ ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
+
+ memcpy(ctx->sa_out, ctx->sa_in, ctx->sa_len * 4);
+ sa = (struct dynamic_sa_ctl *) ctx->sa_out;
+ sa->sa_command_0.bf.dir = DIR_OUTBOUND;
+
+ return 0;
+}
+
+int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ return crypto4xx_setkey_aes(cipher, key, keylen, CRYPTO_MODE_CBC,
+ CRYPTO_FEEDBACK_MODE_NO_FB);
+}
+
+/**
+ * HASH SHA1 Functions
+ */
+static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm,
+ unsigned int sa_len,
+ unsigned char ha,
+ unsigned char hm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct crypto4xx_alg *my_alg = crypto_alg_to_crypto4xx_alg(alg);
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct dynamic_sa_ctl *sa;
+ struct dynamic_sa_hash160 *sa_in;
+ int rc;
+
+ ctx->dev = my_alg->dev;
+ ctx->is_hash = 1;
+ ctx->hash_final = 0;
+
+ /* Create SA */
+ if (ctx->sa_in_dma_addr || ctx->sa_out_dma_addr)
+ crypto4xx_free_sa(ctx);
+
+ rc = crypto4xx_alloc_sa(ctx, sa_len);
+ if (rc)
+ return rc;
+
+ if (ctx->state_record_dma_addr == 0) {
+ crypto4xx_alloc_state_record(ctx);
+ if (!ctx->state_record_dma_addr) {
+ crypto4xx_free_sa(ctx);
+ return -ENOMEM;
+ }
+ }
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct crypto4xx_ctx));
+ sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+ set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV,
+ SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA,
+ SA_NO_HEADER_PROC, ha, SA_CIPHER_ALG_NULL,
+ SA_PAD_TYPE_ZERO, SA_OP_GROUP_BASIC,
+ SA_OPCODE_HASH, DIR_INBOUND);
+ set_dynamic_sa_command_1(sa, 0, SA_HASH_MODE_HASH,
+ CRYPTO_FEEDBACK_MODE_NO_FB, SA_EXTENDED_SN_OFF,
+ SA_SEQ_MASK_OFF, SA_MC_ENABLE,
+ SA_NOT_COPY_PAD, SA_NOT_COPY_PAYLOAD,
+ SA_NOT_COPY_HDR);
+ ctx->direction = DIR_INBOUND;
+ sa->sa_contents = SA_HASH160_CONTENTS;
+ sa_in = (struct dynamic_sa_hash160 *) ctx->sa_in;
+ /* Need to zero hash digest in SA */
+ memset(sa_in->inner_digest, 0, sizeof(sa_in->inner_digest));
+ memset(sa_in->outer_digest, 0, sizeof(sa_in->outer_digest));
+ sa_in->state_ptr = ctx->state_record_dma_addr;
+ ctx->offset_to_sr_ptr = get_dynamic_sa_offset_state_ptr_field(ctx);
+
+ return 0;
+}
+
+int crypto4xx_hash_init(struct ahash_request *req)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ int ds;
+ struct dynamic_sa_ctl *sa;
+
+ sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+ ds = crypto_ahash_digestsize(
+ __crypto_ahash_cast(req->base.tfm));
+ sa->sa_command_0.bf.digest_len = ds >> 2;
+ sa->sa_command_0.bf.load_hash_state = SA_LOAD_HASH_FROM_SA;
+ ctx->is_hash = 1;
+ ctx->direction = DIR_INBOUND;
+
+ return 0;
+}
+
+int crypto4xx_hash_update(struct ahash_request *req)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+ ctx->is_hash = 1;
+ ctx->hash_final = 0;
+ ctx->pd_ctl = 0x11;
+ ctx->direction = DIR_INBOUND;
+
+ return crypto4xx_build_pd(&req->base, ctx, req->src,
+ (struct scatterlist *) req->result,
+ req->nbytes, NULL, 0);
+}
+
+int crypto4xx_hash_final(struct ahash_request *req)
+{
+ return 0;
+}
+
+int crypto4xx_hash_digest(struct ahash_request *req)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+
+ ctx->hash_final = 1;
+ ctx->pd_ctl = 0x11;
+ ctx->direction = DIR_INBOUND;
+
+ return crypto4xx_build_pd(&req->base, ctx, req->src,
+ (struct scatterlist *) req->result,
+ req->nbytes, NULL, 0);
+}
+
+/**
+ * SHA1 Algorithm
+ */
+int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm)
+{
+ return crypto4xx_hash_alg_init(tfm, SA_HASH160_LEN, SA_HASH_ALG_SHA1,
+ SA_HASH_MODE_HASH);
+}
+
+
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.c b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.c
new file mode 100644
index 0000000..13f8e1a
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.c
@@ -0,0 +1,1300 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * This file implements AMCC crypto offload Linux device driver for use with
+ * Linux CryptoAPI.
+ */
+
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock_types.h>
+#include <linux/random.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
+#include <linux/platform_device.h>
+#include <linux/init.h>
+#include <linux/of_platform.h>
+#include <linux/slab.h>
+#include <asm/dcr.h>
+#include <asm/dcr-regs.h>
+#include <asm/cacheflush.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include "crypto4xx_reg_def.h"
+#include "crypto4xx_core.h"
+#include "crypto4xx_sa.h"
+
+#define PPC4XX_SEC_VERSION_STR "0.5"
+
+/**
+ * PPC4xx Crypto Engine Initialization Routine
+ */
+static void crypto4xx_hw_init(struct crypto4xx_device *dev)
+{
+ union ce_ring_size ring_size;
+ union ce_ring_contol ring_ctrl;
+ union ce_part_ring_size part_ring_size;
+ union ce_io_threshold io_threshold;
+ u32 rand_num;
+ union ce_pe_dma_cfg pe_dma_cfg;
+ u32 device_ctrl;
+
+ writel(PPC4XX_BYTE_ORDER, dev->ce_base + CRYPTO4XX_BYTE_ORDER_CFG);
+ /* setup pe dma, include reset sg, pdr and pe, then release reset */
+ pe_dma_cfg.w = 0;
+ pe_dma_cfg.bf.bo_sgpd_en = 1;
+ pe_dma_cfg.bf.bo_data_en = 0;
+ pe_dma_cfg.bf.bo_sa_en = 1;
+ pe_dma_cfg.bf.bo_pd_en = 1;
+ pe_dma_cfg.bf.dynamic_sa_en = 1;
+ pe_dma_cfg.bf.reset_sg = 1;
+ pe_dma_cfg.bf.reset_pdr = 1;
+ pe_dma_cfg.bf.reset_pe = 1;
+ writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+ /* un reset pe,sg and pdr */
+ pe_dma_cfg.bf.pe_mode = 0;
+ pe_dma_cfg.bf.reset_sg = 0;
+ pe_dma_cfg.bf.reset_pdr = 0;
+ pe_dma_cfg.bf.reset_pe = 0;
+ pe_dma_cfg.bf.bo_td_en = 0;
+ writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+ writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_PDR_BASE);
+ writel(dev->pdr_pa, dev->ce_base + CRYPTO4XX_RDR_BASE);
+ writel(PPC4XX_PRNG_CTRL_AUTO_EN, dev->ce_base + CRYPTO4XX_PRNG_CTRL);
+ get_random_bytes(&rand_num, sizeof(rand_num));
+ writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_L);
+ get_random_bytes(&rand_num, sizeof(rand_num));
+ writel(rand_num, dev->ce_base + CRYPTO4XX_PRNG_SEED_H);
+ ring_size.w = 0;
+ ring_size.bf.ring_offset = PPC4XX_PD_SIZE;
+ ring_size.bf.ring_size = PPC4XX_NUM_PD;
+ writel(ring_size.w, dev->ce_base + CRYPTO4XX_RING_SIZE);
+ ring_ctrl.w = 0;
+ writel(ring_ctrl.w, dev->ce_base + CRYPTO4XX_RING_CTRL);
+ device_ctrl = readl(dev->ce_base + CRYPTO4XX_DEVICE_CTRL);
+ device_ctrl |= PPC4XX_DC_3DES_EN;
+ writel(device_ctrl, dev->ce_base + CRYPTO4XX_DEVICE_CTRL);
+ writel(dev->gdr_pa, dev->ce_base + CRYPTO4XX_GATH_RING_BASE);
+ writel(dev->sdr_pa, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE);
+ part_ring_size.w = 0;
+ part_ring_size.bf.sdr_size = PPC4XX_SDR_SIZE;
+ part_ring_size.bf.gdr_size = PPC4XX_GDR_SIZE;
+ writel(part_ring_size.w, dev->ce_base + CRYPTO4XX_PART_RING_SIZE);
+ writel(PPC4XX_SD_BUFFER_SIZE, dev->ce_base + CRYPTO4XX_PART_RING_CFG);
+ io_threshold.w = 0;
+ io_threshold.bf.output_threshold = PPC4XX_OUTPUT_THRESHOLD;
+ io_threshold.bf.input_threshold = PPC4XX_INPUT_THRESHOLD;
+ writel(io_threshold.w, dev->ce_base + CRYPTO4XX_IO_THRESHOLD);
+ writel(0, dev->ce_base + CRYPTO4XX_PDR_BASE_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_RDR_BASE_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_PKT_SRC_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_PKT_DEST_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_SA_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_GATH_RING_BASE_UADDR);
+ writel(0, dev->ce_base + CRYPTO4XX_SCAT_RING_BASE_UADDR);
+ /* un reset pe,sg and pdr */
+ pe_dma_cfg.bf.pe_mode = 1;
+ pe_dma_cfg.bf.reset_sg = 0;
+ pe_dma_cfg.bf.reset_pdr = 0;
+ pe_dma_cfg.bf.reset_pe = 0;
+ pe_dma_cfg.bf.bo_td_en = 0;
+ writel(pe_dma_cfg.w, dev->ce_base + CRYPTO4XX_PE_DMA_CFG);
+ /*clear all pending interrupt*/
+ writel(PPC4XX_INTERRUPT_CLR, dev->ce_base + CRYPTO4XX_INT_CLR);
+ writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
+ writel(PPC4XX_INT_DESCR_CNT, dev->ce_base + CRYPTO4XX_INT_DESCR_CNT);
+ writel(PPC4XX_INT_CFG, dev->ce_base + CRYPTO4XX_INT_CFG);
+ writel(PPC4XX_PD_DONE_INT, dev->ce_base + CRYPTO4XX_INT_EN);
+}
+
+int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size)
+{
+ ctx->sa_in = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4,
+ &ctx->sa_in_dma_addr, GFP_ATOMIC);
+ if (ctx->sa_in == NULL)
+ return -ENOMEM;
+
+ ctx->sa_out = dma_alloc_coherent(ctx->dev->core_dev->device, size * 4,
+ &ctx->sa_out_dma_addr, GFP_ATOMIC);
+ if (ctx->sa_out == NULL) {
+ dma_free_coherent(ctx->dev->core_dev->device,
+ ctx->sa_len * 4,
+ ctx->sa_in, ctx->sa_in_dma_addr);
+ return -ENOMEM;
+ }
+
+ memset(ctx->sa_in, 0, size * 4);
+ memset(ctx->sa_out, 0, size * 4);
+ ctx->sa_len = size;
+
+ return 0;
+}
+
+void crypto4xx_free_sa(struct crypto4xx_ctx *ctx)
+{
+ if (ctx->sa_in != NULL)
+ dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4,
+ ctx->sa_in, ctx->sa_in_dma_addr);
+ if (ctx->sa_out != NULL)
+ dma_free_coherent(ctx->dev->core_dev->device, ctx->sa_len * 4,
+ ctx->sa_out, ctx->sa_out_dma_addr);
+
+ ctx->sa_in_dma_addr = 0;
+ ctx->sa_out_dma_addr = 0;
+ ctx->sa_len = 0;
+}
+
+u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx)
+{
+ ctx->state_record = dma_alloc_coherent(ctx->dev->core_dev->device,
+ sizeof(struct sa_state_record),
+ &ctx->state_record_dma_addr, GFP_ATOMIC);
+ if (!ctx->state_record_dma_addr)
+ return -ENOMEM;
+ memset(ctx->state_record, 0, sizeof(struct sa_state_record));
+
+ return 0;
+}
+
+void crypto4xx_free_state_record(struct crypto4xx_ctx *ctx)
+{
+ if (ctx->state_record != NULL)
+ dma_free_coherent(ctx->dev->core_dev->device,
+ sizeof(struct sa_state_record),
+ ctx->state_record,
+ ctx->state_record_dma_addr);
+ ctx->state_record_dma_addr = 0;
+}
+
+/**
+ * alloc memory for the gather ring
+ * no need to alloc buf for the ring
+ * gdr_tail, gdr_head and gdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev)
+{
+ int i;
+ struct pd_uinfo *pd_uinfo;
+ dev->pdr = dma_alloc_coherent(dev->core_dev->device,
+ sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+ &dev->pdr_pa, GFP_ATOMIC);
+ if (!dev->pdr)
+ return -ENOMEM;
+
+ dev->pdr_uinfo = kzalloc(sizeof(struct pd_uinfo) * PPC4XX_NUM_PD,
+ GFP_KERNEL);
+ if (!dev->pdr_uinfo) {
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+ dev->pdr,
+ dev->pdr_pa);
+ return -ENOMEM;
+ }
+ memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD);
+ dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device,
+ 256 * PPC4XX_NUM_PD,
+ &dev->shadow_sa_pool_pa,
+ GFP_ATOMIC);
+ if (!dev->shadow_sa_pool)
+ return -ENOMEM;
+
+ dev->shadow_sr_pool = dma_alloc_coherent(dev->core_dev->device,
+ sizeof(struct sa_state_record) * PPC4XX_NUM_PD,
+ &dev->shadow_sr_pool_pa, GFP_ATOMIC);
+ if (!dev->shadow_sr_pool)
+ return -ENOMEM;
+ for (i = 0; i < PPC4XX_NUM_PD; i++) {
+ pd_uinfo = (struct pd_uinfo *) (dev->pdr_uinfo +
+ sizeof(struct pd_uinfo) * i);
+
+ /* alloc 256 bytes which is enough for any kind of dynamic sa */
+ pd_uinfo->sa_va = dev->shadow_sa_pool + 256 * i;
+ pd_uinfo->sa_pa = dev->shadow_sa_pool_pa + 256 * i;
+
+ /* alloc state record */
+ pd_uinfo->sr_va = dev->shadow_sr_pool +
+ sizeof(struct sa_state_record) * i;
+ pd_uinfo->sr_pa = dev->shadow_sr_pool_pa +
+ sizeof(struct sa_state_record) * i;
+ }
+
+ return 0;
+}
+
+static void crypto4xx_destroy_pdr(struct crypto4xx_device *dev)
+{
+ if (dev->pdr != NULL)
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct ce_pd) * PPC4XX_NUM_PD,
+ dev->pdr, dev->pdr_pa);
+ if (dev->shadow_sa_pool)
+ dma_free_coherent(dev->core_dev->device, 256 * PPC4XX_NUM_PD,
+ dev->shadow_sa_pool, dev->shadow_sa_pool_pa);
+ if (dev->shadow_sr_pool)
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct sa_state_record) * PPC4XX_NUM_PD,
+ dev->shadow_sr_pool, dev->shadow_sr_pool_pa);
+
+ kfree(dev->pdr_uinfo);
+}
+
+static u32 crypto4xx_get_pd_from_pdr_nolock(struct crypto4xx_device *dev)
+{
+ u32 retval;
+ u32 tmp;
+
+ retval = dev->pdr_head;
+ tmp = (dev->pdr_head + 1) % PPC4XX_NUM_PD;
+
+ if (tmp == dev->pdr_tail)
+ return ERING_WAS_FULL;
+
+ dev->pdr_head = tmp;
+
+ return retval;
+}
+
+static u32 crypto4xx_put_pd_to_pdr(struct crypto4xx_device *dev, u32 idx)
+{
+ struct pd_uinfo *pd_uinfo;
+ unsigned long flags;
+
+ pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo +
+ sizeof(struct pd_uinfo) * idx);
+ spin_lock_irqsave(&dev->core_dev->lock, flags);
+ if (dev->pdr_tail != PPC4XX_LAST_PD)
+ dev->pdr_tail++;
+ else
+ dev->pdr_tail = 0;
+ pd_uinfo->state = PD_ENTRY_FREE;
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+ return 0;
+}
+
+static struct ce_pd *crypto4xx_get_pdp(struct crypto4xx_device *dev,
+ dma_addr_t *pd_dma, u32 idx)
+{
+ *pd_dma = dev->pdr_pa + sizeof(struct ce_pd) * idx;
+
+ return dev->pdr + sizeof(struct ce_pd) * idx;
+}
+
+/**
+ * alloc memory for the gather ring
+ * no need to alloc buf for the ring
+ * gdr_tail, gdr_head and gdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_gdr(struct crypto4xx_device *dev)
+{
+ dev->gdr = dma_alloc_coherent(dev->core_dev->device,
+ sizeof(struct ce_gd) * PPC4XX_NUM_GD,
+ &dev->gdr_pa, GFP_ATOMIC);
+ if (!dev->gdr)
+ return -ENOMEM;
+
+ memset(dev->gdr, 0, sizeof(struct ce_gd) * PPC4XX_NUM_GD);
+
+ return 0;
+}
+
+static inline void crypto4xx_destroy_gdr(struct crypto4xx_device *dev)
+{
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct ce_gd) * PPC4XX_NUM_GD,
+ dev->gdr, dev->gdr_pa);
+}
+
+/*
+ * when this function is called.
+ * preemption or interrupt must be disabled
+ */
+u32 crypto4xx_get_n_gd(struct crypto4xx_device *dev, int n)
+{
+ u32 retval;
+ u32 tmp;
+ if (n >= PPC4XX_NUM_GD)
+ return ERING_WAS_FULL;
+
+ retval = dev->gdr_head;
+ tmp = (dev->gdr_head + n) % PPC4XX_NUM_GD;
+ if (dev->gdr_head > dev->gdr_tail) {
+ if (tmp < dev->gdr_head && tmp >= dev->gdr_tail)
+ return ERING_WAS_FULL;
+ } else if (dev->gdr_head < dev->gdr_tail) {
+ if (tmp < dev->gdr_head || tmp >= dev->gdr_tail)
+ return ERING_WAS_FULL;
+ }
+ dev->gdr_head = tmp;
+
+ return retval;
+}
+
+static u32 crypto4xx_put_gd_to_gdr(struct crypto4xx_device *dev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->core_dev->lock, flags);
+ if (dev->gdr_tail == dev->gdr_head) {
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+ return 0;
+ }
+
+ if (dev->gdr_tail != PPC4XX_LAST_GD)
+ dev->gdr_tail++;
+ else
+ dev->gdr_tail = 0;
+
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+ return 0;
+}
+
+static inline struct ce_gd *crypto4xx_get_gdp(struct crypto4xx_device *dev,
+ dma_addr_t *gd_dma, u32 idx)
+{
+ *gd_dma = dev->gdr_pa + sizeof(struct ce_gd) * idx;
+
+ return (struct ce_gd *) (dev->gdr + sizeof(struct ce_gd) * idx);
+}
+
+/**
+ * alloc memory for the scatter ring
+ * need to alloc buf for the ring
+ * sdr_tail, sdr_head and sdr_count are initialized by this function
+ */
+static u32 crypto4xx_build_sdr(struct crypto4xx_device *dev)
+{
+ int i;
+ struct ce_sd *sd_array;
+
+ /* alloc memory for scatter descriptor ring */
+ dev->sdr = dma_alloc_coherent(dev->core_dev->device,
+ sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+ &dev->sdr_pa, GFP_ATOMIC);
+ if (!dev->sdr)
+ return -ENOMEM;
+
+ dev->scatter_buffer_size = PPC4XX_SD_BUFFER_SIZE;
+ dev->scatter_buffer_va =
+ dma_alloc_coherent(dev->core_dev->device,
+ dev->scatter_buffer_size * PPC4XX_NUM_SD,
+ &dev->scatter_buffer_pa, GFP_ATOMIC);
+ if (!dev->scatter_buffer_va) {
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+ dev->sdr, dev->sdr_pa);
+ return -ENOMEM;
+ }
+
+ sd_array = dev->sdr;
+
+ for (i = 0; i < PPC4XX_NUM_SD; i++) {
+ sd_array[i].ptr = dev->scatter_buffer_pa +
+ dev->scatter_buffer_size * i;
+ }
+
+ return 0;
+}
+
+static void crypto4xx_destroy_sdr(struct crypto4xx_device *dev)
+{
+ if (dev->sdr != NULL)
+ dma_free_coherent(dev->core_dev->device,
+ sizeof(struct ce_sd) * PPC4XX_NUM_SD,
+ dev->sdr, dev->sdr_pa);
+
+ if (dev->scatter_buffer_va != NULL)
+ dma_free_coherent(dev->core_dev->device,
+ dev->scatter_buffer_size * PPC4XX_NUM_SD,
+ dev->scatter_buffer_va,
+ dev->scatter_buffer_pa);
+}
+
+/*
+ * when this function is called.
+ * preemption or interrupt must be disabled
+ */
+static u32 crypto4xx_get_n_sd(struct crypto4xx_device *dev, int n)
+{
+ u32 retval;
+ u32 tmp;
+
+ if (n >= PPC4XX_NUM_SD)
+ return ERING_WAS_FULL;
+
+ retval = dev->sdr_head;
+ tmp = (dev->sdr_head + n) % PPC4XX_NUM_SD;
+ if (dev->sdr_head > dev->gdr_tail) {
+ if (tmp < dev->sdr_head && tmp >= dev->sdr_tail)
+ return ERING_WAS_FULL;
+ } else if (dev->sdr_head < dev->sdr_tail) {
+ if (tmp < dev->sdr_head || tmp >= dev->sdr_tail)
+ return ERING_WAS_FULL;
+ } /* the head = tail, or empty case is already take cared */
+ dev->sdr_head = tmp;
+
+ return retval;
+}
+
+static u32 crypto4xx_put_sd_to_sdr(struct crypto4xx_device *dev)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->core_dev->lock, flags);
+ if (dev->sdr_tail == dev->sdr_head) {
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+ return 0;
+ }
+ if (dev->sdr_tail != PPC4XX_LAST_SD)
+ dev->sdr_tail++;
+ else
+ dev->sdr_tail = 0;
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+ return 0;
+}
+
+static inline struct ce_sd *crypto4xx_get_sdp(struct crypto4xx_device *dev,
+ dma_addr_t *sd_dma, u32 idx)
+{
+ *sd_dma = dev->sdr_pa + sizeof(struct ce_sd) * idx;
+
+ return (struct ce_sd *)(dev->sdr + sizeof(struct ce_sd) * idx);
+}
+
+static u32 crypto4xx_fill_one_page(struct crypto4xx_device *dev,
+ dma_addr_t *addr, u32 *length,
+ u32 *idx, u32 *offset, u32 *nbytes)
+{
+ u32 len;
+
+ if (*length > dev->scatter_buffer_size) {
+ memcpy(phys_to_virt(*addr),
+ dev->scatter_buffer_va +
+ *idx * dev->scatter_buffer_size + *offset,
+ dev->scatter_buffer_size);
+ *offset = 0;
+ *length -= dev->scatter_buffer_size;
+ *nbytes -= dev->scatter_buffer_size;
+ if (*idx == PPC4XX_LAST_SD)
+ *idx = 0;
+ else
+ (*idx)++;
+ *addr = *addr + dev->scatter_buffer_size;
+ return 1;
+ } else if (*length < dev->scatter_buffer_size) {
+ memcpy(phys_to_virt(*addr),
+ dev->scatter_buffer_va +
+ *idx * dev->scatter_buffer_size + *offset, *length);
+ if ((*offset + *length) == dev->scatter_buffer_size) {
+ if (*idx == PPC4XX_LAST_SD)
+ *idx = 0;
+ else
+ (*idx)++;
+ *nbytes -= *length;
+ *offset = 0;
+ } else {
+ *nbytes -= *length;
+ *offset += *length;
+ }
+
+ return 0;
+ } else {
+ len = (*nbytes <= dev->scatter_buffer_size) ?
+ (*nbytes) : dev->scatter_buffer_size;
+ memcpy(phys_to_virt(*addr),
+ dev->scatter_buffer_va +
+ *idx * dev->scatter_buffer_size + *offset,
+ len);
+ *offset = 0;
+ *nbytes -= len;
+
+ if (*idx == PPC4XX_LAST_SD)
+ *idx = 0;
+ else
+ (*idx)++;
+
+ return 0;
+ }
+}
+
+static void crypto4xx_copy_pkt_to_dst(struct crypto4xx_device *dev,
+ struct ce_pd *pd,
+ struct pd_uinfo *pd_uinfo,
+ u32 nbytes,
+ struct scatterlist *dst)
+{
+ dma_addr_t addr;
+ u32 this_sd;
+ u32 offset;
+ u32 len;
+ u32 i;
+ u32 sg_len;
+ struct scatterlist *sg;
+
+ this_sd = pd_uinfo->first_sd;
+ offset = 0;
+ i = 0;
+
+ while (nbytes) {
+ sg = &dst[i];
+ sg_len = sg->length;
+ addr = dma_map_page(dev->core_dev->device, sg_page(sg),
+ sg->offset, sg->length, DMA_TO_DEVICE);
+
+ if (offset == 0) {
+ len = (nbytes <= sg->length) ? nbytes : sg->length;
+ while (crypto4xx_fill_one_page(dev, &addr, &len,
+ &this_sd, &offset, &nbytes))
+ ;
+ if (!nbytes)
+ return;
+ i++;
+ } else {
+ len = (nbytes <= (dev->scatter_buffer_size - offset)) ?
+ nbytes : (dev->scatter_buffer_size - offset);
+ len = (sg->length < len) ? sg->length : len;
+ while (crypto4xx_fill_one_page(dev, &addr, &len,
+ &this_sd, &offset, &nbytes))
+ ;
+ if (!nbytes)
+ return;
+ sg_len -= len;
+ if (sg_len) {
+ addr += len;
+ while (crypto4xx_fill_one_page(dev, &addr,
+ &sg_len, &this_sd, &offset, &nbytes))
+ ;
+ }
+ i++;
+ }
+ }
+}
+
+static u32 crypto4xx_copy_digest_to_dst(struct pd_uinfo *pd_uinfo,
+ struct crypto4xx_ctx *ctx)
+{
+ struct dynamic_sa_ctl *sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+ struct sa_state_record *state_record =
+ (struct sa_state_record *) pd_uinfo->sr_va;
+
+ if (sa->sa_command_0.bf.hash_alg == SA_HASH_ALG_SHA1) {
+ memcpy((void *) pd_uinfo->dest_va, state_record->save_digest,
+ SA_HASH_ALG_SHA1_DIGEST_SIZE);
+ }
+
+ return 0;
+}
+
+static void crypto4xx_ret_sg_desc(struct crypto4xx_device *dev,
+ struct pd_uinfo *pd_uinfo)
+{
+ int i;
+ if (pd_uinfo->num_gd) {
+ for (i = 0; i < pd_uinfo->num_gd; i++)
+ crypto4xx_put_gd_to_gdr(dev);
+ pd_uinfo->first_gd = 0xffffffff;
+ pd_uinfo->num_gd = 0;
+ }
+ if (pd_uinfo->num_sd) {
+ for (i = 0; i < pd_uinfo->num_sd; i++)
+ crypto4xx_put_sd_to_sdr(dev);
+
+ pd_uinfo->first_sd = 0xffffffff;
+ pd_uinfo->num_sd = 0;
+ }
+}
+
+static u32 crypto4xx_ablkcipher_done(struct crypto4xx_device *dev,
+ struct pd_uinfo *pd_uinfo,
+ struct ce_pd *pd)
+{
+ struct crypto4xx_ctx *ctx;
+ struct ablkcipher_request *ablk_req;
+ struct scatterlist *dst;
+ dma_addr_t addr;
+
+ ablk_req = ablkcipher_request_cast(pd_uinfo->async_req);
+ ctx = crypto_tfm_ctx(ablk_req->base.tfm);
+
+ if (pd_uinfo->using_sd) {
+ crypto4xx_copy_pkt_to_dst(dev, pd, pd_uinfo, ablk_req->nbytes,
+ ablk_req->dst);
+ } else {
+ dst = pd_uinfo->dest_va;
+ addr = dma_map_page(dev->core_dev->device, sg_page(dst),
+ dst->offset, dst->length, DMA_FROM_DEVICE);
+ }
+ crypto4xx_ret_sg_desc(dev, pd_uinfo);
+ if (ablk_req->base.complete != NULL)
+ ablk_req->base.complete(&ablk_req->base, 0);
+
+ return 0;
+}
+
+static u32 crypto4xx_ahash_done(struct crypto4xx_device *dev,
+ struct pd_uinfo *pd_uinfo)
+{
+ struct crypto4xx_ctx *ctx;
+ struct ahash_request *ahash_req;
+
+ ahash_req = ahash_request_cast(pd_uinfo->async_req);
+ ctx = crypto_tfm_ctx(ahash_req->base.tfm);
+
+ crypto4xx_copy_digest_to_dst(pd_uinfo,
+ crypto_tfm_ctx(ahash_req->base.tfm));
+ crypto4xx_ret_sg_desc(dev, pd_uinfo);
+ /* call user provided callback function x */
+ if (ahash_req->base.complete != NULL)
+ ahash_req->base.complete(&ahash_req->base, 0);
+
+ return 0;
+}
+
+static u32 crypto4xx_pd_done(struct crypto4xx_device *dev, u32 idx)
+{
+ struct ce_pd *pd;
+ struct pd_uinfo *pd_uinfo;
+
+ pd = dev->pdr + sizeof(struct ce_pd)*idx;
+ pd_uinfo = dev->pdr_uinfo + sizeof(struct pd_uinfo)*idx;
+ if (crypto_tfm_alg_type(pd_uinfo->async_req->tfm) ==
+ CRYPTO_ALG_TYPE_ABLKCIPHER)
+ return crypto4xx_ablkcipher_done(dev, pd_uinfo, pd);
+ else
+ return crypto4xx_ahash_done(dev, pd_uinfo);
+}
+
+/**
+ * Note: Only use this function to copy items that is word aligned.
+ */
+void crypto4xx_memcpy_le(unsigned int *dst,
+ const unsigned char *buf,
+ int len)
+{
+ u8 *tmp;
+ for (; len >= 4; buf += 4, len -= 4)
+ *dst++ = cpu_to_le32(*(unsigned int *) buf);
+
+ tmp = (u8 *)dst;
+ switch (len) {
+ case 3:
+ *tmp++ = 0;
+ *tmp++ = *(buf+2);
+ *tmp++ = *(buf+1);
+ *tmp++ = *buf;
+ break;
+ case 2:
+ *tmp++ = 0;
+ *tmp++ = 0;
+ *tmp++ = *(buf+1);
+ *tmp++ = *buf;
+ break;
+ case 1:
+ *tmp++ = 0;
+ *tmp++ = 0;
+ *tmp++ = 0;
+ *tmp++ = *buf;
+ break;
+ default:
+ break;
+ }
+}
+
+static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev)
+{
+ crypto4xx_destroy_pdr(core_dev->dev);
+ crypto4xx_destroy_gdr(core_dev->dev);
+ crypto4xx_destroy_sdr(core_dev->dev);
+ dev_set_drvdata(core_dev->device, NULL);
+ iounmap(core_dev->dev->ce_base);
+ kfree(core_dev->dev);
+ kfree(core_dev);
+}
+
+void crypto4xx_return_pd(struct crypto4xx_device *dev,
+ u32 pd_entry, struct ce_pd *pd,
+ struct pd_uinfo *pd_uinfo)
+{
+ /* irq should be already disabled */
+ dev->pdr_head = pd_entry;
+ pd->pd_ctl.w = 0;
+ pd->pd_ctl_len.w = 0;
+ pd_uinfo->state = PD_ENTRY_FREE;
+}
+
+/*
+ * derive number of elements in scatterlist
+ * Shamlessly copy from talitos.c
+ */
+static int get_sg_count(struct scatterlist *sg_list, int nbytes)
+{
+ struct scatterlist *sg = sg_list;
+ int sg_nents = 0;
+
+ while (nbytes) {
+ sg_nents++;
+ if (sg->length > nbytes)
+ break;
+ nbytes -= sg->length;
+ sg = sg_next(sg);
+ }
+
+ return sg_nents;
+}
+
+static u32 get_next_gd(u32 current)
+{
+ if (current != PPC4XX_LAST_GD)
+ return current + 1;
+ else
+ return 0;
+}
+
+static u32 get_next_sd(u32 current)
+{
+ if (current != PPC4XX_LAST_SD)
+ return current + 1;
+ else
+ return 0;
+}
+
+u32 crypto4xx_build_pd(struct crypto_async_request *req,
+ struct crypto4xx_ctx *ctx,
+ struct scatterlist *src,
+ struct scatterlist *dst,
+ unsigned int datalen,
+ void *iv, u32 iv_len)
+{
+ struct crypto4xx_device *dev = ctx->dev;
+ dma_addr_t addr, pd_dma, sd_dma, gd_dma;
+ struct dynamic_sa_ctl *sa;
+ struct scatterlist *sg;
+ struct ce_gd *gd;
+ struct ce_pd *pd;
+ u32 num_gd, num_sd;
+ u32 fst_gd = 0xffffffff;
+ u32 fst_sd = 0xffffffff;
+ u32 pd_entry;
+ unsigned long flags;
+ struct pd_uinfo *pd_uinfo = NULL;
+ unsigned int nbytes = datalen, idx;
+ unsigned int ivlen = 0;
+ u32 gd_idx = 0;
+
+ /* figure how many gd is needed */
+ num_gd = get_sg_count(src, datalen);
+ if (num_gd == 1)
+ num_gd = 0;
+
+ /* figure how many sd is needed */
+ if (sg_is_last(dst) || ctx->is_hash) {
+ num_sd = 0;
+ } else {
+ if (datalen > PPC4XX_SD_BUFFER_SIZE) {
+ num_sd = datalen / PPC4XX_SD_BUFFER_SIZE;
+ if (datalen % PPC4XX_SD_BUFFER_SIZE)
+ num_sd++;
+ } else {
+ num_sd = 1;
+ }
+ }
+
+ /*
+ * The follow section of code needs to be protected
+ * The gather ring and scatter ring needs to be consecutive
+ * In case of run out of any kind of descriptor, the descriptor
+ * already got must be return the original place.
+ */
+ spin_lock_irqsave(&dev->core_dev->lock, flags);
+ if (num_gd) {
+ fst_gd = crypto4xx_get_n_gd(dev, num_gd);
+ if (fst_gd == ERING_WAS_FULL) {
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+ return -EAGAIN;
+ }
+ }
+ if (num_sd) {
+ fst_sd = crypto4xx_get_n_sd(dev, num_sd);
+ if (fst_sd == ERING_WAS_FULL) {
+ if (num_gd)
+ dev->gdr_head = fst_gd;
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+ return -EAGAIN;
+ }
+ }
+ pd_entry = crypto4xx_get_pd_from_pdr_nolock(dev);
+ if (pd_entry == ERING_WAS_FULL) {
+ if (num_gd)
+ dev->gdr_head = fst_gd;
+ if (num_sd)
+ dev->sdr_head = fst_sd;
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+ return -EAGAIN;
+ }
+ spin_unlock_irqrestore(&dev->core_dev->lock, flags);
+
+ pd_uinfo = (struct pd_uinfo *)(dev->pdr_uinfo +
+ sizeof(struct pd_uinfo) * pd_entry);
+ pd = crypto4xx_get_pdp(dev, &pd_dma, pd_entry);
+ pd_uinfo->async_req = req;
+ pd_uinfo->num_gd = num_gd;
+ pd_uinfo->num_sd = num_sd;
+
+ if (iv_len || ctx->is_hash) {
+ ivlen = iv_len;
+ pd->sa = pd_uinfo->sa_pa;
+ sa = (struct dynamic_sa_ctl *) pd_uinfo->sa_va;
+ if (ctx->direction == DIR_INBOUND)
+ memcpy(sa, ctx->sa_in, ctx->sa_len * 4);
+ else
+ memcpy(sa, ctx->sa_out, ctx->sa_len * 4);
+
+ memcpy((void *) sa + ctx->offset_to_sr_ptr,
+ &pd_uinfo->sr_pa, 4);
+
+ if (iv_len)
+ crypto4xx_memcpy_le(pd_uinfo->sr_va, iv, iv_len);
+ } else {
+ if (ctx->direction == DIR_INBOUND) {
+ pd->sa = ctx->sa_in_dma_addr;
+ sa = (struct dynamic_sa_ctl *) ctx->sa_in;
+ } else {
+ pd->sa = ctx->sa_out_dma_addr;
+ sa = (struct dynamic_sa_ctl *) ctx->sa_out;
+ }
+ }
+ pd->sa_len = ctx->sa_len;
+ if (num_gd) {
+ /* get first gd we are going to use */
+ gd_idx = fst_gd;
+ pd_uinfo->first_gd = fst_gd;
+ pd_uinfo->num_gd = num_gd;
+ gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx);
+ pd->src = gd_dma;
+ /* enable gather */
+ sa->sa_command_0.bf.gather = 1;
+ idx = 0;
+ src = &src[0];
+ /* walk the sg, and setup gather array */
+ while (nbytes) {
+ sg = &src[idx];
+ addr = dma_map_page(dev->core_dev->device, sg_page(sg),
+ sg->offset, sg->length, DMA_TO_DEVICE);
+ gd->ptr = addr;
+ gd->ctl_len.len = sg->length;
+ gd->ctl_len.done = 0;
+ gd->ctl_len.ready = 1;
+ if (sg->length >= nbytes)
+ break;
+ nbytes -= sg->length;
+ gd_idx = get_next_gd(gd_idx);
+ gd = crypto4xx_get_gdp(dev, &gd_dma, gd_idx);
+ idx++;
+ }
+ } else {
+ pd->src = (u32)dma_map_page(dev->core_dev->device, sg_page(src),
+ src->offset, src->length, DMA_TO_DEVICE);
+ /*
+ * Disable gather in sa command
+ */
+ sa->sa_command_0.bf.gather = 0;
+ /*
+ * Indicate gather array is not used
+ */
+ pd_uinfo->first_gd = 0xffffffff;
+ pd_uinfo->num_gd = 0;
+ }
+ if (ctx->is_hash || sg_is_last(dst)) {
+ /*
+ * we know application give us dst a whole piece of memory
+ * no need to use scatter ring.
+ * In case of is_hash, the icv is always at end of src data.
+ */
+ pd_uinfo->using_sd = 0;
+ pd_uinfo->first_sd = 0xffffffff;
+ pd_uinfo->num_sd = 0;
+ pd_uinfo->dest_va = dst;
+ sa->sa_command_0.bf.scatter = 0;
+ if (ctx->is_hash)
+ pd->dest = virt_to_phys((void *)dst);
+ else
+ pd->dest = (u32)dma_map_page(dev->core_dev->device,
+ sg_page(dst), dst->offset,
+ dst->length, DMA_TO_DEVICE);
+ } else {
+ struct ce_sd *sd = NULL;
+ u32 sd_idx = fst_sd;
+ nbytes = datalen;
+ sa->sa_command_0.bf.scatter = 1;
+ pd_uinfo->using_sd = 1;
+ pd_uinfo->dest_va = dst;
+ pd_uinfo->first_sd = fst_sd;
+ pd_uinfo->num_sd = num_sd;
+ sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx);
+ pd->dest = sd_dma;
+ /* setup scatter descriptor */
+ sd->ctl.done = 0;
+ sd->ctl.rdy = 1;
+ /* sd->ptr should be setup by sd_init routine*/
+ idx = 0;
+ if (nbytes >= PPC4XX_SD_BUFFER_SIZE)
+ nbytes -= PPC4XX_SD_BUFFER_SIZE;
+ else
+ nbytes = 0;
+ while (nbytes) {
+ sd_idx = get_next_sd(sd_idx);
+ sd = crypto4xx_get_sdp(dev, &sd_dma, sd_idx);
+ /* setup scatter descriptor */
+ sd->ctl.done = 0;
+ sd->ctl.rdy = 1;
+ if (nbytes >= PPC4XX_SD_BUFFER_SIZE)
+ nbytes -= PPC4XX_SD_BUFFER_SIZE;
+ else
+ /*
+ * SD entry can hold PPC4XX_SD_BUFFER_SIZE,
+ * which is more than nbytes, so done.
+ */
+ nbytes = 0;
+ }
+ }
+
+ sa->sa_command_1.bf.hash_crypto_offset = 0;
+ pd->pd_ctl.w = ctx->pd_ctl;
+ pd->pd_ctl_len.w = 0x00400000 | (ctx->bypass << 24) | datalen;
+ pd_uinfo->state = PD_ENTRY_INUSE;
+ wmb();
+ /* write any value to push engine to read a pd */
+ writel(1, dev->ce_base + CRYPTO4XX_INT_DESCR_RD);
+ return -EINPROGRESS;
+}
+
+/**
+ * Algorithm Registration Functions
+ */
+static int crypto4xx_alg_init(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct crypto4xx_alg *amcc_alg = crypto_alg_to_crypto4xx_alg(alg);
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->dev = amcc_alg->dev;
+ ctx->sa_in = NULL;
+ ctx->sa_out = NULL;
+ ctx->sa_in_dma_addr = 0;
+ ctx->sa_out_dma_addr = 0;
+ ctx->sa_len = 0;
+
+ switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
+ default:
+ tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct crypto4xx_ctx));
+ break;
+ }
+
+ return 0;
+}
+
+static void crypto4xx_alg_exit(struct crypto_tfm *tfm)
+{
+ struct crypto4xx_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto4xx_free_sa(ctx);
+ crypto4xx_free_state_record(ctx);
+}
+
+int crypto4xx_register_alg(struct crypto4xx_device *sec_dev,
+ struct crypto4xx_alg_common *crypto_alg,
+ int array_size)
+{
+ struct crypto4xx_alg *alg;
+ int i;
+ int rc = 0;
+
+ for (i = 0; i < array_size; i++) {
+ alg = kzalloc(sizeof(struct crypto4xx_alg), GFP_KERNEL);
+ if (!alg)
+ return -ENOMEM;
+
+ alg->alg = crypto_alg[i];
+ alg->dev = sec_dev;
+
+ switch (alg->alg.type) {
+ case CRYPTO_ALG_TYPE_AHASH:
+ rc = crypto_register_ahash(&alg->alg.u.hash);
+ break;
+
+ default:
+ rc = crypto_register_alg(&alg->alg.u.cipher);
+ break;
+ }
+
+ if (rc) {
+ list_del(&alg->entry);
+ kfree(alg);
+ } else {
+ list_add_tail(&alg->entry, &sec_dev->alg_list);
+ }
+ }
+
+ return 0;
+}
+
+static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev)
+{
+ struct crypto4xx_alg *alg, *tmp;
+
+ list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) {
+ list_del(&alg->entry);
+ switch (alg->alg.type) {
+ case CRYPTO_ALG_TYPE_AHASH:
+ crypto_unregister_ahash(&alg->alg.u.hash);
+ break;
+
+ default:
+ crypto_unregister_alg(&alg->alg.u.cipher);
+ }
+ kfree(alg);
+ }
+}
+
+static void crypto4xx_bh_tasklet_cb(unsigned long data)
+{
+ struct device *dev = (struct device *)data;
+ struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
+ struct pd_uinfo *pd_uinfo;
+ struct ce_pd *pd;
+ u32 tail;
+
+ while (core_dev->dev->pdr_head != core_dev->dev->pdr_tail) {
+ tail = core_dev->dev->pdr_tail;
+ pd_uinfo = core_dev->dev->pdr_uinfo +
+ sizeof(struct pd_uinfo)*tail;
+ pd = core_dev->dev->pdr + sizeof(struct ce_pd) * tail;
+ if ((pd_uinfo->state == PD_ENTRY_INUSE) &&
+ pd->pd_ctl.bf.pe_done &&
+ !pd->pd_ctl.bf.host_ready) {
+ pd->pd_ctl.bf.pe_done = 0;
+ crypto4xx_pd_done(core_dev->dev, tail);
+ crypto4xx_put_pd_to_pdr(core_dev->dev, tail);
+ pd_uinfo->state = PD_ENTRY_FREE;
+ } else {
+ /* if tail not done, break */
+ break;
+ }
+ }
+}
+
+/**
+ * Top Half of isr.
+ */
+static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data)
+{
+ struct device *dev = (struct device *)data;
+ struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
+
+ if (core_dev->dev->ce_base == 0)
+ return 0;
+
+ writel(PPC4XX_INTERRUPT_CLR,
+ core_dev->dev->ce_base + CRYPTO4XX_INT_CLR);
+ tasklet_schedule(&core_dev->tasklet);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * Supported Crypto Algorithms
+ */
+struct crypto4xx_alg_common crypto4xx_alg[] = {
+ /* Crypto AES modes */
+ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-ppc4xx",
+ .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct crypto4xx_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_init = crypto4xx_alg_init,
+ .cra_exit = crypto4xx_alg_exit,
+ .cra_module = THIS_MODULE,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_IV_SIZE,
+ .setkey = crypto4xx_setkey_aes_cbc,
+ .encrypt = crypto4xx_encrypt,
+ .decrypt = crypto4xx_decrypt,
+ }
+ }
+ }},
+};
+
+/**
+ * Module Initialization Routine
+ */
+static int __init crypto4xx_probe(struct platform_device *ofdev)
+{
+ int rc;
+ struct resource res;
+ struct device *dev = &ofdev->dev;
+ struct crypto4xx_core_device *core_dev;
+
+ rc = of_address_to_resource(ofdev->dev.of_node, 0, &res);
+ if (rc)
+ return -ENODEV;
+
+ if (of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto")) {
+ mtdcri(SDR0, PPC460EX_SDR0_SRST,
+ mfdcri(SDR0, PPC460EX_SDR0_SRST) | PPC460EX_CE_RESET);
+ mtdcri(SDR0, PPC460EX_SDR0_SRST,
+ mfdcri(SDR0, PPC460EX_SDR0_SRST) & ~PPC460EX_CE_RESET);
+ } else if (of_find_compatible_node(NULL, NULL,
+ "amcc,ppc405ex-crypto")) {
+ mtdcri(SDR0, PPC405EX_SDR0_SRST,
+ mfdcri(SDR0, PPC405EX_SDR0_SRST) | PPC405EX_CE_RESET);
+ mtdcri(SDR0, PPC405EX_SDR0_SRST,
+ mfdcri(SDR0, PPC405EX_SDR0_SRST) & ~PPC405EX_CE_RESET);
+ } else if (of_find_compatible_node(NULL, NULL,
+ "amcc,ppc460sx-crypto")) {
+ mtdcri(SDR0, PPC460SX_SDR0_SRST,
+ mfdcri(SDR0, PPC460SX_SDR0_SRST) | PPC460SX_CE_RESET);
+ mtdcri(SDR0, PPC460SX_SDR0_SRST,
+ mfdcri(SDR0, PPC460SX_SDR0_SRST) & ~PPC460SX_CE_RESET);
+ } else {
+ printk(KERN_ERR "Crypto Function Not supported!\n");
+ return -EINVAL;
+ }
+
+ core_dev = kzalloc(sizeof(struct crypto4xx_core_device), GFP_KERNEL);
+ if (!core_dev)
+ return -ENOMEM;
+
+ dev_set_drvdata(dev, core_dev);
+ core_dev->ofdev = ofdev;
+ core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL);
+ if (!core_dev->dev)
+ goto err_alloc_dev;
+
+ core_dev->dev->core_dev = core_dev;
+ core_dev->device = dev;
+ spin_lock_init(&core_dev->lock);
+ INIT_LIST_HEAD(&core_dev->dev->alg_list);
+ rc = crypto4xx_build_pdr(core_dev->dev);
+ if (rc)
+ goto err_build_pdr;
+
+ rc = crypto4xx_build_gdr(core_dev->dev);
+ if (rc)
+ goto err_build_gdr;
+
+ rc = crypto4xx_build_sdr(core_dev->dev);
+ if (rc)
+ goto err_build_sdr;
+
+ /* Init tasklet for bottom half processing */
+ tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb,
+ (unsigned long) dev);
+
+ /* Register for Crypto isr, Crypto Engine IRQ */
+ core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0);
+ rc = request_irq(core_dev->irq, crypto4xx_ce_interrupt_handler, 0,
+ core_dev->dev->name, dev);
+ if (rc)
+ goto err_request_irq;
+
+ core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0);
+ if (!core_dev->dev->ce_base) {
+ dev_err(dev, "failed to of_iomap\n");
+ goto err_iomap;
+ }
+
+ /* need to setup pdr, rdr, gdr and sdr before this */
+ crypto4xx_hw_init(core_dev->dev);
+
+ /* Register security algorithms with Linux CryptoAPI */
+ rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg,
+ ARRAY_SIZE(crypto4xx_alg));
+ if (rc)
+ goto err_start_dev;
+
+ return 0;
+
+err_start_dev:
+ iounmap(core_dev->dev->ce_base);
+err_iomap:
+ free_irq(core_dev->irq, dev);
+ irq_dispose_mapping(core_dev->irq);
+ tasklet_kill(&core_dev->tasklet);
+err_request_irq:
+ crypto4xx_destroy_sdr(core_dev->dev);
+err_build_sdr:
+ crypto4xx_destroy_gdr(core_dev->dev);
+err_build_gdr:
+ crypto4xx_destroy_pdr(core_dev->dev);
+err_build_pdr:
+ kfree(core_dev->dev);
+err_alloc_dev:
+ kfree(core_dev);
+
+ return rc;
+}
+
+static int __exit crypto4xx_remove(struct platform_device *ofdev)
+{
+ struct device *dev = &ofdev->dev;
+ struct crypto4xx_core_device *core_dev = dev_get_drvdata(dev);
+
+ free_irq(core_dev->irq, dev);
+ irq_dispose_mapping(core_dev->irq);
+
+ tasklet_kill(&core_dev->tasklet);
+ /* Un-register with Linux CryptoAPI */
+ crypto4xx_unregister_alg(core_dev->dev);
+ /* Free all allocated memory */
+ crypto4xx_stop_all(core_dev);
+
+ return 0;
+}
+
+static const struct of_device_id crypto4xx_match[] = {
+ { .compatible = "amcc,ppc4xx-crypto",},
+ { },
+};
+
+static struct platform_driver crypto4xx_driver = {
+ .driver = {
+ .name = "crypto4xx",
+ .owner = THIS_MODULE,
+ .of_match_table = crypto4xx_match,
+ },
+ .probe = crypto4xx_probe,
+ .remove = crypto4xx_remove,
+};
+
+module_platform_driver(crypto4xx_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("James Hsiao <jhsiao@amcc.com>");
+MODULE_DESCRIPTION("Driver for AMCC PPC4xx crypto accelerator");
+
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.h b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.h
new file mode 100644
index 0000000..bac0bde
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_core.h
@@ -0,0 +1,196 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * This is the header file for AMCC Crypto offload Linux device driver for
+ * use with Linux CryptoAPI.
+
+ */
+
+#ifndef __CRYPTO4XX_CORE_H__
+#define __CRYPTO4XX_CORE_H__
+
+#include <crypto/internal/hash.h>
+
+#define PPC460SX_SDR0_SRST 0x201
+#define PPC405EX_SDR0_SRST 0x200
+#define PPC460EX_SDR0_SRST 0x201
+#define PPC460EX_CE_RESET 0x08000000
+#define PPC460SX_CE_RESET 0x20000000
+#define PPC405EX_CE_RESET 0x00000008
+
+#define CRYPTO4XX_CRYPTO_PRIORITY 300
+#define PPC4XX_LAST_PD 63
+#define PPC4XX_NUM_PD 64
+#define PPC4XX_LAST_GD 1023
+#define PPC4XX_NUM_GD 1024
+#define PPC4XX_LAST_SD 63
+#define PPC4XX_NUM_SD 64
+#define PPC4XX_SD_BUFFER_SIZE 2048
+
+#define PD_ENTRY_INUSE 1
+#define PD_ENTRY_FREE 0
+#define ERING_WAS_FULL 0xffffffff
+
+struct crypto4xx_device;
+
+struct pd_uinfo {
+ struct crypto4xx_device *dev;
+ u32 state;
+ u32 using_sd;
+ u32 first_gd; /* first gather discriptor
+ used by this packet */
+ u32 num_gd; /* number of gather discriptor
+ used by this packet */
+ u32 first_sd; /* first scatter discriptor
+ used by this packet */
+ u32 num_sd; /* number of scatter discriptors
+ used by this packet */
+ void *sa_va; /* shadow sa, when using cp from ctx->sa */
+ u32 sa_pa;
+ void *sr_va; /* state record for shadow sa */
+ u32 sr_pa;
+ struct scatterlist *dest_va;
+ struct crypto_async_request *async_req; /* base crypto request
+ for this packet */
+};
+
+struct crypto4xx_device {
+ struct crypto4xx_core_device *core_dev;
+ char *name;
+ u64 ce_phy_address;
+ void __iomem *ce_base;
+
+ void *pdr; /* base address of packet
+ descriptor ring */
+ dma_addr_t pdr_pa; /* physical address used to
+ program ce pdr_base_register */
+ void *gdr; /* gather descriptor ring */
+ dma_addr_t gdr_pa; /* physical address used to
+ program ce gdr_base_register */
+ void *sdr; /* scatter descriptor ring */
+ dma_addr_t sdr_pa; /* physical address used to
+ program ce sdr_base_register */
+ void *scatter_buffer_va;
+ dma_addr_t scatter_buffer_pa;
+ u32 scatter_buffer_size;
+
+ void *shadow_sa_pool; /* pool of memory for sa in pd_uinfo */
+ dma_addr_t shadow_sa_pool_pa;
+ void *shadow_sr_pool; /* pool of memory for sr in pd_uinfo */
+ dma_addr_t shadow_sr_pool_pa;
+ u32 pdr_tail;
+ u32 pdr_head;
+ u32 gdr_tail;
+ u32 gdr_head;
+ u32 sdr_tail;
+ u32 sdr_head;
+ void *pdr_uinfo;
+ struct list_head alg_list; /* List of algorithm supported
+ by this device */
+};
+
+struct crypto4xx_core_device {
+ struct device *device;
+ struct platform_device *ofdev;
+ struct crypto4xx_device *dev;
+ u32 int_status;
+ u32 irq;
+ struct tasklet_struct tasklet;
+ spinlock_t lock;
+};
+
+struct crypto4xx_ctx {
+ struct crypto4xx_device *dev;
+ void *sa_in;
+ dma_addr_t sa_in_dma_addr;
+ void *sa_out;
+ dma_addr_t sa_out_dma_addr;
+ void *state_record;
+ dma_addr_t state_record_dma_addr;
+ u32 sa_len;
+ u32 offset_to_sr_ptr; /* offset to state ptr, in dynamic sa */
+ u32 direction;
+ u32 next_hdr;
+ u32 save_iv;
+ u32 pd_ctl_len;
+ u32 pd_ctl;
+ u32 bypass;
+ u32 is_hash;
+ u32 hash_final;
+};
+
+struct crypto4xx_req_ctx {
+ struct crypto4xx_device *dev; /* Device in which
+ operation to send to */
+ void *sa;
+ u32 sa_dma_addr;
+ u16 sa_len;
+};
+
+struct crypto4xx_alg_common {
+ u32 type;
+ union {
+ struct crypto_alg cipher;
+ struct ahash_alg hash;
+ } u;
+};
+
+struct crypto4xx_alg {
+ struct list_head entry;
+ struct crypto4xx_alg_common alg;
+ struct crypto4xx_device *dev;
+};
+
+static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg(
+ struct crypto_alg *x)
+{
+ switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) {
+ case CRYPTO_ALG_TYPE_AHASH:
+ return container_of(__crypto_ahash_alg(x),
+ struct crypto4xx_alg, alg.u.hash);
+ }
+
+ return container_of(x, struct crypto4xx_alg, alg.u.cipher);
+}
+
+extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size);
+extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx);
+extern u32 crypto4xx_alloc_sa_rctx(struct crypto4xx_ctx *ctx,
+ struct crypto4xx_ctx *rctx);
+extern void crypto4xx_free_sa_rctx(struct crypto4xx_ctx *rctx);
+extern void crypto4xx_free_ctx(struct crypto4xx_ctx *ctx);
+extern u32 crypto4xx_alloc_state_record(struct crypto4xx_ctx *ctx);
+extern u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx);
+extern u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx);
+extern u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx);
+extern void crypto4xx_memcpy_le(unsigned int *dst,
+ const unsigned char *buf, int len);
+extern u32 crypto4xx_build_pd(struct crypto_async_request *req,
+ struct crypto4xx_ctx *ctx,
+ struct scatterlist *src,
+ struct scatterlist *dst,
+ unsigned int datalen,
+ void *iv, u32 iv_len);
+extern int crypto4xx_setkey_aes_cbc(struct crypto_ablkcipher *cipher,
+ const u8 *key, unsigned int keylen);
+extern int crypto4xx_encrypt(struct ablkcipher_request *req);
+extern int crypto4xx_decrypt(struct ablkcipher_request *req);
+extern int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm);
+extern int crypto4xx_hash_digest(struct ahash_request *req);
+extern int crypto4xx_hash_final(struct ahash_request *req);
+extern int crypto4xx_hash_update(struct ahash_request *req);
+extern int crypto4xx_hash_init(struct ahash_request *req);
+#endif
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_reg_def.h b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_reg_def.h
new file mode 100644
index 0000000..5f5fbc0
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_reg_def.h
@@ -0,0 +1,284 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * This filr defines the register set for Security Subsystem
+ */
+
+#ifndef __CRYPTO4XX_REG_DEF_H__
+#define __CRYPTO4XX_REG_DEF_H__
+
+/* CRYPTO4XX Register offset */
+#define CRYPTO4XX_DESCRIPTOR 0x00000000
+#define CRYPTO4XX_CTRL_STAT 0x00000000
+#define CRYPTO4XX_SOURCE 0x00000004
+#define CRYPTO4XX_DEST 0x00000008
+#define CRYPTO4XX_SA 0x0000000C
+#define CRYPTO4XX_SA_LENGTH 0x00000010
+#define CRYPTO4XX_LENGTH 0x00000014
+
+#define CRYPTO4XX_PE_DMA_CFG 0x00000040
+#define CRYPTO4XX_PE_DMA_STAT 0x00000044
+#define CRYPTO4XX_PDR_BASE 0x00000048
+#define CRYPTO4XX_RDR_BASE 0x0000004c
+#define CRYPTO4XX_RING_SIZE 0x00000050
+#define CRYPTO4XX_RING_CTRL 0x00000054
+#define CRYPTO4XX_INT_RING_STAT 0x00000058
+#define CRYPTO4XX_EXT_RING_STAT 0x0000005c
+#define CRYPTO4XX_IO_THRESHOLD 0x00000060
+#define CRYPTO4XX_GATH_RING_BASE 0x00000064
+#define CRYPTO4XX_SCAT_RING_BASE 0x00000068
+#define CRYPTO4XX_PART_RING_SIZE 0x0000006c
+#define CRYPTO4XX_PART_RING_CFG 0x00000070
+
+#define CRYPTO4XX_PDR_BASE_UADDR 0x00000080
+#define CRYPTO4XX_RDR_BASE_UADDR 0x00000084
+#define CRYPTO4XX_PKT_SRC_UADDR 0x00000088
+#define CRYPTO4XX_PKT_DEST_UADDR 0x0000008c
+#define CRYPTO4XX_SA_UADDR 0x00000090
+#define CRYPTO4XX_GATH_RING_BASE_UADDR 0x000000A0
+#define CRYPTO4XX_SCAT_RING_BASE_UADDR 0x000000A4
+
+#define CRYPTO4XX_SEQ_RD 0x00000408
+#define CRYPTO4XX_SEQ_MASK_RD 0x0000040C
+
+#define CRYPTO4XX_SA_CMD_0 0x00010600
+#define CRYPTO4XX_SA_CMD_1 0x00010604
+
+#define CRYPTO4XX_STATE_PTR 0x000106dc
+#define CRYPTO4XX_STATE_IV 0x00010700
+#define CRYPTO4XX_STATE_HASH_BYTE_CNT_0 0x00010710
+#define CRYPTO4XX_STATE_HASH_BYTE_CNT_1 0x00010714
+
+#define CRYPTO4XX_STATE_IDIGEST_0 0x00010718
+#define CRYPTO4XX_STATE_IDIGEST_1 0x0001071c
+
+#define CRYPTO4XX_DATA_IN 0x00018000
+#define CRYPTO4XX_DATA_OUT 0x0001c000
+
+#define CRYPTO4XX_INT_UNMASK_STAT 0x000500a0
+#define CRYPTO4XX_INT_MASK_STAT 0x000500a4
+#define CRYPTO4XX_INT_CLR 0x000500a4
+#define CRYPTO4XX_INT_EN 0x000500a8
+
+#define CRYPTO4XX_INT_PKA 0x00000002
+#define CRYPTO4XX_INT_PDR_DONE 0x00008000
+#define CRYPTO4XX_INT_MA_WR_ERR 0x00020000
+#define CRYPTO4XX_INT_MA_RD_ERR 0x00010000
+#define CRYPTO4XX_INT_PE_ERR 0x00000200
+#define CRYPTO4XX_INT_USER_DMA_ERR 0x00000040
+#define CRYPTO4XX_INT_SLAVE_ERR 0x00000010
+#define CRYPTO4XX_INT_MASTER_ERR 0x00000008
+#define CRYPTO4XX_INT_ERROR 0x00030258
+
+#define CRYPTO4XX_INT_CFG 0x000500ac
+#define CRYPTO4XX_INT_DESCR_RD 0x000500b0
+#define CRYPTO4XX_INT_DESCR_CNT 0x000500b4
+#define CRYPTO4XX_INT_TIMEOUT_CNT 0x000500b8
+
+#define CRYPTO4XX_DEVICE_CTRL 0x00060080
+#define CRYPTO4XX_DEVICE_ID 0x00060084
+#define CRYPTO4XX_DEVICE_INFO 0x00060088
+#define CRYPTO4XX_DMA_USER_SRC 0x00060094
+#define CRYPTO4XX_DMA_USER_DEST 0x00060098
+#define CRYPTO4XX_DMA_USER_CMD 0x0006009C
+
+#define CRYPTO4XX_DMA_CFG 0x000600d4
+#define CRYPTO4XX_BYTE_ORDER_CFG 0x000600d8
+#define CRYPTO4XX_ENDIAN_CFG 0x000600d8
+
+#define CRYPTO4XX_PRNG_STAT 0x00070000
+#define CRYPTO4XX_PRNG_CTRL 0x00070004
+#define CRYPTO4XX_PRNG_SEED_L 0x00070008
+#define CRYPTO4XX_PRNG_SEED_H 0x0007000c
+
+#define CRYPTO4XX_PRNG_RES_0 0x00070020
+#define CRYPTO4XX_PRNG_RES_1 0x00070024
+#define CRYPTO4XX_PRNG_RES_2 0x00070028
+#define CRYPTO4XX_PRNG_RES_3 0x0007002C
+
+#define CRYPTO4XX_PRNG_LFSR_L 0x00070030
+#define CRYPTO4XX_PRNG_LFSR_H 0x00070034
+
+/**
+ * Initialize CRYPTO ENGINE registers, and memory bases.
+ */
+#define PPC4XX_PDR_POLL 0x3ff
+#define PPC4XX_OUTPUT_THRESHOLD 2
+#define PPC4XX_INPUT_THRESHOLD 2
+#define PPC4XX_PD_SIZE 6
+#define PPC4XX_CTX_DONE_INT 0x2000
+#define PPC4XX_PD_DONE_INT 0x8000
+#define PPC4XX_BYTE_ORDER 0x22222
+#define PPC4XX_INTERRUPT_CLR 0x3ffff
+#define PPC4XX_PRNG_CTRL_AUTO_EN 0x3
+#define PPC4XX_DC_3DES_EN 1
+#define PPC4XX_INT_DESCR_CNT 4
+#define PPC4XX_INT_TIMEOUT_CNT 0
+#define PPC4XX_INT_CFG 1
+/**
+ * all follow define are ad hoc
+ */
+#define PPC4XX_RING_RETRY 100
+#define PPC4XX_RING_POLL 100
+#define PPC4XX_SDR_SIZE PPC4XX_NUM_SD
+#define PPC4XX_GDR_SIZE PPC4XX_NUM_GD
+
+/**
+ * Generic Security Association (SA) with all possible fields. These will
+ * never likely used except for reference purpose. These structure format
+ * can be not changed as the hardware expects them to be layout as defined.
+ * Field can be removed or reduced but ordering can not be changed.
+ */
+#define CRYPTO4XX_DMA_CFG_OFFSET 0x40
+union ce_pe_dma_cfg {
+ struct {
+ u32 rsv:7;
+ u32 dir_host:1;
+ u32 rsv1:2;
+ u32 bo_td_en:1;
+ u32 dis_pdr_upd:1;
+ u32 bo_sgpd_en:1;
+ u32 bo_data_en:1;
+ u32 bo_sa_en:1;
+ u32 bo_pd_en:1;
+ u32 rsv2:4;
+ u32 dynamic_sa_en:1;
+ u32 pdr_mode:2;
+ u32 pe_mode:1;
+ u32 rsv3:5;
+ u32 reset_sg:1;
+ u32 reset_pdr:1;
+ u32 reset_pe:1;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define CRYPTO4XX_PDR_BASE_OFFSET 0x48
+#define CRYPTO4XX_RDR_BASE_OFFSET 0x4c
+#define CRYPTO4XX_RING_SIZE_OFFSET 0x50
+union ce_ring_size {
+ struct {
+ u32 ring_offset:16;
+ u32 rsv:6;
+ u32 ring_size:10;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define CRYPTO4XX_RING_CONTROL_OFFSET 0x54
+union ce_ring_contol {
+ struct {
+ u32 continuous:1;
+ u32 rsv:5;
+ u32 ring_retry_divisor:10;
+ u32 rsv1:4;
+ u32 ring_poll_divisor:10;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define CRYPTO4XX_IO_THRESHOLD_OFFSET 0x60
+union ce_io_threshold {
+ struct {
+ u32 rsv:6;
+ u32 output_threshold:10;
+ u32 rsv1:6;
+ u32 input_threshold:10;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define CRYPTO4XX_GATHER_RING_BASE_OFFSET 0x64
+#define CRYPTO4XX_SCATTER_RING_BASE_OFFSET 0x68
+
+union ce_part_ring_size {
+ struct {
+ u32 sdr_size:16;
+ u32 gdr_size:16;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define MAX_BURST_SIZE_32 0
+#define MAX_BURST_SIZE_64 1
+#define MAX_BURST_SIZE_128 2
+#define MAX_BURST_SIZE_256 3
+
+/* gather descriptor control length */
+struct gd_ctl_len {
+ u32 len:16;
+ u32 rsv:14;
+ u32 done:1;
+ u32 ready:1;
+} __attribute__((packed));
+
+struct ce_gd {
+ u32 ptr;
+ struct gd_ctl_len ctl_len;
+} __attribute__((packed));
+
+struct sd_ctl {
+ u32 ctl:30;
+ u32 done:1;
+ u32 rdy:1;
+} __attribute__((packed));
+
+struct ce_sd {
+ u32 ptr;
+ struct sd_ctl ctl;
+} __attribute__((packed));
+
+#define PD_PAD_CTL_32 0x10
+#define PD_PAD_CTL_64 0x20
+#define PD_PAD_CTL_128 0x40
+#define PD_PAD_CTL_256 0x80
+union ce_pd_ctl {
+ struct {
+ u32 pd_pad_ctl:8;
+ u32 status:8;
+ u32 next_hdr:8;
+ u32 rsv:2;
+ u32 cached_sa:1;
+ u32 hash_final:1;
+ u32 init_arc4:1;
+ u32 rsv1:1;
+ u32 pe_done:1;
+ u32 host_ready:1;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+union ce_pd_ctl_len {
+ struct {
+ u32 bypass:8;
+ u32 pe_done:1;
+ u32 host_ready:1;
+ u32 rsv:2;
+ u32 pkt_len:20;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+struct ce_pd {
+ union ce_pd_ctl pd_ctl;
+ u32 src;
+ u32 dest;
+ u32 sa; /* get from ctx->sa_dma_addr */
+ u32 sa_len; /* only if dynamic sa is used */
+ union ce_pd_ctl_len pd_ctl_len;
+
+} __attribute__((packed));
+#endif
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.c b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.c
new file mode 100644
index 0000000..de8a7a4
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.c
@@ -0,0 +1,108 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * @file crypto4xx_sa.c
+ *
+ * This file implements the security context
+ * associate format.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/mod_devicetable.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock_types.h>
+#include <linux/highmem.h>
+#include <linux/scatterlist.h>
+#include <linux/crypto.h>
+#include <crypto/algapi.h>
+#include <crypto/des.h>
+#include "crypto4xx_reg_def.h"
+#include "crypto4xx_sa.h"
+#include "crypto4xx_core.h"
+
+u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx)
+{
+ u32 offset;
+ union dynamic_sa_contents cts;
+
+ if (ctx->direction == DIR_INBOUND)
+ cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents;
+ else
+ cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents;
+ offset = cts.bf.key_size
+ + cts.bf.inner_size
+ + cts.bf.outer_size
+ + cts.bf.spi
+ + cts.bf.seq_num0
+ + cts.bf.seq_num1
+ + cts.bf.seq_num_mask0
+ + cts.bf.seq_num_mask1
+ + cts.bf.seq_num_mask2
+ + cts.bf.seq_num_mask3;
+
+ return sizeof(struct dynamic_sa_ctl) + offset * 4;
+}
+
+u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx)
+{
+ u32 offset;
+ union dynamic_sa_contents cts;
+
+ if (ctx->direction == DIR_INBOUND)
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
+ else
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
+ offset = cts.bf.key_size
+ + cts.bf.inner_size
+ + cts.bf.outer_size
+ + cts.bf.spi
+ + cts.bf.seq_num0
+ + cts.bf.seq_num1
+ + cts.bf.seq_num_mask0
+ + cts.bf.seq_num_mask1
+ + cts.bf.seq_num_mask2
+ + cts.bf.seq_num_mask3
+ + cts.bf.iv0
+ + cts.bf.iv1
+ + cts.bf.iv2
+ + cts.bf.iv3;
+
+ return sizeof(struct dynamic_sa_ctl) + offset * 4;
+}
+
+u32 get_dynamic_sa_iv_size(struct crypto4xx_ctx *ctx)
+{
+ union dynamic_sa_contents cts;
+
+ if (ctx->direction == DIR_INBOUND)
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
+ else
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
+ return (cts.bf.iv0 + cts.bf.iv1 + cts.bf.iv2 + cts.bf.iv3) * 4;
+}
+
+u32 get_dynamic_sa_offset_key_field(struct crypto4xx_ctx *ctx)
+{
+ union dynamic_sa_contents cts;
+
+ if (ctx->direction == DIR_INBOUND)
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_in)->sa_contents;
+ else
+ cts.w = ((struct dynamic_sa_ctl *) ctx->sa_out)->sa_contents;
+
+ return sizeof(struct dynamic_sa_ctl);
+}
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.h b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.h
new file mode 100644
index 0000000..1352d58
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/amcc/crypto4xx_sa.h
@@ -0,0 +1,243 @@
+/**
+ * AMCC SoC PPC4xx Crypto Driver
+ *
+ * Copyright (c) 2008 Applied Micro Circuits Corporation.
+ * All rights reserved. James Hsiao <jhsiao@amcc.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * This file defines the security context
+ * associate format.
+ */
+
+#ifndef __CRYPTO4XX_SA_H__
+#define __CRYPTO4XX_SA_H__
+
+#define AES_IV_SIZE 16
+
+/**
+ * Contents of Dynamic Security Association (SA) with all possible fields
+ */
+union dynamic_sa_contents {
+ struct {
+ u32 arc4_state_ptr:1;
+ u32 arc4_ij_ptr:1;
+ u32 state_ptr:1;
+ u32 iv3:1;
+ u32 iv2:1;
+ u32 iv1:1;
+ u32 iv0:1;
+ u32 seq_num_mask3:1;
+ u32 seq_num_mask2:1;
+ u32 seq_num_mask1:1;
+ u32 seq_num_mask0:1;
+ u32 seq_num1:1;
+ u32 seq_num0:1;
+ u32 spi:1;
+ u32 outer_size:5;
+ u32 inner_size:5;
+ u32 key_size:4;
+ u32 cmd_size:4;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define DIR_OUTBOUND 0
+#define DIR_INBOUND 1
+#define SA_OP_GROUP_BASIC 0
+#define SA_OPCODE_ENCRYPT 0
+#define SA_OPCODE_DECRYPT 0
+#define SA_OPCODE_HASH 3
+#define SA_CIPHER_ALG_DES 0
+#define SA_CIPHER_ALG_3DES 1
+#define SA_CIPHER_ALG_ARC4 2
+#define SA_CIPHER_ALG_AES 3
+#define SA_CIPHER_ALG_KASUMI 4
+#define SA_CIPHER_ALG_NULL 15
+
+#define SA_HASH_ALG_MD5 0
+#define SA_HASH_ALG_SHA1 1
+#define SA_HASH_ALG_NULL 15
+#define SA_HASH_ALG_SHA1_DIGEST_SIZE 20
+
+#define SA_LOAD_HASH_FROM_SA 0
+#define SA_LOAD_HASH_FROM_STATE 2
+#define SA_NOT_LOAD_HASH 3
+#define SA_LOAD_IV_FROM_SA 0
+#define SA_LOAD_IV_FROM_INPUT 1
+#define SA_LOAD_IV_FROM_STATE 2
+#define SA_LOAD_IV_GEN_IV 3
+
+#define SA_PAD_TYPE_CONSTANT 2
+#define SA_PAD_TYPE_ZERO 3
+#define SA_PAD_TYPE_TLS 5
+#define SA_PAD_TYPE_DTLS 5
+#define SA_NOT_SAVE_HASH 0
+#define SA_SAVE_HASH 1
+#define SA_NOT_SAVE_IV 0
+#define SA_SAVE_IV 1
+#define SA_HEADER_PROC 1
+#define SA_NO_HEADER_PROC 0
+
+union sa_command_0 {
+ struct {
+ u32 scatter:1;
+ u32 gather:1;
+ u32 save_hash_state:1;
+ u32 save_iv:1;
+ u32 load_hash_state:2;
+ u32 load_iv:2;
+ u32 digest_len:4;
+ u32 hdr_proc:1;
+ u32 extend_pad:1;
+ u32 stream_cipher_pad:1;
+ u32 rsv:1;
+ u32 hash_alg:4;
+ u32 cipher_alg:4;
+ u32 pad_type:2;
+ u32 op_group:2;
+ u32 dir:1;
+ u32 opcode:3;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+#define CRYPTO_MODE_ECB 0
+#define CRYPTO_MODE_CBC 1
+
+#define CRYPTO_FEEDBACK_MODE_NO_FB 0
+#define CRYPTO_FEEDBACK_MODE_64BIT_OFB 0
+#define CRYPTO_FEEDBACK_MODE_8BIT_CFB 1
+#define CRYPTO_FEEDBACK_MODE_1BIT_CFB 2
+#define CRYPTO_FEEDBACK_MODE_128BIT_CFB 3
+
+#define SA_AES_KEY_LEN_128 2
+#define SA_AES_KEY_LEN_192 3
+#define SA_AES_KEY_LEN_256 4
+
+#define SA_REV2 1
+/**
+ * The follow defines bits sa_command_1
+ * In Basic hash mode this bit define simple hash or hmac.
+ * In IPsec mode, this bit define muting control.
+ */
+#define SA_HASH_MODE_HASH 0
+#define SA_HASH_MODE_HMAC 1
+#define SA_MC_ENABLE 0
+#define SA_MC_DISABLE 1
+#define SA_NOT_COPY_HDR 0
+#define SA_COPY_HDR 1
+#define SA_NOT_COPY_PAD 0
+#define SA_COPY_PAD 1
+#define SA_NOT_COPY_PAYLOAD 0
+#define SA_COPY_PAYLOAD 1
+#define SA_EXTENDED_SN_OFF 0
+#define SA_EXTENDED_SN_ON 1
+#define SA_SEQ_MASK_OFF 0
+#define SA_SEQ_MASK_ON 1
+
+union sa_command_1 {
+ struct {
+ u32 crypto_mode31:1;
+ u32 save_arc4_state:1;
+ u32 arc4_stateful:1;
+ u32 key_len:5;
+ u32 hash_crypto_offset:8;
+ u32 sa_rev:2;
+ u32 byte_offset:1;
+ u32 hmac_muting:1;
+ u32 feedback_mode:2;
+ u32 crypto_mode9_8:2;
+ u32 extended_seq_num:1;
+ u32 seq_num_mask:1;
+ u32 mutable_bit_proc:1;
+ u32 ip_version:1;
+ u32 copy_pad:1;
+ u32 copy_payload:1;
+ u32 copy_hdr:1;
+ u32 rsv1:1;
+ } bf;
+ u32 w;
+} __attribute__((packed));
+
+struct dynamic_sa_ctl {
+ u32 sa_contents;
+ union sa_command_0 sa_command_0;
+ union sa_command_1 sa_command_1;
+} __attribute__((packed));
+
+/**
+ * State Record for Security Association (SA)
+ */
+struct sa_state_record {
+ u32 save_iv[4];
+ u32 save_hash_byte_cnt[2];
+ u32 save_digest[16];
+} __attribute__((packed));
+
+/**
+ * Security Association (SA) for AES128
+ *
+ */
+struct dynamic_sa_aes128 {
+ struct dynamic_sa_ctl ctrl;
+ u32 key[4];
+ u32 iv[4]; /* for CBC, OFC, and CFB mode */
+ u32 state_ptr;
+ u32 reserved;
+} __attribute__((packed));
+
+#define SA_AES128_LEN (sizeof(struct dynamic_sa_aes128)/4)
+#define SA_AES128_CONTENTS 0x3e000042
+
+/*
+ * Security Association (SA) for AES192
+ */
+struct dynamic_sa_aes192 {
+ struct dynamic_sa_ctl ctrl;
+ u32 key[6];
+ u32 iv[4]; /* for CBC, OFC, and CFB mode */
+ u32 state_ptr;
+ u32 reserved;
+} __attribute__((packed));
+
+#define SA_AES192_LEN (sizeof(struct dynamic_sa_aes192)/4)
+#define SA_AES192_CONTENTS 0x3e000062
+
+/**
+ * Security Association (SA) for AES256
+ */
+struct dynamic_sa_aes256 {
+ struct dynamic_sa_ctl ctrl;
+ u32 key[8];
+ u32 iv[4]; /* for CBC, OFC, and CFB mode */
+ u32 state_ptr;
+ u32 reserved;
+} __attribute__((packed));
+
+#define SA_AES256_LEN (sizeof(struct dynamic_sa_aes256)/4)
+#define SA_AES256_CONTENTS 0x3e000082
+#define SA_AES_CONTENTS 0x3e000002
+
+/**
+ * Security Association (SA) for HASH160: HMAC-SHA1
+ */
+struct dynamic_sa_hash160 {
+ struct dynamic_sa_ctl ctrl;
+ u32 inner_digest[5];
+ u32 outer_digest[5];
+ u32 state_ptr;
+ u32 reserved;
+} __attribute__((packed));
+#define SA_HASH160_LEN (sizeof(struct dynamic_sa_hash160)/4)
+#define SA_HASH160_CONTENTS 0x2000a502
+
+#endif
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Kconfig b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Kconfig
new file mode 100644
index 0000000..2d876bb
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Kconfig
@@ -0,0 +1,72 @@
+config CRYPTO_DEV_FSL_CAAM
+ tristate "Freescale CAAM-Multicore driver backend"
+ depends on FSL_SOC
+ help
+ Enables the driver module for Freescale's Cryptographic Accelerator
+ and Assurance Module (CAAM), also known as the SEC version 4 (SEC4).
+ This module adds a job ring operation interface, and configures h/w
+ to operate as a DPAA component automatically, depending
+ on h/w feature availability.
+
+ To compile this driver as a module, choose M here: the module
+ will be called caam.
+
+config CRYPTO_DEV_FSL_CAAM_RINGSIZE
+ int "Job Ring size"
+ depends on CRYPTO_DEV_FSL_CAAM
+ range 2 9
+ default "9"
+ help
+ Select size of Job Rings as a power of 2, within the
+ range 2-9 (ring size 4-512).
+ Examples:
+ 2 => 4
+ 3 => 8
+ 4 => 16
+ 5 => 32
+ 6 => 64
+ 7 => 128
+ 8 => 256
+ 9 => 512
+
+config CRYPTO_DEV_FSL_CAAM_INTC
+ bool "Job Ring interrupt coalescing"
+ depends on CRYPTO_DEV_FSL_CAAM
+ default y
+ help
+ Enable the Job Ring's interrupt coalescing feature.
+
+config CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD
+ int "Job Ring interrupt coalescing count threshold"
+ depends on CRYPTO_DEV_FSL_CAAM_INTC
+ range 1 255
+ default 255
+ help
+ Select number of descriptor completions to queue before
+ raising an interrupt, in the range 1-255. Note that a selection
+ of 1 functionally defeats the coalescing feature, and a selection
+ equal or greater than the job ring size will force timeouts.
+
+config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
+ int "Job Ring interrupt coalescing timer threshold"
+ depends on CRYPTO_DEV_FSL_CAAM_INTC
+ range 1 65535
+ default 2048
+ help
+ Select number of bus clocks/64 to timeout in the case that one or
+ more descriptor completions are queued without reaching the count
+ threshold. Range is 1-65535.
+
+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API
+ tristate "Register algorithm implementations with the Crypto API"
+ depends on CRYPTO_DEV_FSL_CAAM
+ default y
+ select CRYPTO_ALGAPI
+ select CRYPTO_AUTHENC
+ help
+ Selecting this will offload crypto for users of the
+ scatterlist crypto API (such as the linux native IPSec
+ stack) to the SEC4 via job ring.
+
+ To compile this as a module, choose M here: the module
+ will be called caamalg.
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Makefile b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Makefile
new file mode 100644
index 0000000..ef39011
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/Makefile
@@ -0,0 +1,8 @@
+#
+# Makefile for the CAAM backend and dependent components
+#
+
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o
+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o
+
+caam-objs := ctrl.o jr.o error.o
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/caamalg.c b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/caamalg.c
new file mode 100644
index 0000000..534a364
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/caamalg.c
@@ -0,0 +1,2419 @@
+/*
+ * caam - Freescale FSL CAAM support for crypto API
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Based on talitos crypto API driver.
+ *
+ * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
+ *
+ * --------------- ---------------
+ * | JobDesc #1 |-------------------->| ShareDesc |
+ * | *(packet 1) | | (PDB) |
+ * --------------- |------------->| (hashKey) |
+ * . | | (cipherKey) |
+ * . | |-------->| (operation) |
+ * --------------- | | ---------------
+ * | JobDesc #2 |------| |
+ * | *(packet 2) | |
+ * --------------- |
+ * . |
+ * . |
+ * --------------- |
+ * | JobDesc #3 |------------
+ * | *(packet 3) |
+ * ---------------
+ *
+ * The SharedDesc never changes for a connection unless rekeyed, but
+ * each packet will likely be in a different place. So all we need
+ * to know to process the packet is where the input is, where the
+ * output goes, and what context we want to process with. Context is
+ * in the SharedDesc, packet references in the JobDesc.
+ *
+ * So, a job desc looks like:
+ *
+ * ---------------------
+ * | Header |
+ * | ShareDesc Pointer |
+ * | SEQ_OUT_PTR |
+ * | (output buffer) |
+ * | SEQ_IN_PTR |
+ * | (input buffer) |
+ * | LOAD (to DECO) |
+ * ---------------------
+ */
+
+#include "compat.h"
+
+#include "regs.h"
+#include "intern.h"
+#include "desc_constr.h"
+#include "jr.h"
+#include "error.h"
+
+/*
+ * crypto alg
+ */
+#define CAAM_CRA_PRIORITY 3000
+/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
+#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
+ SHA512_DIGEST_SIZE * 2)
+/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
+#define CAAM_MAX_IV_LENGTH 16
+
+/* length of descriptors text */
+#define DESC_JOB_IO_LEN (CAAM_CMD_SZ * 3 + CAAM_PTR_SZ * 3)
+
+#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
+#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 16 * CAAM_CMD_SZ)
+#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 21 * CAAM_CMD_SZ)
+#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
+
+#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
+#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
+ 20 * CAAM_CMD_SZ)
+#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
+ 15 * CAAM_CMD_SZ)
+
+#define DESC_MAX_USED_BYTES (DESC_AEAD_GIVENC_LEN + \
+ CAAM_MAX_KEY_SIZE)
+#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
+
+#ifdef DEBUG
+/* for print_hex_dumps with line references */
+#define xstr(s) str(s)
+#define str(s) #s
+#define debug(format, arg...) printk(format, arg)
+#else
+#define debug(format, arg...)
+#endif
+
+/* Set DK bit in class 1 operation if shared */
+static inline void append_dec_op1(u32 *desc, u32 type)
+{
+ u32 *jump_cmd, *uncond_jump_cmd;
+
+ jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
+ append_operation(desc, type | OP_ALG_AS_INITFINAL |
+ OP_ALG_DECRYPT);
+ uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
+ set_jump_tgt_here(desc, jump_cmd);
+ append_operation(desc, type | OP_ALG_AS_INITFINAL |
+ OP_ALG_DECRYPT | OP_ALG_AAI_DK);
+ set_jump_tgt_here(desc, uncond_jump_cmd);
+}
+
+/*
+ * Wait for completion of class 1 key loading before allowing
+ * error propagation
+ */
+static inline void append_dec_shr_done(u32 *desc)
+{
+ u32 *jump_cmd;
+
+ jump_cmd = append_jump(desc, JUMP_CLASS_CLASS1 | JUMP_TEST_ALL);
+ set_jump_tgt_here(desc, jump_cmd);
+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
+}
+
+/*
+ * For aead functions, read payload and write payload,
+ * both of which are specified in req->src and req->dst
+ */
+static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
+{
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
+ KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
+}
+
+/*
+ * For aead encrypt and decrypt, read iv for both classes
+ */
+static inline void aead_append_ld_iv(u32 *desc, int ivsize)
+{
+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | ivsize);
+ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | ivsize);
+}
+
+/*
+ * For ablkcipher encrypt and decrypt, read from req->src and
+ * write to req->dst
+ */
+static inline void ablkcipher_append_src_dst(u32 *desc)
+{
+ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); \
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | \
+ KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); \
+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); \
+}
+
+/*
+ * If all data, including src (with assoc and iv) or dst (with iv only) are
+ * contiguous
+ */
+#define GIV_SRC_CONTIG 1
+#define GIV_DST_CONTIG (1 << 1)
+
+/*
+ * per-session context
+ */
+struct caam_ctx {
+ struct device *jrdev;
+ u32 sh_desc_enc[DESC_MAX_USED_LEN];
+ u32 sh_desc_dec[DESC_MAX_USED_LEN];
+ u32 sh_desc_givenc[DESC_MAX_USED_LEN];
+ dma_addr_t sh_desc_enc_dma;
+ dma_addr_t sh_desc_dec_dma;
+ dma_addr_t sh_desc_givenc_dma;
+ u32 class1_alg_type;
+ u32 class2_alg_type;
+ u32 alg_op;
+ u8 key[CAAM_MAX_KEY_SIZE];
+ dma_addr_t key_dma;
+ unsigned int enckeylen;
+ unsigned int split_key_len;
+ unsigned int split_key_pad_len;
+ unsigned int authsize;
+};
+
+static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
+ int keys_fit_inline)
+{
+ if (keys_fit_inline) {
+ append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
+ ctx->split_key_len, CLASS_2 |
+ KEY_DEST_MDHA_SPLIT | KEY_ENC);
+ append_key_as_imm(desc, (void *)ctx->key +
+ ctx->split_key_pad_len, ctx->enckeylen,
+ ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+ } else {
+ append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
+ KEY_DEST_MDHA_SPLIT | KEY_ENC);
+ append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
+ ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
+ }
+}
+
+static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
+ int keys_fit_inline)
+{
+ u32 *key_jump_cmd;
+
+ init_sh_desc(desc, HDR_SHARE_WAIT);
+
+ /* Skip if already shared */
+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+ JUMP_COND_SHRD);
+
+ append_key_aead(desc, ctx, keys_fit_inline);
+
+ set_jump_tgt_here(desc, key_jump_cmd);
+
+ /* Propagate errors from shared to job descriptor */
+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
+}
+
+static int aead_set_sh_desc(struct crypto_aead *aead)
+{
+ struct aead_tfm *tfm = &aead->base.crt_aead;
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ bool keys_fit_inline = 0;
+ u32 *key_jump_cmd, *jump_cmd;
+ u32 geniv, moveiv;
+ u32 *desc;
+
+ if (!ctx->enckeylen || !ctx->authsize)
+ return 0;
+
+ /*
+ * Job Descriptor and Shared Descriptors
+ * must all fit into the 64-word Descriptor h/w Buffer
+ */
+ if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
+ ctx->split_key_pad_len + ctx->enckeylen <=
+ CAAM_DESC_BYTES_MAX)
+ keys_fit_inline = 1;
+
+ /* aead_encrypt shared descriptor */
+ desc = ctx->sh_desc_enc;
+
+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
+
+ /* Class 2 operation */
+ append_operation(desc, ctx->class2_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+
+ /* cryptlen = seqoutlen - authsize */
+ append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
+
+ /* assoclen + cryptlen = seqinlen - ivsize */
+ append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
+
+ /* assoclen + cryptlen = (assoclen + cryptlen) - cryptlen */
+ append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
+
+ /* read assoc before reading payload */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+ KEY_VLF);
+ aead_append_ld_iv(desc, tfm->ivsize);
+
+ /* Class 1 operation */
+ append_operation(desc, ctx->class1_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+
+ /* Read and write cryptlen bytes */
+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
+
+ /* Write ICV */
+ append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
+ LDST_SRCDST_BYTE_CONTEXT);
+
+ ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
+ desc_bytes(desc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
+ dev_err(jrdev, "unable to map shared descriptor\n");
+ return -ENOMEM;
+ }
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead enc shdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+#endif
+
+ /*
+ * Job Descriptor and Shared Descriptors
+ * must all fit into the 64-word Descriptor h/w Buffer
+ */
+ if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
+ ctx->split_key_pad_len + ctx->enckeylen <=
+ CAAM_DESC_BYTES_MAX)
+ keys_fit_inline = 1;
+
+ desc = ctx->sh_desc_dec;
+
+ /* aead_decrypt shared descriptor */
+ init_sh_desc(desc, HDR_SHARE_WAIT);
+
+ /* Skip if already shared */
+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+ JUMP_COND_SHRD);
+
+ append_key_aead(desc, ctx, keys_fit_inline);
+
+ /* Only propagate error immediately if shared */
+ jump_cmd = append_jump(desc, JUMP_TEST_ALL);
+ set_jump_tgt_here(desc, key_jump_cmd);
+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
+ set_jump_tgt_here(desc, jump_cmd);
+
+ /* Class 2 operation */
+ append_operation(desc, ctx->class2_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
+
+ /* assoclen + cryptlen = seqinlen - ivsize */
+ append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
+ ctx->authsize + tfm->ivsize)
+ /* assoclen = (assoclen + cryptlen) - cryptlen */
+ append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
+ append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
+
+ /* read assoc before reading payload */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+ KEY_VLF);
+
+ aead_append_ld_iv(desc, tfm->ivsize);
+
+ append_dec_op1(desc, ctx->class1_alg_type);
+
+ /* Read and write cryptlen bytes */
+ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
+
+ /* Load ICV */
+ append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
+ FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
+ append_dec_shr_done(desc);
+
+ ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
+ desc_bytes(desc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
+ dev_err(jrdev, "unable to map shared descriptor\n");
+ return -ENOMEM;
+ }
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead dec shdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+#endif
+
+ /*
+ * Job Descriptor and Shared Descriptors
+ * must all fit into the 64-word Descriptor h/w Buffer
+ */
+ if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
+ ctx->split_key_pad_len + ctx->enckeylen <=
+ CAAM_DESC_BYTES_MAX)
+ keys_fit_inline = 1;
+
+ /* aead_givencrypt shared descriptor */
+ desc = ctx->sh_desc_givenc;
+
+ init_sh_desc_key_aead(desc, ctx, keys_fit_inline);
+
+ /* Generate IV */
+ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
+ NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
+ NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
+ append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
+ append_move(desc, MOVE_SRC_INFIFO |
+ MOVE_DEST_CLASS1CTX | (tfm->ivsize << MOVE_LEN_SHIFT));
+ append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
+
+ /* Copy IV to class 1 context */
+ append_move(desc, MOVE_SRC_CLASS1CTX |
+ MOVE_DEST_OUTFIFO | (tfm->ivsize << MOVE_LEN_SHIFT));
+
+ /* Return to encryption */
+ append_operation(desc, ctx->class2_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+
+ /* ivsize + cryptlen = seqoutlen - authsize */
+ append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
+
+ /* assoclen = seqinlen - (ivsize + cryptlen) */
+ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
+
+ /* read assoc before reading payload */
+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
+ KEY_VLF);
+
+ /* Copy iv from class 1 ctx to class 2 fifo*/
+ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
+ NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
+ append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
+ append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
+ LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
+
+ /* Class 1 operation */
+ append_operation(desc, ctx->class1_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+
+ /* Will write ivsize + cryptlen */
+ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+
+ /* Not need to reload iv */
+ append_seq_fifo_load(desc, tfm->ivsize,
+ FIFOLD_CLASS_SKIP);
+
+ /* Will read cryptlen */
+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
+
+ /* Write ICV */
+ append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
+ LDST_SRCDST_BYTE_CONTEXT);
+
+ ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
+ desc_bytes(desc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
+ dev_err(jrdev, "unable to map shared descriptor\n");
+ return -ENOMEM;
+ }
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead givenc shdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+#endif
+
+ return 0;
+}
+
+static int aead_setauthsize(struct crypto_aead *authenc,
+ unsigned int authsize)
+{
+ struct caam_ctx *ctx = crypto_aead_ctx(authenc);
+
+ ctx->authsize = authsize;
+ aead_set_sh_desc(authenc);
+
+ return 0;
+}
+
+struct split_key_result {
+ struct completion completion;
+ int err;
+};
+
+static void split_key_done(struct device *dev, u32 *desc, u32 err,
+ void *context)
+{
+ struct split_key_result *res = context;
+
+#ifdef DEBUG
+ dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+
+ if (err) {
+ char tmp[CAAM_ERROR_STR_MAX];
+
+ dev_err(dev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+ }
+
+ res->err = err;
+
+ complete(&res->completion);
+}
+
+/*
+get a split ipad/opad key
+
+Split key generation-----------------------------------------------
+
+[00] 0xb0810008 jobdesc: stidx=1 share=never len=8
+[01] 0x04000014 key: class2->keyreg len=20
+ @0xffe01000
+[03] 0x84410014 operation: cls2-op sha1 hmac init dec
+[04] 0x24940000 fifold: class2 msgdata-last2 len=0 imm
+[05] 0xa4000001 jump: class2 local all ->1 [06]
+[06] 0x64260028 fifostr: class2 mdsplit-jdk len=40
+ @0xffe04000
+*/
+static u32 gen_split_key(struct caam_ctx *ctx, const u8 *key_in, u32 authkeylen)
+{
+ struct device *jrdev = ctx->jrdev;
+ u32 *desc;
+ struct split_key_result result;
+ dma_addr_t dma_addr_in, dma_addr_out;
+ int ret = 0;
+
+ desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
+
+ init_job_desc(desc, 0);
+
+ dma_addr_in = dma_map_single(jrdev, (void *)key_in, authkeylen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, dma_addr_in)) {
+ dev_err(jrdev, "unable to map key input memory\n");
+ kfree(desc);
+ return -ENOMEM;
+ }
+ append_key(desc, dma_addr_in, authkeylen, CLASS_2 |
+ KEY_DEST_CLASS_REG);
+
+ /* Sets MDHA up into an HMAC-INIT */
+ append_operation(desc, ctx->alg_op | OP_ALG_DECRYPT |
+ OP_ALG_AS_INIT);
+
+ /*
+ * do a FIFO_LOAD of zero, this will trigger the internal key expansion
+ into both pads inside MDHA
+ */
+ append_fifo_load_as_imm(desc, NULL, 0, LDST_CLASS_2_CCB |
+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2);
+
+ /*
+ * FIFO_STORE with the explicit split-key content store
+ * (0x26 output type)
+ */
+ dma_addr_out = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(jrdev, dma_addr_out)) {
+ dev_err(jrdev, "unable to map key output memory\n");
+ kfree(desc);
+ return -ENOMEM;
+ }
+ append_fifo_store(desc, dma_addr_out, ctx->split_key_len,
+ LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK);
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, authkeylen, 1);
+ print_hex_dump(KERN_ERR, "jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
+#endif
+
+ result.err = 0;
+ init_completion(&result.completion);
+
+ ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
+ if (!ret) {
+ /* in progress */
+ wait_for_completion_interruptible(&result.completion);
+ ret = result.err;
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+ ctx->split_key_pad_len, 1);
+#endif
+ }
+
+ dma_unmap_single(jrdev, dma_addr_out, ctx->split_key_pad_len,
+ DMA_FROM_DEVICE);
+ dma_unmap_single(jrdev, dma_addr_in, authkeylen, DMA_TO_DEVICE);
+
+ kfree(desc);
+
+ return ret;
+}
+
+static int aead_setkey(struct crypto_aead *aead,
+ const u8 *key, unsigned int keylen)
+{
+ /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
+ static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ struct rtattr *rta = (void *)key;
+ struct crypto_authenc_key_param *param;
+ unsigned int authkeylen;
+ unsigned int enckeylen;
+ int ret = 0;
+
+ param = RTA_DATA(rta);
+ enckeylen = be32_to_cpu(param->enckeylen);
+
+ key += RTA_ALIGN(rta->rta_len);
+ keylen -= RTA_ALIGN(rta->rta_len);
+
+ if (keylen < enckeylen)
+ goto badkey;
+
+ authkeylen = keylen - enckeylen;
+
+ if (keylen > CAAM_MAX_KEY_SIZE)
+ goto badkey;
+
+ /* Pick class 2 key length from algorithm submask */
+ ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
+ OP_ALG_ALGSEL_SHIFT] * 2;
+ ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
+
+#ifdef DEBUG
+ printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
+ keylen, enckeylen, authkeylen);
+ printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
+ ctx->split_key_len, ctx->split_key_pad_len);
+ print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+
+ ret = gen_split_key(ctx, key, authkeylen);
+ if (ret) {
+ goto badkey;
+ }
+
+ /* postpend encryption key to auth split key */
+ memcpy(ctx->key + ctx->split_key_pad_len, key + authkeylen, enckeylen);
+
+ ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
+ enckeylen, DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->key_dma)) {
+ dev_err(jrdev, "unable to map key i/o memory\n");
+ return -ENOMEM;
+ }
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ctx.key@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
+ ctx->split_key_pad_len + enckeylen, 1);
+#endif
+
+ ctx->enckeylen = enckeylen;
+
+ ret = aead_set_sh_desc(aead);
+ if (ret) {
+ dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
+ enckeylen, DMA_TO_DEVICE);
+ }
+
+ return ret;
+badkey:
+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+}
+
+static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
+ const u8 *key, unsigned int keylen)
+{
+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+ struct ablkcipher_tfm *tfm = &ablkcipher->base.crt_ablkcipher;
+ struct device *jrdev = ctx->jrdev;
+ int ret = 0;
+ u32 *key_jump_cmd, *jump_cmd;
+ u32 *desc;
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "key in @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
+#endif
+
+ memcpy(ctx->key, key, keylen);
+ ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->key_dma)) {
+ dev_err(jrdev, "unable to map key i/o memory\n");
+ return -ENOMEM;
+ }
+ ctx->enckeylen = keylen;
+
+ /* ablkcipher_encrypt shared descriptor */
+ desc = ctx->sh_desc_enc;
+ init_sh_desc(desc, HDR_SHARE_WAIT);
+ /* Skip if already shared */
+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+ JUMP_COND_SHRD);
+
+ /* Load class1 key only */
+ append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
+ ctx->enckeylen, CLASS_1 |
+ KEY_DEST_CLASS_REG);
+
+ set_jump_tgt_here(desc, key_jump_cmd);
+
+ /* Propagate errors from shared to job descriptor */
+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
+
+ /* Load iv */
+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | tfm->ivsize);
+
+ /* Load operation */
+ append_operation(desc, ctx->class1_alg_type |
+ OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
+
+ /* Perform operation */
+ ablkcipher_append_src_dst(desc);
+
+ ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
+ desc_bytes(desc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
+ dev_err(jrdev, "unable to map shared descriptor\n");
+ return -ENOMEM;
+ }
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ablkcipher enc shdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+#endif
+ /* ablkcipher_decrypt shared descriptor */
+ desc = ctx->sh_desc_dec;
+
+ init_sh_desc(desc, HDR_SHARE_WAIT);
+ /* Skip if already shared */
+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
+ JUMP_COND_SHRD);
+
+ /* Load class1 key only */
+ append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
+ ctx->enckeylen, CLASS_1 |
+ KEY_DEST_CLASS_REG);
+
+ /* For aead, only propagate error immediately if shared */
+ jump_cmd = append_jump(desc, JUMP_TEST_ALL);
+ set_jump_tgt_here(desc, key_jump_cmd);
+ append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD);
+ set_jump_tgt_here(desc, jump_cmd);
+
+ /* load IV */
+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
+ LDST_CLASS_1_CCB | tfm->ivsize);
+
+ /* Choose operation */
+ append_dec_op1(desc, ctx->class1_alg_type);
+
+ /* Perform operation */
+ ablkcipher_append_src_dst(desc);
+
+ /* Wait for key to load before allowing propagating error */
+ append_dec_shr_done(desc);
+
+ ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
+ desc_bytes(desc),
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
+ dev_err(jrdev, "unable to map shared descriptor\n");
+ return -ENOMEM;
+ }
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ablkcipher dec shdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, desc,
+ desc_bytes(desc), 1);
+#endif
+
+ return ret;
+}
+
+struct link_tbl_entry {
+ u64 ptr;
+ u32 len;
+ u8 reserved;
+ u8 buf_pool_id;
+ u16 offset;
+};
+
+/*
+ * aead_edesc - s/w-extended aead descriptor
+ * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
+ * @link_tbl_bytes: length of dma mapped link_tbl space
+ * @link_tbl_dma: bus physical mapped address of h/w link table
+ * @hw_desc: the h/w job descriptor followed by any referenced link tables
+ */
+struct aead_edesc {
+ int assoc_nents;
+ int src_nents;
+ int dst_nents;
+ dma_addr_t iv_dma;
+ int link_tbl_bytes;
+ dma_addr_t link_tbl_dma;
+ struct link_tbl_entry *link_tbl;
+ u32 hw_desc[0];
+};
+
+/*
+ * ablkcipher_edesc - s/w-extended ablkcipher descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @iv_dma: dma address of iv for checking continuity and link table
+ * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
+ * @link_tbl_bytes: length of dma mapped link_tbl space
+ * @link_tbl_dma: bus physical mapped address of h/w link table
+ * @hw_desc: the h/w job descriptor followed by any referenced link tables
+ */
+struct ablkcipher_edesc {
+ int src_nents;
+ int dst_nents;
+ dma_addr_t iv_dma;
+ int link_tbl_bytes;
+ dma_addr_t link_tbl_dma;
+ struct link_tbl_entry *link_tbl;
+ u32 hw_desc[0];
+};
+
+static void caam_unmap(struct device *dev, struct scatterlist *src,
+ struct scatterlist *dst, int src_nents, int dst_nents,
+ dma_addr_t iv_dma, int ivsize, dma_addr_t link_tbl_dma,
+ int link_tbl_bytes)
+{
+ if (unlikely(dst != src)) {
+ dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
+ } else {
+ dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
+ }
+
+ if (iv_dma)
+ dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
+ if (link_tbl_bytes)
+ dma_unmap_single(dev, link_tbl_dma, link_tbl_bytes,
+ DMA_TO_DEVICE);
+}
+
+static void aead_unmap(struct device *dev,
+ struct aead_edesc *edesc,
+ struct aead_request *req)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ int ivsize = crypto_aead_ivsize(aead);
+
+ dma_unmap_sg(dev, req->assoc, edesc->assoc_nents, DMA_TO_DEVICE);
+
+ caam_unmap(dev, req->src, req->dst,
+ edesc->src_nents, edesc->dst_nents,
+ edesc->iv_dma, ivsize, edesc->link_tbl_dma,
+ edesc->link_tbl_bytes);
+}
+
+static void ablkcipher_unmap(struct device *dev,
+ struct ablkcipher_edesc *edesc,
+ struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+
+ caam_unmap(dev, req->src, req->dst,
+ edesc->src_nents, edesc->dst_nents,
+ edesc->iv_dma, ivsize, edesc->link_tbl_dma,
+ edesc->link_tbl_bytes);
+}
+
+static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
+ void *context)
+{
+ struct aead_request *req = context;
+ struct aead_edesc *edesc;
+#ifdef DEBUG
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ int ivsize = crypto_aead_ivsize(aead);
+
+ dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+
+ edesc = (struct aead_edesc *)((char *)desc -
+ offsetof(struct aead_edesc, hw_desc));
+
+ if (err) {
+ char tmp[CAAM_ERROR_STR_MAX];
+
+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+ }
+
+ aead_unmap(jrdev, edesc, req);
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
+ req->assoclen , 1);
+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
+ edesc->src_nents ? 100 : ivsize, 1);
+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->src_nents ? 100 : req->cryptlen +
+ ctx->authsize + 4, 1);
+#endif
+
+ kfree(edesc);
+
+ aead_request_complete(req, err);
+}
+
+static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
+ void *context)
+{
+ struct aead_request *req = context;
+ struct aead_edesc *edesc;
+#ifdef DEBUG
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ int ivsize = crypto_aead_ivsize(aead);
+
+ dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+
+ edesc = (struct aead_edesc *)((char *)desc -
+ offsetof(struct aead_edesc, hw_desc));
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+ ivsize, 1);
+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
+ req->cryptlen, 1);
+#endif
+
+ if (err) {
+ char tmp[CAAM_ERROR_STR_MAX];
+
+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+ }
+
+ aead_unmap(jrdev, edesc, req);
+
+ /*
+ * verify hw auth check passed else return -EBADMSG
+ */
+ if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
+ err = -EBADMSG;
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "iphdrout@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4,
+ ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
+ sizeof(struct iphdr) + req->assoclen +
+ ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
+ ctx->authsize + 36, 1);
+ if (!err && edesc->link_tbl_bytes) {
+ struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
+ print_hex_dump(KERN_ERR, "sglastout@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
+ sg->length + ctx->authsize + 16, 1);
+ }
+#endif
+
+ kfree(edesc);
+
+ aead_request_complete(req, err);
+}
+
+static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
+ void *context)
+{
+ struct ablkcipher_request *req = context;
+ struct ablkcipher_edesc *edesc;
+#ifdef DEBUG
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+
+ dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+
+ edesc = (struct ablkcipher_edesc *)((char *)desc -
+ offsetof(struct ablkcipher_edesc, hw_desc));
+
+ if (err) {
+ char tmp[CAAM_ERROR_STR_MAX];
+
+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+ }
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+ edesc->src_nents > 1 ? 100 : ivsize, 1);
+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+#endif
+
+ ablkcipher_unmap(jrdev, edesc, req);
+ kfree(edesc);
+
+ ablkcipher_request_complete(req, err);
+}
+
+static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
+ void *context)
+{
+ struct ablkcipher_request *req = context;
+ struct ablkcipher_edesc *edesc;
+#ifdef DEBUG
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+
+ dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
+#endif
+
+ edesc = (struct ablkcipher_edesc *)((char *)desc -
+ offsetof(struct ablkcipher_edesc, hw_desc));
+ if (err) {
+ char tmp[CAAM_ERROR_STR_MAX];
+
+ dev_err(jrdev, "%08x: %s\n", err, caam_jr_strstatus(tmp, err));
+ }
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "dstiv @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+ ivsize, 1);
+ print_hex_dump(KERN_ERR, "dst @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
+#endif
+
+ ablkcipher_unmap(jrdev, edesc, req);
+ kfree(edesc);
+
+ ablkcipher_request_complete(req, err);
+}
+
+static void sg_to_link_tbl_one(struct link_tbl_entry *link_tbl_ptr,
+ dma_addr_t dma, u32 len, u32 offset)
+{
+ link_tbl_ptr->ptr = dma;
+ link_tbl_ptr->len = len;
+ link_tbl_ptr->reserved = 0;
+ link_tbl_ptr->buf_pool_id = 0;
+ link_tbl_ptr->offset = offset;
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "link_tbl_ptr@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, link_tbl_ptr,
+ sizeof(struct link_tbl_entry), 1);
+#endif
+}
+
+/*
+ * convert scatterlist to h/w link table format
+ * but does not have final bit; instead, returns last entry
+ */
+static struct link_tbl_entry *sg_to_link_tbl(struct scatterlist *sg,
+ int sg_count, struct link_tbl_entry
+ *link_tbl_ptr, u32 offset)
+{
+ while (sg_count) {
+ sg_to_link_tbl_one(link_tbl_ptr, sg_dma_address(sg),
+ sg_dma_len(sg), offset);
+ link_tbl_ptr++;
+ sg = sg_next(sg);
+ sg_count--;
+ }
+ return link_tbl_ptr - 1;
+}
+
+/*
+ * convert scatterlist to h/w link table format
+ * scatterlist must have been previously dma mapped
+ */
+static void sg_to_link_tbl_last(struct scatterlist *sg, int sg_count,
+ struct link_tbl_entry *link_tbl_ptr, u32 offset)
+{
+ link_tbl_ptr = sg_to_link_tbl(sg, sg_count, link_tbl_ptr, offset);
+ link_tbl_ptr->len |= 0x40000000;
+}
+
+/*
+ * Fill in aead job descriptor
+ */
+static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
+ struct aead_edesc *edesc,
+ struct aead_request *req,
+ bool all_contig, bool encrypt)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ int ivsize = crypto_aead_ivsize(aead);
+ int authsize = ctx->authsize;
+ u32 *desc = edesc->hw_desc;
+ u32 out_options = 0, in_options;
+ dma_addr_t dst_dma, src_dma;
+ int len, link_tbl_index = 0;
+
+#ifdef DEBUG
+ debug("assoclen %d cryptlen %d authsize %d\n",
+ req->assoclen, req->cryptlen, authsize);
+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
+ req->assoclen , 1);
+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
+ edesc->src_nents ? 100 : ivsize, 1);
+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->src_nents ? 100 : req->cryptlen, 1);
+ print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
+ desc_bytes(sh_desc), 1);
+#endif
+
+ len = desc_len(sh_desc);
+ init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
+
+ if (all_contig) {
+ src_dma = sg_dma_address(req->assoc);
+ in_options = 0;
+ } else {
+ src_dma = edesc->link_tbl_dma;
+ link_tbl_index += (edesc->assoc_nents ? : 1) + 1 +
+ (edesc->src_nents ? : 1);
+ in_options = LDST_SGF;
+ }
+ if (encrypt)
+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
+ req->cryptlen - authsize, in_options);
+ else
+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
+ req->cryptlen, in_options);
+
+ if (likely(req->src == req->dst)) {
+ if (all_contig) {
+ dst_dma = sg_dma_address(req->src);
+ } else {
+ dst_dma = src_dma + sizeof(struct link_tbl_entry) *
+ ((edesc->assoc_nents ? : 1) + 1);
+ out_options = LDST_SGF;
+ }
+ } else {
+ if (!edesc->dst_nents) {
+ dst_dma = sg_dma_address(req->dst);
+ } else {
+ dst_dma = edesc->link_tbl_dma +
+ link_tbl_index *
+ sizeof(struct link_tbl_entry);
+ out_options = LDST_SGF;
+ }
+ }
+ if (encrypt)
+ append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options);
+ else
+ append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
+ out_options);
+}
+
+/*
+ * Fill in aead givencrypt job descriptor
+ */
+static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
+ struct aead_edesc *edesc,
+ struct aead_request *req,
+ int contig)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ int ivsize = crypto_aead_ivsize(aead);
+ int authsize = ctx->authsize;
+ u32 *desc = edesc->hw_desc;
+ u32 out_options = 0, in_options;
+ dma_addr_t dst_dma, src_dma;
+ int len, link_tbl_index = 0;
+
+#ifdef DEBUG
+ debug("assoclen %d cryptlen %d authsize %d\n",
+ req->assoclen, req->cryptlen, authsize);
+ print_hex_dump(KERN_ERR, "assoc @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
+ req->assoclen , 1);
+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
+ print_hex_dump(KERN_ERR, "shrdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
+ desc_bytes(sh_desc), 1);
+#endif
+
+ len = desc_len(sh_desc);
+ init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
+
+ if (contig & GIV_SRC_CONTIG) {
+ src_dma = sg_dma_address(req->assoc);
+ in_options = 0;
+ } else {
+ src_dma = edesc->link_tbl_dma;
+ link_tbl_index += edesc->assoc_nents + 1 + edesc->src_nents;
+ in_options = LDST_SGF;
+ }
+ append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize +
+ req->cryptlen - authsize, in_options);
+
+ if (contig & GIV_DST_CONTIG) {
+ dst_dma = edesc->iv_dma;
+ } else {
+ if (likely(req->src == req->dst)) {
+ dst_dma = src_dma + sizeof(struct link_tbl_entry) *
+ edesc->assoc_nents;
+ out_options = LDST_SGF;
+ } else {
+ dst_dma = edesc->link_tbl_dma +
+ link_tbl_index *
+ sizeof(struct link_tbl_entry);
+ out_options = LDST_SGF;
+ }
+ }
+
+ append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen, out_options);
+}
+
+/*
+ * Fill in ablkcipher job descriptor
+ */
+static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
+ struct ablkcipher_edesc *edesc,
+ struct ablkcipher_request *req,
+ bool iv_contig)
+{
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+ u32 *desc = edesc->hw_desc;
+ u32 out_options = 0, in_options;
+ dma_addr_t dst_dma, src_dma;
+ int len, link_tbl_index = 0;
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "presciv@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, req->info,
+ ivsize, 1);
+ print_hex_dump(KERN_ERR, "src @"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ edesc->src_nents ? 100 : req->nbytes, 1);
+#endif
+
+ len = desc_len(sh_desc);
+ init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
+
+ if (iv_contig) {
+ src_dma = edesc->iv_dma;
+ in_options = 0;
+ } else {
+ src_dma = edesc->link_tbl_dma;
+ link_tbl_index += (iv_contig ? 0 : 1) + edesc->src_nents;
+ in_options = LDST_SGF;
+ }
+ append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
+
+ if (likely(req->src == req->dst)) {
+ if (!edesc->src_nents && iv_contig) {
+ dst_dma = sg_dma_address(req->src);
+ } else {
+ dst_dma = edesc->link_tbl_dma +
+ sizeof(struct link_tbl_entry);
+ out_options = LDST_SGF;
+ }
+ } else {
+ if (!edesc->dst_nents) {
+ dst_dma = sg_dma_address(req->dst);
+ } else {
+ dst_dma = edesc->link_tbl_dma +
+ link_tbl_index * sizeof(struct link_tbl_entry);
+ out_options = LDST_SGF;
+ }
+ }
+ append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
+}
+
+/*
+ * derive number of elements in scatterlist
+ */
+static int sg_count(struct scatterlist *sg_list, int nbytes)
+{
+ struct scatterlist *sg = sg_list;
+ int sg_nents = 0;
+
+ while (nbytes > 0) {
+ sg_nents++;
+ nbytes -= sg->length;
+ if (!sg_is_last(sg) && (sg + 1)->length == 0)
+ BUG(); /* Not support chaining */
+ sg = scatterwalk_sg_next(sg);
+ }
+
+ if (likely(sg_nents == 1))
+ return 0;
+
+ return sg_nents;
+}
+
+/*
+ * allocate and map the aead extended descriptor
+ */
+static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
+ int desc_bytes, bool *all_contig_ptr)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+ CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+ int assoc_nents, src_nents, dst_nents = 0;
+ struct aead_edesc *edesc;
+ dma_addr_t iv_dma = 0;
+ int sgc;
+ bool all_contig = true;
+ int ivsize = crypto_aead_ivsize(aead);
+ int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
+
+ assoc_nents = sg_count(req->assoc, req->assoclen);
+ src_nents = sg_count(req->src, req->cryptlen);
+
+ if (unlikely(req->dst != req->src))
+ dst_nents = sg_count(req->dst, req->cryptlen);
+
+ sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
+ DMA_BIDIRECTIONAL);
+ if (likely(req->src == req->dst)) {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_BIDIRECTIONAL);
+ } else {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_TO_DEVICE);
+ sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
+ DMA_FROM_DEVICE);
+ }
+
+ /* Check if data are contiguous */
+ iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
+ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
+ iv_dma || src_nents || iv_dma + ivsize !=
+ sg_dma_address(req->src)) {
+ all_contig = false;
+ assoc_nents = assoc_nents ? : 1;
+ src_nents = src_nents ? : 1;
+ link_tbl_len = assoc_nents + 1 + src_nents;
+ }
+ link_tbl_len += dst_nents;
+
+ link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
+
+ /* allocate space for base edesc and hw desc commands, link tables */
+ edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
+ link_tbl_bytes, GFP_DMA | flags);
+ if (!edesc) {
+ dev_err(jrdev, "could not allocate extended descriptor\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ edesc->assoc_nents = assoc_nents;
+ edesc->src_nents = src_nents;
+ edesc->dst_nents = dst_nents;
+ edesc->iv_dma = iv_dma;
+ edesc->link_tbl_bytes = link_tbl_bytes;
+ edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
+ desc_bytes;
+ edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
+ link_tbl_bytes, DMA_TO_DEVICE);
+ *all_contig_ptr = all_contig;
+
+ link_tbl_index = 0;
+ if (!all_contig) {
+ sg_to_link_tbl(req->assoc,
+ (assoc_nents ? : 1),
+ edesc->link_tbl +
+ link_tbl_index, 0);
+ link_tbl_index += assoc_nents ? : 1;
+ sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
+ iv_dma, ivsize, 0);
+ link_tbl_index += 1;
+ sg_to_link_tbl_last(req->src,
+ (src_nents ? : 1),
+ edesc->link_tbl +
+ link_tbl_index, 0);
+ link_tbl_index += src_nents ? : 1;
+ }
+ if (dst_nents) {
+ sg_to_link_tbl_last(req->dst, dst_nents,
+ edesc->link_tbl + link_tbl_index, 0);
+ }
+
+ return edesc;
+}
+
+static int aead_encrypt(struct aead_request *req)
+{
+ struct aead_edesc *edesc;
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ bool all_contig;
+ u32 *desc;
+ int ret = 0;
+
+ req->cryptlen += ctx->authsize;
+
+ /* allocate extended descriptor */
+ edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
+ CAAM_CMD_SZ, &all_contig);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* Create and submit job descriptor */
+ init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
+ all_contig, true);
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+#endif
+
+ desc = edesc->hw_desc;
+ ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
+ if (!ret) {
+ ret = -EINPROGRESS;
+ } else {
+ aead_unmap(jrdev, edesc, req);
+ kfree(edesc);
+ }
+
+ return ret;
+}
+
+static int aead_decrypt(struct aead_request *req)
+{
+ struct aead_edesc *edesc;
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ bool all_contig;
+ u32 *desc;
+ int ret = 0;
+
+ /* allocate extended descriptor */
+ edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
+ CAAM_CMD_SZ, &all_contig);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "dec src@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ req->cryptlen, 1);
+#endif
+
+ /* Create and submit job descriptor*/
+ init_aead_job(ctx->sh_desc_dec,
+ ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+#endif
+
+ desc = edesc->hw_desc;
+ ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
+ if (!ret) {
+ ret = -EINPROGRESS;
+ } else {
+ aead_unmap(jrdev, edesc, req);
+ kfree(edesc);
+ }
+
+ return ret;
+}
+
+/*
+ * allocate and map the aead extended descriptor for aead givencrypt
+ */
+static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
+ *greq, int desc_bytes,
+ u32 *contig_ptr)
+{
+ struct aead_request *req = &greq->areq;
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+ CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
+ int assoc_nents, src_nents, dst_nents = 0;
+ struct aead_edesc *edesc;
+ dma_addr_t iv_dma = 0;
+ int sgc;
+ u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
+ int ivsize = crypto_aead_ivsize(aead);
+ int link_tbl_index, link_tbl_len = 0, link_tbl_bytes;
+
+ assoc_nents = sg_count(req->assoc, req->assoclen);
+ src_nents = sg_count(req->src, req->cryptlen);
+
+ if (unlikely(req->dst != req->src))
+ dst_nents = sg_count(req->dst, req->cryptlen);
+
+ sgc = dma_map_sg(jrdev, req->assoc, assoc_nents ? : 1,
+ DMA_BIDIRECTIONAL);
+ if (likely(req->src == req->dst)) {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_BIDIRECTIONAL);
+ } else {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_TO_DEVICE);
+ sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
+ DMA_FROM_DEVICE);
+ }
+
+ /* Check if data are contiguous */
+ iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
+ if (assoc_nents || sg_dma_address(req->assoc) + req->assoclen !=
+ iv_dma || src_nents || iv_dma + ivsize != sg_dma_address(req->src))
+ contig &= ~GIV_SRC_CONTIG;
+ if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
+ contig &= ~GIV_DST_CONTIG;
+ if (unlikely(req->src != req->dst)) {
+ dst_nents = dst_nents ? : 1;
+ link_tbl_len += 1;
+ }
+ if (!(contig & GIV_SRC_CONTIG)) {
+ assoc_nents = assoc_nents ? : 1;
+ src_nents = src_nents ? : 1;
+ link_tbl_len += assoc_nents + 1 + src_nents;
+ if (likely(req->src == req->dst))
+ contig &= ~GIV_DST_CONTIG;
+ }
+ link_tbl_len += dst_nents;
+
+ link_tbl_bytes = link_tbl_len * sizeof(struct link_tbl_entry);
+
+ /* allocate space for base edesc and hw desc commands, link tables */
+ edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
+ link_tbl_bytes, GFP_DMA | flags);
+ if (!edesc) {
+ dev_err(jrdev, "could not allocate extended descriptor\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ edesc->assoc_nents = assoc_nents;
+ edesc->src_nents = src_nents;
+ edesc->dst_nents = dst_nents;
+ edesc->iv_dma = iv_dma;
+ edesc->link_tbl_bytes = link_tbl_bytes;
+ edesc->link_tbl = (void *)edesc + sizeof(struct aead_edesc) +
+ desc_bytes;
+ edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
+ link_tbl_bytes, DMA_TO_DEVICE);
+ *contig_ptr = contig;
+
+ link_tbl_index = 0;
+ if (!(contig & GIV_SRC_CONTIG)) {
+ sg_to_link_tbl(req->assoc, assoc_nents,
+ edesc->link_tbl +
+ link_tbl_index, 0);
+ link_tbl_index += assoc_nents;
+ sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
+ iv_dma, ivsize, 0);
+ link_tbl_index += 1;
+ sg_to_link_tbl_last(req->src, src_nents,
+ edesc->link_tbl +
+ link_tbl_index, 0);
+ link_tbl_index += src_nents;
+ }
+ if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
+ sg_to_link_tbl_one(edesc->link_tbl + link_tbl_index,
+ iv_dma, ivsize, 0);
+ link_tbl_index += 1;
+ sg_to_link_tbl_last(req->dst, dst_nents,
+ edesc->link_tbl + link_tbl_index, 0);
+ }
+
+ return edesc;
+}
+
+static int aead_givencrypt(struct aead_givcrypt_request *areq)
+{
+ struct aead_request *req = &areq->areq;
+ struct aead_edesc *edesc;
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct caam_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *jrdev = ctx->jrdev;
+ u32 contig;
+ u32 *desc;
+ int ret = 0;
+
+ req->cryptlen += ctx->authsize;
+
+ /* allocate extended descriptor */
+ edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
+ CAAM_CMD_SZ, &contig);
+
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "giv src@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
+ req->cryptlen, 1);
+#endif
+
+ /* Create and submit job descriptor*/
+ init_aead_giv_job(ctx->sh_desc_givenc,
+ ctx->sh_desc_givenc_dma, edesc, req, contig);
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "aead jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+#endif
+
+ desc = edesc->hw_desc;
+ ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
+ if (!ret) {
+ ret = -EINPROGRESS;
+ } else {
+ aead_unmap(jrdev, edesc, req);
+ kfree(edesc);
+ }
+
+ return ret;
+}
+
+/*
+ * allocate and map the ablkcipher extended descriptor for ablkcipher
+ */
+static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
+ *req, int desc_bytes,
+ bool *iv_contig_out)
+{
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+ struct device *jrdev = ctx->jrdev;
+ gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
+ CRYPTO_TFM_REQ_MAY_SLEEP)) ?
+ GFP_KERNEL : GFP_ATOMIC;
+ int src_nents, dst_nents = 0, link_tbl_bytes;
+ struct ablkcipher_edesc *edesc;
+ dma_addr_t iv_dma = 0;
+ bool iv_contig = false;
+ int sgc;
+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
+ int link_tbl_index;
+
+ src_nents = sg_count(req->src, req->nbytes);
+
+ if (unlikely(req->dst != req->src))
+ dst_nents = sg_count(req->dst, req->nbytes);
+
+ if (likely(req->src == req->dst)) {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_BIDIRECTIONAL);
+ } else {
+ sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
+ DMA_TO_DEVICE);
+ sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
+ DMA_FROM_DEVICE);
+ }
+
+ /*
+ * Check if iv can be contiguous with source and destination.
+ * If so, include it. If not, create scatterlist.
+ */
+ iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
+ if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
+ iv_contig = true;
+ else
+ src_nents = src_nents ? : 1;
+ link_tbl_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
+ sizeof(struct link_tbl_entry);
+
+ /* allocate space for base edesc and hw desc commands, link tables */
+ edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
+ link_tbl_bytes, GFP_DMA | flags);
+ if (!edesc) {
+ dev_err(jrdev, "could not allocate extended descriptor\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ edesc->src_nents = src_nents;
+ edesc->dst_nents = dst_nents;
+ edesc->link_tbl_bytes = link_tbl_bytes;
+ edesc->link_tbl = (void *)edesc + sizeof(struct ablkcipher_edesc) +
+ desc_bytes;
+
+ link_tbl_index = 0;
+ if (!iv_contig) {
+ sg_to_link_tbl_one(edesc->link_tbl, iv_dma, ivsize, 0);
+ sg_to_link_tbl_last(req->src, src_nents,
+ edesc->link_tbl + 1, 0);
+ link_tbl_index += 1 + src_nents;
+ }
+
+ if (unlikely(dst_nents)) {
+ sg_to_link_tbl_last(req->dst, dst_nents,
+ edesc->link_tbl + link_tbl_index, 0);
+ }
+
+ edesc->link_tbl_dma = dma_map_single(jrdev, edesc->link_tbl,
+ link_tbl_bytes, DMA_TO_DEVICE);
+ edesc->iv_dma = iv_dma;
+
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ablkcipher link_tbl@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->link_tbl,
+ link_tbl_bytes, 1);
+#endif
+
+ *iv_contig_out = iv_contig;
+ return edesc;
+}
+
+static int ablkcipher_encrypt(struct ablkcipher_request *req)
+{
+ struct ablkcipher_edesc *edesc;
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+ struct device *jrdev = ctx->jrdev;
+ bool iv_contig;
+ u32 *desc;
+ int ret = 0;
+
+ /* allocate extended descriptor */
+ edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
+ CAAM_CMD_SZ, &iv_contig);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* Create and submit job descriptor*/
+ init_ablkcipher_job(ctx->sh_desc_enc,
+ ctx->sh_desc_enc_dma, edesc, req, iv_contig);
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+#endif
+ desc = edesc->hw_desc;
+ ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
+
+ if (!ret) {
+ ret = -EINPROGRESS;
+ } else {
+ ablkcipher_unmap(jrdev, edesc, req);
+ kfree(edesc);
+ }
+
+ return ret;
+}
+
+static int ablkcipher_decrypt(struct ablkcipher_request *req)
+{
+ struct ablkcipher_edesc *edesc;
+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
+ struct device *jrdev = ctx->jrdev;
+ bool iv_contig;
+ u32 *desc;
+ int ret = 0;
+
+ /* allocate extended descriptor */
+ edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
+ CAAM_CMD_SZ, &iv_contig);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* Create and submit job descriptor*/
+ init_ablkcipher_job(ctx->sh_desc_dec,
+ ctx->sh_desc_dec_dma, edesc, req, iv_contig);
+ desc = edesc->hw_desc;
+#ifdef DEBUG
+ print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"xstr(__LINE__)": ",
+ DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
+ desc_bytes(edesc->hw_desc), 1);
+#endif
+
+ ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
+ if (!ret) {
+ ret = -EINPROGRESS;
+ } else {
+ ablkcipher_unmap(jrdev, edesc, req);
+ kfree(edesc);
+ }
+
+ return ret;
+}
+
+#define template_aead template_u.aead
+#define template_ablkcipher template_u.ablkcipher
+struct caam_alg_template {
+ char name[CRYPTO_MAX_ALG_NAME];
+ char driver_name[CRYPTO_MAX_ALG_NAME];
+ unsigned int blocksize;
+ u32 type;
+ union {
+ struct ablkcipher_alg ablkcipher;
+ struct aead_alg aead;
+ struct blkcipher_alg blkcipher;
+ struct cipher_alg cipher;
+ struct compress_alg compress;
+ struct rng_alg rng;
+ } template_u;
+ u32 class1_alg_type;
+ u32 class2_alg_type;
+ u32 alg_op;
+};
+
+static struct caam_alg_template driver_algs[] = {
+ /* single-pass ipsec_esp descriptor */
+ {
+ .name = "authenc(hmac(md5),cbc(aes))",
+ .driver_name = "authenc-hmac-md5-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha1),cbc(aes))",
+ .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha224),cbc(aes))",
+ .driver_name = "authenc-hmac-sha224-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA224_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha256),cbc(aes))",
+ .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha384),cbc(aes))",
+ .driver_name = "authenc-hmac-sha384-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA384_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
+ },
+
+ {
+ .name = "authenc(hmac(sha512),cbc(aes))",
+ .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA512_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(md5),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha1),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha224),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA224_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha256),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha384),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA384_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha512),cbc(des3_ede))",
+ .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA512_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(md5),cbc(des))",
+ .driver_name = "authenc-hmac-md5-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha1),cbc(des))",
+ .driver_name = "authenc-hmac-sha1-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha224),cbc(des))",
+ .driver_name = "authenc-hmac-sha224-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA224_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha256),cbc(des))",
+ .driver_name = "authenc-hmac-sha256-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha384),cbc(des))",
+ .driver_name = "authenc-hmac-sha384-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA384_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
+ },
+ {
+ .name = "authenc(hmac(sha512),cbc(des))",
+ .driver_name = "authenc-hmac-sha512-cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_AEAD,
+ .template_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA512_DIGEST_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
+ OP_ALG_AAI_HMAC_PRECOMP,
+ .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
+ },
+ /* ablkcipher descriptor */
+ {
+ .name = "cbc(aes)",
+ .driver_name = "cbc-aes-caam",
+ .blocksize = AES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .template_ablkcipher = {
+ .setkey = ablkcipher_setkey,
+ .encrypt = ablkcipher_encrypt,
+ .decrypt = ablkcipher_decrypt,
+ .geniv = "eseqiv",
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
+ },
+ {
+ .name = "cbc(des3_ede)",
+ .driver_name = "cbc-3des-caam",
+ .blocksize = DES3_EDE_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .template_ablkcipher = {
+ .setkey = ablkcipher_setkey,
+ .encrypt = ablkcipher_encrypt,
+ .decrypt = ablkcipher_decrypt,
+ .geniv = "eseqiv",
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
+ },
+ {
+ .name = "cbc(des)",
+ .driver_name = "cbc-des-caam",
+ .blocksize = DES_BLOCK_SIZE,
+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .template_ablkcipher = {
+ .setkey = ablkcipher_setkey,
+ .encrypt = ablkcipher_encrypt,
+ .decrypt = ablkcipher_decrypt,
+ .geniv = "eseqiv",
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ },
+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
+ }
+};
+
+struct caam_crypto_alg {
+ struct list_head entry;
+ struct device *ctrldev;
+ int class1_alg_type;
+ int class2_alg_type;
+ int alg_op;
+ struct crypto_alg crypto_alg;
+};
+
+static int caam_cra_init(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct caam_crypto_alg *caam_alg =
+ container_of(alg, struct caam_crypto_alg, crypto_alg);
+ struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct caam_drv_private *priv = dev_get_drvdata(caam_alg->ctrldev);
+ int tgt_jr = atomic_inc_return(&priv->tfm_count);
+
+ /*
+ * distribute tfms across job rings to ensure in-order
+ * crypto request processing per tfm
+ */
+ ctx->jrdev = priv->algapi_jr[(tgt_jr / 2) % priv->num_jrs_for_algapi];
+
+ /* copy descriptor header template value */
+ ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
+ ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
+ ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
+
+ return 0;
+}
+
+static void caam_cra_exit(struct crypto_tfm *tfm)
+{
+ struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (ctx->sh_desc_enc_dma &&
+ !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
+ dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
+ desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
+ if (ctx->sh_desc_dec_dma &&
+ !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
+ dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
+ desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
+ if (ctx->sh_desc_givenc_dma &&
+ !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
+ dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
+ desc_bytes(ctx->sh_desc_givenc),
+ DMA_TO_DEVICE);
+}
+
+static void __exit caam_algapi_exit(void)
+{
+
+ struct device_node *dev_node;
+ struct platform_device *pdev;
+ struct device *ctrldev;
+ struct caam_drv_private *priv;
+ struct caam_crypto_alg *t_alg, *n;
+ int i, err;
+
+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
+ if (!dev_node)
+ return;
+
+ pdev = of_find_device_by_node(dev_node);
+ if (!pdev)
+ return;
+
+ ctrldev = &pdev->dev;
+ of_node_put(dev_node);
+ priv = dev_get_drvdata(ctrldev);
+
+ if (!priv->alg_list.next)
+ return;
+
+ list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
+ crypto_unregister_alg(&t_alg->crypto_alg);
+ list_del(&t_alg->entry);
+ kfree(t_alg);
+ }
+
+ for (i = 0; i < priv->total_jobrs; i++) {
+ err = caam_jr_deregister(priv->algapi_jr[i]);
+ if (err < 0)
+ break;
+ }
+ kfree(priv->algapi_jr);
+}
+
+static struct caam_crypto_alg *caam_alg_alloc(struct device *ctrldev,
+ struct caam_alg_template
+ *template)
+{
+ struct caam_crypto_alg *t_alg;
+ struct crypto_alg *alg;
+
+ t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
+ if (!t_alg) {
+ dev_err(ctrldev, "failed to allocate t_alg\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ alg = &t_alg->crypto_alg;
+
+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
+ template->driver_name);
+ alg->cra_module = THIS_MODULE;
+ alg->cra_init = caam_cra_init;
+ alg->cra_exit = caam_cra_exit;
+ alg->cra_priority = CAAM_CRA_PRIORITY;
+ alg->cra_blocksize = template->blocksize;
+ alg->cra_alignmask = 0;
+ alg->cra_ctxsize = sizeof(struct caam_ctx);
+ alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
+ template->type;
+ switch (template->type) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ alg->cra_type = &crypto_ablkcipher_type;
+ alg->cra_ablkcipher = template->template_ablkcipher;
+ break;
+ case CRYPTO_ALG_TYPE_AEAD:
+ alg->cra_type = &crypto_aead_type;
+ alg->cra_aead = template->template_aead;
+ break;
+ }
+
+ t_alg->class1_alg_type = template->class1_alg_type;
+ t_alg->class2_alg_type = template->class2_alg_type;
+ t_alg->alg_op = template->alg_op;
+ t_alg->ctrldev = ctrldev;
+
+ return t_alg;
+}
+
+static int __init caam_algapi_init(void)
+{
+ struct device_node *dev_node;
+ struct platform_device *pdev;
+ struct device *ctrldev, **jrdev;
+ struct caam_drv_private *priv;
+ int i = 0, err = 0;
+
+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
+ if (!dev_node)
+ return -ENODEV;
+
+ pdev = of_find_device_by_node(dev_node);
+ if (!pdev)
+ return -ENODEV;
+
+ ctrldev = &pdev->dev;
+ priv = dev_get_drvdata(ctrldev);
+ of_node_put(dev_node);
+
+ INIT_LIST_HEAD(&priv->alg_list);
+
+ jrdev = kmalloc(sizeof(*jrdev) * priv->total_jobrs, GFP_KERNEL);
+ if (!jrdev)
+ return -ENOMEM;
+
+ for (i = 0; i < priv->total_jobrs; i++) {
+ err = caam_jr_register(ctrldev, &jrdev[i]);
+ if (err < 0)
+ break;
+ }
+ if (err < 0 && i == 0) {
+ dev_err(ctrldev, "algapi error in job ring registration: %d\n",
+ err);
+ kfree(jrdev);
+ return err;
+ }
+
+ priv->num_jrs_for_algapi = i;
+ priv->algapi_jr = jrdev;
+ atomic_set(&priv->tfm_count, -1);
+
+ /* register crypto algorithms the device supports */
+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+ /* TODO: check if h/w supports alg */
+ struct caam_crypto_alg *t_alg;
+
+ t_alg = caam_alg_alloc(ctrldev, &driver_algs[i]);
+ if (IS_ERR(t_alg)) {
+ err = PTR_ERR(t_alg);
+ dev_warn(ctrldev, "%s alg allocation failed\n",
+ driver_algs[i].driver_name);
+ continue;
+ }
+
+ err = crypto_register_alg(&t_alg->crypto_alg);
+ if (err) {
+ dev_warn(ctrldev, "%s alg registration failed\n",
+ t_alg->crypto_alg.cra_driver_name);
+ kfree(t_alg);
+ } else
+ list_add_tail(&t_alg->entry, &priv->alg_list);
+ }
+ if (!list_empty(&priv->alg_list))
+ dev_info(ctrldev, "%s algorithms registered in /proc/crypto\n",
+ (char *)of_get_property(dev_node, "compatible", NULL));
+
+ return err;
+}
+
+module_init(caam_algapi_init);
+module_exit(caam_algapi_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("FSL CAAM support for crypto API");
+MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/compat.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/compat.h
new file mode 100644
index 0000000..a63bc65
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/compat.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef CAAM_COMPAT_H
+#define CAAM_COMPAT_H
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/of_platform.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/rtnetlink.h>
+#include <linux/in.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/debugfs.h>
+#include <linux/circ_buf.h>
+#include <net/xfrm.h>
+
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/internal/skcipher.h>
+
+#endif /* !defined(CAAM_COMPAT_H) */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/ctrl.c b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/ctrl.c
new file mode 100644
index 0000000..c5f61c5
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/ctrl.c
@@ -0,0 +1,247 @@
+/*
+ * CAAM control-plane driver backend
+ * Controller-level driver, kernel property detection, initialization
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#include "compat.h"
+#include "regs.h"
+#include "intern.h"
+#include "jr.h"
+
+static int caam_remove(struct platform_device *pdev)
+{
+ struct device *ctrldev;
+ struct caam_drv_private *ctrlpriv;
+ struct caam_drv_private_jr *jrpriv;
+ struct caam_full __iomem *topregs;
+ int ring, ret = 0;
+
+ ctrldev = &pdev->dev;
+ ctrlpriv = dev_get_drvdata(ctrldev);
+ topregs = (struct caam_full __iomem *)ctrlpriv->ctrl;
+
+ /* shut down JobRs */
+ for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
+ ret |= caam_jr_shutdown(ctrlpriv->jrdev[ring]);
+ jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]);
+ irq_dispose_mapping(jrpriv->irq);
+ }
+
+ /* Shut down debug views */
+#ifdef CONFIG_DEBUG_FS
+ debugfs_remove_recursive(ctrlpriv->dfs_root);
+#endif
+
+ /* Unmap controller region */
+ iounmap(&topregs->ctrl);
+
+ kfree(ctrlpriv->jrdev);
+ kfree(ctrlpriv);
+
+ return ret;
+}
+
+/* Probe routine for CAAM top (controller) level */
+static int caam_probe(struct platform_device *pdev)
+{
+ int ring, rspec;
+ struct device *dev;
+ struct device_node *nprop, *np;
+ struct caam_ctrl __iomem *ctrl;
+ struct caam_full __iomem *topregs;
+ struct caam_drv_private *ctrlpriv;
+#ifdef CONFIG_DEBUG_FS
+ struct caam_perfmon *perfmon;
+#endif
+
+ ctrlpriv = kzalloc(sizeof(struct caam_drv_private), GFP_KERNEL);
+ if (!ctrlpriv)
+ return -ENOMEM;
+
+ dev = &pdev->dev;
+ dev_set_drvdata(dev, ctrlpriv);
+ ctrlpriv->pdev = pdev;
+ nprop = pdev->dev.of_node;
+
+ /* Get configuration properties from device tree */
+ /* First, get register page */
+ ctrl = of_iomap(nprop, 0);
+ if (ctrl == NULL) {
+ dev_err(dev, "caam: of_iomap() failed\n");
+ return -ENOMEM;
+ }
+ ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl;
+
+ /* topregs used to derive pointers to CAAM sub-blocks only */
+ topregs = (struct caam_full __iomem *)ctrl;
+
+ /* Get the IRQ of the controller (for security violations only) */
+ ctrlpriv->secvio_irq = of_irq_to_resource(nprop, 0, NULL);
+
+ /*
+ * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel,
+ * 36-bit pointers in master configuration register
+ */
+ setbits32(&topregs->ctrl.mcr, MCFGR_WDENABLE |
+ (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0));
+
+ if (sizeof(dma_addr_t) == sizeof(u64))
+ dma_set_mask(dev, DMA_BIT_MASK(36));
+
+ /*
+ * Detect and enable JobRs
+ * First, find out how many ring spec'ed, allocate references
+ * for all, then go probe each one.
+ */
+ rspec = 0;
+ for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring")
+ rspec++;
+ ctrlpriv->jrdev = kzalloc(sizeof(struct device *) * rspec, GFP_KERNEL);
+ if (ctrlpriv->jrdev == NULL) {
+ iounmap(&topregs->ctrl);
+ return -ENOMEM;
+ }
+
+ ring = 0;
+ ctrlpriv->total_jobrs = 0;
+ for_each_compatible_node(np, NULL, "fsl,sec-v4.0-job-ring") {
+ caam_jr_probe(pdev, np, ring);
+ ctrlpriv->total_jobrs++;
+ ring++;
+ }
+
+ /* Check to see if QI present. If so, enable */
+ ctrlpriv->qi_present = !!(rd_reg64(&topregs->ctrl.perfmon.comp_parms) &
+ CTPR_QI_MASK);
+ if (ctrlpriv->qi_present) {
+ ctrlpriv->qi = (struct caam_queue_if __force *)&topregs->qi;
+ /* This is all that's required to physically enable QI */
+ wr_reg32(&topregs->qi.qi_control_lo, QICTL_DQEN);
+ }
+
+ /* If no QI and no rings specified, quit and go home */
+ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) {
+ dev_err(dev, "no queues configured, terminating\n");
+ caam_remove(pdev);
+ return -ENOMEM;
+ }
+
+ /* NOTE: RTIC detection ought to go here, around Si time */
+
+ /* Initialize queue allocator lock */
+ spin_lock_init(&ctrlpriv->jr_alloc_lock);
+
+ /* Report "alive" for developer to see */
+ dev_info(dev, "device ID = 0x%016llx\n",
+ rd_reg64(&topregs->ctrl.perfmon.caam_id));
+ dev_info(dev, "job rings = %d, qi = %d\n",
+ ctrlpriv->total_jobrs, ctrlpriv->qi_present);
+
+#ifdef CONFIG_DEBUG_FS
+ /*
+ * FIXME: needs better naming distinction, as some amalgamation of
+ * "caam" and nprop->full_name. The OF name isn't distinctive,
+ * but does separate instances
+ */
+ perfmon = (struct caam_perfmon __force *)&ctrl->perfmon;
+
+ ctrlpriv->dfs_root = debugfs_create_dir("caam", NULL);
+ ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root);
+
+ /* Controller-level - performance monitor counters */
+ ctrlpriv->ctl_rq_dequeued =
+ debugfs_create_u64("rq_dequeued",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->req_dequeued);
+ ctrlpriv->ctl_ob_enc_req =
+ debugfs_create_u64("ob_rq_encrypted",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ob_enc_req);
+ ctrlpriv->ctl_ib_dec_req =
+ debugfs_create_u64("ib_rq_decrypted",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ib_dec_req);
+ ctrlpriv->ctl_ob_enc_bytes =
+ debugfs_create_u64("ob_bytes_encrypted",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ob_enc_bytes);
+ ctrlpriv->ctl_ob_prot_bytes =
+ debugfs_create_u64("ob_bytes_protected",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ob_prot_bytes);
+ ctrlpriv->ctl_ib_dec_bytes =
+ debugfs_create_u64("ib_bytes_decrypted",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ib_dec_bytes);
+ ctrlpriv->ctl_ib_valid_bytes =
+ debugfs_create_u64("ib_bytes_validated",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->ib_valid_bytes);
+
+ /* Controller level - global status values */
+ ctrlpriv->ctl_faultaddr =
+ debugfs_create_u64("fault_addr",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->faultaddr);
+ ctrlpriv->ctl_faultdetail =
+ debugfs_create_u32("fault_detail",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->faultdetail);
+ ctrlpriv->ctl_faultstatus =
+ debugfs_create_u32("fault_status",
+ S_IRUSR | S_IRGRP | S_IROTH,
+ ctrlpriv->ctl, &perfmon->status);
+
+ /* Internal covering keys (useful in non-secure mode only) */
+ ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0];
+ ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ ctrlpriv->ctl_kek = debugfs_create_blob("kek",
+ S_IRUSR |
+ S_IRGRP | S_IROTH,
+ ctrlpriv->ctl,
+ &ctrlpriv->ctl_kek_wrap);
+
+ ctrlpriv->ctl_tkek_wrap.data = &ctrlpriv->ctrl->tkek[0];
+ ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ ctrlpriv->ctl_tkek = debugfs_create_blob("tkek",
+ S_IRUSR |
+ S_IRGRP | S_IROTH,
+ ctrlpriv->ctl,
+ &ctrlpriv->ctl_tkek_wrap);
+
+ ctrlpriv->ctl_tdsk_wrap.data = &ctrlpriv->ctrl->tdsk[0];
+ ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32);
+ ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk",
+ S_IRUSR |
+ S_IRGRP | S_IROTH,
+ ctrlpriv->ctl,
+ &ctrlpriv->ctl_tdsk_wrap);
+#endif
+ return 0;
+}
+
+static struct of_device_id caam_match[] = {
+ {
+ .compatible = "fsl,sec-v4.0",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, caam_match);
+
+static struct platform_driver caam_driver = {
+ .driver = {
+ .name = "caam",
+ .owner = THIS_MODULE,
+ .of_match_table = caam_match,
+ },
+ .probe = caam_probe,
+ .remove = __devexit_p(caam_remove),
+};
+
+module_platform_driver(caam_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("FSL CAAM request backend");
+MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc.h
new file mode 100644
index 0000000..a17c295
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc.h
@@ -0,0 +1,1604 @@
+/*
+ * CAAM descriptor composition header
+ * Definitions to support CAAM descriptor instruction generation
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef DESC_H
+#define DESC_H
+
+/* Max size of any CAAM descriptor in 32-bit words, inclusive of header */
+#define MAX_CAAM_DESCSIZE 64
+
+/* Block size of any entity covered/uncovered with a KEK/TKEK */
+#define KEK_BLOCKSIZE 16
+
+/*
+ * Supported descriptor command types as they show up
+ * inside a descriptor command word.
+ */
+#define CMD_SHIFT 27
+#define CMD_MASK 0xf8000000
+
+#define CMD_KEY (0x00 << CMD_SHIFT)
+#define CMD_SEQ_KEY (0x01 << CMD_SHIFT)
+#define CMD_LOAD (0x02 << CMD_SHIFT)
+#define CMD_SEQ_LOAD (0x03 << CMD_SHIFT)
+#define CMD_FIFO_LOAD (0x04 << CMD_SHIFT)
+#define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT)
+#define CMD_STORE (0x0a << CMD_SHIFT)
+#define CMD_SEQ_STORE (0x0b << CMD_SHIFT)
+#define CMD_FIFO_STORE (0x0c << CMD_SHIFT)
+#define CMD_SEQ_FIFO_STORE (0x0d << CMD_SHIFT)
+#define CMD_MOVE_LEN (0x0e << CMD_SHIFT)
+#define CMD_MOVE (0x0f << CMD_SHIFT)
+#define CMD_OPERATION (0x10 << CMD_SHIFT)
+#define CMD_SIGNATURE (0x12 << CMD_SHIFT)
+#define CMD_JUMP (0x14 << CMD_SHIFT)
+#define CMD_MATH (0x15 << CMD_SHIFT)
+#define CMD_DESC_HDR (0x16 << CMD_SHIFT)
+#define CMD_SHARED_DESC_HDR (0x17 << CMD_SHIFT)
+#define CMD_SEQ_IN_PTR (0x1e << CMD_SHIFT)
+#define CMD_SEQ_OUT_PTR (0x1f << CMD_SHIFT)
+
+/* General-purpose class selector for all commands */
+#define CLASS_SHIFT 25
+#define CLASS_MASK (0x03 << CLASS_SHIFT)
+
+#define CLASS_NONE (0x00 << CLASS_SHIFT)
+#define CLASS_1 (0x01 << CLASS_SHIFT)
+#define CLASS_2 (0x02 << CLASS_SHIFT)
+#define CLASS_BOTH (0x03 << CLASS_SHIFT)
+
+/*
+ * Descriptor header command constructs
+ * Covers shared, job, and trusted descriptor headers
+ */
+
+/*
+ * Do Not Run - marks a descriptor inexecutable if there was
+ * a preceding error somewhere
+ */
+#define HDR_DNR 0x01000000
+
+/*
+ * ONE - should always be set. Combination of ONE (always
+ * set) and ZRO (always clear) forms an endianness sanity check
+ */
+#define HDR_ONE 0x00800000
+#define HDR_ZRO 0x00008000
+
+/* Start Index or SharedDesc Length */
+#define HDR_START_IDX_MASK 0x3f
+#define HDR_START_IDX_SHIFT 16
+
+/* If shared descriptor header, 6-bit length */
+#define HDR_DESCLEN_SHR_MASK 0x3f
+
+/* If non-shared header, 7-bit length */
+#define HDR_DESCLEN_MASK 0x7f
+
+/* This is a TrustedDesc (if not SharedDesc) */
+#define HDR_TRUSTED 0x00004000
+
+/* Make into TrustedDesc (if not SharedDesc) */
+#define HDR_MAKE_TRUSTED 0x00002000
+
+/* Save context if self-shared (if SharedDesc) */
+#define HDR_SAVECTX 0x00001000
+
+/* Next item points to SharedDesc */
+#define HDR_SHARED 0x00001000
+
+/*
+ * Reverse Execution Order - execute JobDesc first, then
+ * execute SharedDesc (normally SharedDesc goes first).
+ */
+#define HDR_REVERSE 0x00000800
+
+/* Propogate DNR property to SharedDesc */
+#define HDR_PROP_DNR 0x00000800
+
+/* JobDesc/SharedDesc share property */
+#define HDR_SD_SHARE_MASK 0x03
+#define HDR_SD_SHARE_SHIFT 8
+#define HDR_JD_SHARE_MASK 0x07
+#define HDR_JD_SHARE_SHIFT 8
+
+#define HDR_SHARE_NEVER (0x00 << HDR_SD_SHARE_SHIFT)
+#define HDR_SHARE_WAIT (0x01 << HDR_SD_SHARE_SHIFT)
+#define HDR_SHARE_SERIAL (0x02 << HDR_SD_SHARE_SHIFT)
+#define HDR_SHARE_ALWAYS (0x03 << HDR_SD_SHARE_SHIFT)
+#define HDR_SHARE_DEFER (0x04 << HDR_SD_SHARE_SHIFT)
+
+/* JobDesc/SharedDesc descriptor length */
+#define HDR_JD_LENGTH_MASK 0x7f
+#define HDR_SD_LENGTH_MASK 0x3f
+
+/*
+ * KEY/SEQ_KEY Command Constructs
+ */
+
+/* Key Destination Class: 01 = Class 1, 02 - Class 2 */
+#define KEY_DEST_CLASS_SHIFT 25 /* use CLASS_1 or CLASS_2 */
+#define KEY_DEST_CLASS_MASK (0x03 << KEY_DEST_CLASS_SHIFT)
+
+/* Scatter-Gather Table/Variable Length Field */
+#define KEY_SGF 0x01000000
+#define KEY_VLF 0x01000000
+
+/* Immediate - Key follows command in the descriptor */
+#define KEY_IMM 0x00800000
+
+/*
+ * Encrypted - Key is encrypted either with the KEK, or
+ * with the TDKEK if TK is set
+ */
+#define KEY_ENC 0x00400000
+
+/*
+ * No Write Back - Do not allow key to be FIFO STOREd
+ */
+#define KEY_NWB 0x00200000
+
+/*
+ * Enhanced Encryption of Key
+ */
+#define KEY_EKT 0x00100000
+
+/*
+ * Encrypted with Trusted Key
+ */
+#define KEY_TK 0x00008000
+
+/*
+ * KDEST - Key Destination: 0 - class key register,
+ * 1 - PKHA 'e', 2 - AFHA Sbox, 3 - MDHA split-key
+ */
+#define KEY_DEST_SHIFT 16
+#define KEY_DEST_MASK (0x03 << KEY_DEST_SHIFT)
+
+#define KEY_DEST_CLASS_REG (0x00 << KEY_DEST_SHIFT)
+#define KEY_DEST_PKHA_E (0x01 << KEY_DEST_SHIFT)
+#define KEY_DEST_AFHA_SBOX (0x02 << KEY_DEST_SHIFT)
+#define KEY_DEST_MDHA_SPLIT (0x03 << KEY_DEST_SHIFT)
+
+/* Length in bytes */
+#define KEY_LENGTH_MASK 0x000003ff
+
+/*
+ * LOAD/SEQ_LOAD/STORE/SEQ_STORE Command Constructs
+ */
+
+/*
+ * Load/Store Destination: 0 = class independent CCB,
+ * 1 = class 1 CCB, 2 = class 2 CCB, 3 = DECO
+ */
+#define LDST_CLASS_SHIFT 25
+#define LDST_CLASS_MASK (0x03 << LDST_CLASS_SHIFT)
+#define LDST_CLASS_IND_CCB (0x00 << LDST_CLASS_SHIFT)
+#define LDST_CLASS_1_CCB (0x01 << LDST_CLASS_SHIFT)
+#define LDST_CLASS_2_CCB (0x02 << LDST_CLASS_SHIFT)
+#define LDST_CLASS_DECO (0x03 << LDST_CLASS_SHIFT)
+
+/* Scatter-Gather Table/Variable Length Field */
+#define LDST_SGF 0x01000000
+#define LDST_VLF LDST_SGF
+
+/* Immediate - Key follows this command in descriptor */
+#define LDST_IMM_MASK 1
+#define LDST_IMM_SHIFT 23
+#define LDST_IMM (LDST_IMM_MASK << LDST_IMM_SHIFT)
+
+/* SRC/DST - Destination for LOAD, Source for STORE */
+#define LDST_SRCDST_SHIFT 16
+#define LDST_SRCDST_MASK (0x7f << LDST_SRCDST_SHIFT)
+
+#define LDST_SRCDST_BYTE_CONTEXT (0x20 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_BYTE_KEY (0x40 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_BYTE_INFIFO (0x7c << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_BYTE_OUTFIFO (0x7e << LDST_SRCDST_SHIFT)
+
+#define LDST_SRCDST_WORD_MODE_REG (0x00 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_KEYSZ_REG (0x01 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DATASZ_REG (0x02 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_ICVSZ_REG (0x03 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_CHACTRL (0x06 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECOCTRL (0x06 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_IRQCTRL (0x07 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_PCLOVRD (0x07 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_CLRW (0x08 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_MATH0 (0x08 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_STAT (0x09 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_MATH1 (0x09 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_MATH2 (0x0a << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_AAD_SZ (0x0b << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DECO_MATH3 (0x0b << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_CLASS1_ICV_SZ (0x0c << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_ALTDS_CLASS1 (0x0f << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_PKHA_A_SZ (0x10 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_PKHA_N_SZ (0x12 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_PKHA_E_SZ (0x13 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_DESCBUF (0x40 << LDST_SRCDST_SHIFT)
+#define LDST_SRCDST_WORD_INFO_FIFO (0x7a << LDST_SRCDST_SHIFT)
+
+/* Offset in source/destination */
+#define LDST_OFFSET_SHIFT 8
+#define LDST_OFFSET_MASK (0xff << LDST_OFFSET_SHIFT)
+
+/* LDOFF definitions used when DST = LDST_SRCDST_WORD_DECOCTRL */
+/* These could also be shifted by LDST_OFFSET_SHIFT - this reads better */
+#define LDOFF_CHG_SHARE_SHIFT 0
+#define LDOFF_CHG_SHARE_MASK (0x3 << LDOFF_CHG_SHARE_SHIFT)
+#define LDOFF_CHG_SHARE_NEVER (0x1 << LDOFF_CHG_SHARE_SHIFT)
+#define LDOFF_CHG_SHARE_OK_PROP (0x2 << LDOFF_CHG_SHARE_SHIFT)
+#define LDOFF_CHG_SHARE_OK_NO_PROP (0x3 << LDOFF_CHG_SHARE_SHIFT)
+
+#define LDOFF_ENABLE_AUTO_NFIFO (1 << 2)
+#define LDOFF_DISABLE_AUTO_NFIFO (1 << 3)
+
+#define LDOFF_CHG_NONSEQLIODN_SHIFT 4
+#define LDOFF_CHG_NONSEQLIODN_MASK (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT)
+#define LDOFF_CHG_NONSEQLIODN_SEQ (0x1 << LDOFF_CHG_NONSEQLIODN_SHIFT)
+#define LDOFF_CHG_NONSEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_NONSEQLIODN_SHIFT)
+#define LDOFF_CHG_NONSEQLIODN_TRUSTED (0x3 << LDOFF_CHG_NONSEQLIODN_SHIFT)
+
+#define LDOFF_CHG_SEQLIODN_SHIFT 6
+#define LDOFF_CHG_SEQLIODN_MASK (0x3 << LDOFF_CHG_SEQLIODN_SHIFT)
+#define LDOFF_CHG_SEQLIODN_SEQ (0x1 << LDOFF_CHG_SEQLIODN_SHIFT)
+#define LDOFF_CHG_SEQLIODN_NON_SEQ (0x2 << LDOFF_CHG_SEQLIODN_SHIFT)
+#define LDOFF_CHG_SEQLIODN_TRUSTED (0x3 << LDOFF_CHG_SEQLIODN_SHIFT)
+
+/* Data length in bytes */
+#define LDST_LEN_SHIFT 0
+#define LDST_LEN_MASK (0xff << LDST_LEN_SHIFT)
+
+/* Special Length definitions when dst=deco-ctrl */
+#define LDLEN_ENABLE_OSL_COUNT (1 << 7)
+#define LDLEN_RST_CHA_OFIFO_PTR (1 << 6)
+#define LDLEN_RST_OFIFO (1 << 5)
+#define LDLEN_SET_OFIFO_OFF_VALID (1 << 4)
+#define LDLEN_SET_OFIFO_OFF_RSVD (1 << 3)
+#define LDLEN_SET_OFIFO_OFFSET_SHIFT 0
+#define LDLEN_SET_OFIFO_OFFSET_MASK (3 << LDLEN_SET_OFIFO_OFFSET_SHIFT)
+
+/*
+ * FIFO_LOAD/FIFO_STORE/SEQ_FIFO_LOAD/SEQ_FIFO_STORE
+ * Command Constructs
+ */
+
+/*
+ * Load Destination: 0 = skip (SEQ_FIFO_LOAD only),
+ * 1 = Load for Class1, 2 = Load for Class2, 3 = Load both
+ * Store Source: 0 = normal, 1 = Class1key, 2 = Class2key
+ */
+#define FIFOLD_CLASS_SHIFT 25
+#define FIFOLD_CLASS_MASK (0x03 << FIFOLD_CLASS_SHIFT)
+#define FIFOLD_CLASS_SKIP (0x00 << FIFOLD_CLASS_SHIFT)
+#define FIFOLD_CLASS_CLASS1 (0x01 << FIFOLD_CLASS_SHIFT)
+#define FIFOLD_CLASS_CLASS2 (0x02 << FIFOLD_CLASS_SHIFT)
+#define FIFOLD_CLASS_BOTH (0x03 << FIFOLD_CLASS_SHIFT)
+
+#define FIFOST_CLASS_SHIFT 25
+#define FIFOST_CLASS_MASK (0x03 << FIFOST_CLASS_SHIFT)
+#define FIFOST_CLASS_NORMAL (0x00 << FIFOST_CLASS_SHIFT)
+#define FIFOST_CLASS_CLASS1KEY (0x01 << FIFOST_CLASS_SHIFT)
+#define FIFOST_CLASS_CLASS2KEY (0x02 << FIFOST_CLASS_SHIFT)
+
+/*
+ * Scatter-Gather Table/Variable Length Field
+ * If set for FIFO_LOAD, refers to a SG table. Within
+ * SEQ_FIFO_LOAD, is variable input sequence
+ */
+#define FIFOLDST_SGF_SHIFT 24
+#define FIFOLDST_SGF_MASK (1 << FIFOLDST_SGF_SHIFT)
+#define FIFOLDST_VLF_MASK (1 << FIFOLDST_SGF_SHIFT)
+#define FIFOLDST_SGF (1 << FIFOLDST_SGF_SHIFT)
+#define FIFOLDST_VLF (1 << FIFOLDST_SGF_SHIFT)
+
+/* Immediate - Data follows command in descriptor */
+#define FIFOLD_IMM_SHIFT 23
+#define FIFOLD_IMM_MASK (1 << FIFOLD_IMM_SHIFT)
+#define FIFOLD_IMM (1 << FIFOLD_IMM_SHIFT)
+
+/* Continue - Not the last FIFO store to come */
+#define FIFOST_CONT_SHIFT 23
+#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT)
+#define FIFOST_CONT_MASK (1 << FIFOST_CONT_SHIFT)
+
+/*
+ * Extended Length - use 32-bit extended length that
+ * follows the pointer field. Illegal with IMM set
+ */
+#define FIFOLDST_EXT_SHIFT 22
+#define FIFOLDST_EXT_MASK (1 << FIFOLDST_EXT_SHIFT)
+#define FIFOLDST_EXT (1 << FIFOLDST_EXT_SHIFT)
+
+/* Input data type.*/
+#define FIFOLD_TYPE_SHIFT 16
+#define FIFOLD_CONT_TYPE_SHIFT 19 /* shift past last-flush bits */
+#define FIFOLD_TYPE_MASK (0x3f << FIFOLD_TYPE_SHIFT)
+
+/* PK types */
+#define FIFOLD_TYPE_PK (0x00 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_MASK (0x30 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_TYPEMASK (0x0f << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_A0 (0x00 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_A1 (0x01 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_A2 (0x02 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_A3 (0x03 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_B0 (0x04 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_B1 (0x05 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_B2 (0x06 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_B3 (0x07 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT)
+
+/* Other types. Need to OR in last/flush bits as desired */
+#define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_MSG (0x10 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_MSG1OUT2 (0x18 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_IV (0x20 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_BITDATA (0x28 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_AAD (0x30 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_ICV (0x38 << FIFOLD_TYPE_SHIFT)
+
+/* Last/Flush bits for use with "other" types above */
+#define FIFOLD_TYPE_ACT_MASK (0x07 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_NOACTION (0x00 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_FLUSH1 (0x01 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LAST1 (0x02 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LAST2FLUSH (0x03 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LAST2 (0x04 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LAST2FLUSH1 (0x05 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LASTBOTH (0x06 << FIFOLD_TYPE_SHIFT)
+#define FIFOLD_TYPE_LASTBOTHFL (0x07 << FIFOLD_TYPE_SHIFT)
+
+#define FIFOLDST_LEN_MASK 0xffff
+#define FIFOLDST_EXT_LEN_MASK 0xffffffff
+
+/* Output data types */
+#define FIFOST_TYPE_SHIFT 16
+#define FIFOST_TYPE_MASK (0x3f << FIFOST_TYPE_SHIFT)
+
+#define FIFOST_TYPE_PKHA_A0 (0x00 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_A1 (0x01 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_A2 (0x02 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_A3 (0x03 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_B0 (0x04 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_B1 (0x05 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_B2 (0x06 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_B3 (0x07 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_AF_SBOX_JKEK (0x10 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_PKHA_E_TKEK (0x23 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_KEY_KEK (0x24 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_KEY_TKEK (0x25 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_SPLIT_KEK (0x26 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_SPLIT_TKEK (0x27 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_OUTFIFO_KEK (0x28 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_OUTFIFO_TKEK (0x29 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT)
+#define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT)
+
+/*
+ * OPERATION Command Constructs
+ */
+
+/* Operation type selectors - OP TYPE */
+#define OP_TYPE_SHIFT 24
+#define OP_TYPE_MASK (0x07 << OP_TYPE_SHIFT)
+
+#define OP_TYPE_UNI_PROTOCOL (0x00 << OP_TYPE_SHIFT)
+#define OP_TYPE_PK (0x01 << OP_TYPE_SHIFT)
+#define OP_TYPE_CLASS1_ALG (0x02 << OP_TYPE_SHIFT)
+#define OP_TYPE_CLASS2_ALG (0x04 << OP_TYPE_SHIFT)
+#define OP_TYPE_DECAP_PROTOCOL (0x06 << OP_TYPE_SHIFT)
+#define OP_TYPE_ENCAP_PROTOCOL (0x07 << OP_TYPE_SHIFT)
+
+/* ProtocolID selectors - PROTID */
+#define OP_PCLID_SHIFT 16
+#define OP_PCLID_MASK (0xff << 16)
+
+/* Assuming OP_TYPE = OP_TYPE_UNI_PROTOCOL */
+#define OP_PCLID_IKEV1_PRF (0x01 << OP_PCLID_SHIFT)
+#define OP_PCLID_IKEV2_PRF (0x02 << OP_PCLID_SHIFT)
+#define OP_PCLID_SSL30_PRF (0x08 << OP_PCLID_SHIFT)
+#define OP_PCLID_TLS10_PRF (0x09 << OP_PCLID_SHIFT)
+#define OP_PCLID_TLS11_PRF (0x0a << OP_PCLID_SHIFT)
+#define OP_PCLID_DTLS10_PRF (0x0c << OP_PCLID_SHIFT)
+#define OP_PCLID_PRF (0x06 << OP_PCLID_SHIFT)
+#define OP_PCLID_BLOB (0x0d << OP_PCLID_SHIFT)
+#define OP_PCLID_SECRETKEY (0x11 << OP_PCLID_SHIFT)
+#define OP_PCLID_PUBLICKEYPAIR (0x14 << OP_PCLID_SHIFT)
+#define OP_PCLID_DSASIGN (0x15 << OP_PCLID_SHIFT)
+#define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT)
+
+/* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */
+#define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT)
+#define OP_PCLID_SRTP (0x02 << OP_PCLID_SHIFT)
+#define OP_PCLID_MACSEC (0x03 << OP_PCLID_SHIFT)
+#define OP_PCLID_WIFI (0x04 << OP_PCLID_SHIFT)
+#define OP_PCLID_WIMAX (0x05 << OP_PCLID_SHIFT)
+#define OP_PCLID_SSL30 (0x08 << OP_PCLID_SHIFT)
+#define OP_PCLID_TLS10 (0x09 << OP_PCLID_SHIFT)
+#define OP_PCLID_TLS11 (0x0a << OP_PCLID_SHIFT)
+#define OP_PCLID_TLS12 (0x0b << OP_PCLID_SHIFT)
+#define OP_PCLID_DTLS (0x0c << OP_PCLID_SHIFT)
+
+/*
+ * ProtocolInfo selectors
+ */
+#define OP_PCLINFO_MASK 0xffff
+
+/* for OP_PCLID_IPSEC */
+#define OP_PCL_IPSEC_CIPHER_MASK 0xff00
+#define OP_PCL_IPSEC_AUTH_MASK 0x00ff
+
+#define OP_PCL_IPSEC_DES_IV64 0x0100
+#define OP_PCL_IPSEC_DES 0x0200
+#define OP_PCL_IPSEC_3DES 0x0300
+#define OP_PCL_IPSEC_AES_CBC 0x0c00
+#define OP_PCL_IPSEC_AES_CTR 0x0d00
+#define OP_PCL_IPSEC_AES_XTS 0x1600
+#define OP_PCL_IPSEC_AES_CCM8 0x0e00
+#define OP_PCL_IPSEC_AES_CCM12 0x0f00
+#define OP_PCL_IPSEC_AES_CCM16 0x1000
+#define OP_PCL_IPSEC_AES_GCM8 0x1200
+#define OP_PCL_IPSEC_AES_GCM12 0x1300
+#define OP_PCL_IPSEC_AES_GCM16 0x1400
+
+#define OP_PCL_IPSEC_HMAC_NULL 0x0000
+#define OP_PCL_IPSEC_HMAC_MD5_96 0x0001
+#define OP_PCL_IPSEC_HMAC_SHA1_96 0x0002
+#define OP_PCL_IPSEC_AES_XCBC_MAC_96 0x0005
+#define OP_PCL_IPSEC_HMAC_MD5_128 0x0006
+#define OP_PCL_IPSEC_HMAC_SHA1_160 0x0007
+#define OP_PCL_IPSEC_HMAC_SHA2_256_128 0x000c
+#define OP_PCL_IPSEC_HMAC_SHA2_384_192 0x000d
+#define OP_PCL_IPSEC_HMAC_SHA2_512_256 0x000e
+
+/* For SRTP - OP_PCLID_SRTP */
+#define OP_PCL_SRTP_CIPHER_MASK 0xff00
+#define OP_PCL_SRTP_AUTH_MASK 0x00ff
+
+#define OP_PCL_SRTP_AES_CTR 0x0d00
+
+#define OP_PCL_SRTP_HMAC_SHA1_160 0x0007
+
+/* For SSL 3.0 - OP_PCLID_SSL30 */
+#define OP_PCL_SSL30_AES_128_CBC_SHA 0x002f
+#define OP_PCL_SSL30_AES_128_CBC_SHA_2 0x0030
+#define OP_PCL_SSL30_AES_128_CBC_SHA_3 0x0031
+#define OP_PCL_SSL30_AES_128_CBC_SHA_4 0x0032
+#define OP_PCL_SSL30_AES_128_CBC_SHA_5 0x0033
+#define OP_PCL_SSL30_AES_128_CBC_SHA_6 0x0034
+#define OP_PCL_SSL30_AES_128_CBC_SHA_7 0x008c
+#define OP_PCL_SSL30_AES_128_CBC_SHA_8 0x0090
+#define OP_PCL_SSL30_AES_128_CBC_SHA_9 0x0094
+#define OP_PCL_SSL30_AES_128_CBC_SHA_10 0xc004
+#define OP_PCL_SSL30_AES_128_CBC_SHA_11 0xc009
+#define OP_PCL_SSL30_AES_128_CBC_SHA_12 0xc00e
+#define OP_PCL_SSL30_AES_128_CBC_SHA_13 0xc013
+#define OP_PCL_SSL30_AES_128_CBC_SHA_14 0xc018
+#define OP_PCL_SSL30_AES_128_CBC_SHA_15 0xc01d
+#define OP_PCL_SSL30_AES_128_CBC_SHA_16 0xc01e
+#define OP_PCL_SSL30_AES_128_CBC_SHA_17 0xc01f
+
+#define OP_PCL_SSL30_AES_256_CBC_SHA 0x0035
+#define OP_PCL_SSL30_AES_256_CBC_SHA_2 0x0036
+#define OP_PCL_SSL30_AES_256_CBC_SHA_3 0x0037
+#define OP_PCL_SSL30_AES_256_CBC_SHA_4 0x0038
+#define OP_PCL_SSL30_AES_256_CBC_SHA_5 0x0039
+#define OP_PCL_SSL30_AES_256_CBC_SHA_6 0x003a
+#define OP_PCL_SSL30_AES_256_CBC_SHA_7 0x008d
+#define OP_PCL_SSL30_AES_256_CBC_SHA_8 0x0091
+#define OP_PCL_SSL30_AES_256_CBC_SHA_9 0x0095
+#define OP_PCL_SSL30_AES_256_CBC_SHA_10 0xc005
+#define OP_PCL_SSL30_AES_256_CBC_SHA_11 0xc00a
+#define OP_PCL_SSL30_AES_256_CBC_SHA_12 0xc00f
+#define OP_PCL_SSL30_AES_256_CBC_SHA_13 0xc014
+#define OP_PCL_SSL30_AES_256_CBC_SHA_14 0xc019
+#define OP_PCL_SSL30_AES_256_CBC_SHA_15 0xc020
+#define OP_PCL_SSL30_AES_256_CBC_SHA_16 0xc021
+#define OP_PCL_SSL30_AES_256_CBC_SHA_17 0xc022
+
+#define OP_PCL_SSL30_3DES_EDE_CBC_MD5 0x0023
+
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA 0x001f
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_2 0x008b
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_3 0x008f
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_4 0x0093
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_5 0x000a
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_6 0x000d
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_7 0x0010
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_8 0x0013
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_9 0x0016
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_10 0x001b
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_11 0xc003
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_12 0xc008
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_13 0xc00d
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_14 0xc012
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_15 0xc017
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_16 0xc01a
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_17 0xc01b
+#define OP_PCL_SSL30_3DES_EDE_CBC_SHA_18 0xc01c
+
+#define OP_PCL_SSL30_DES40_CBC_MD5 0x0029
+
+#define OP_PCL_SSL30_DES_CBC_MD5 0x0022
+
+#define OP_PCL_SSL30_DES40_CBC_SHA 0x0008
+#define OP_PCL_SSL30_DES40_CBC_SHA_2 0x000b
+#define OP_PCL_SSL30_DES40_CBC_SHA_3 0x000e
+#define OP_PCL_SSL30_DES40_CBC_SHA_4 0x0011
+#define OP_PCL_SSL30_DES40_CBC_SHA_5 0x0014
+#define OP_PCL_SSL30_DES40_CBC_SHA_6 0x0019
+#define OP_PCL_SSL30_DES40_CBC_SHA_7 0x0026
+
+#define OP_PCL_SSL30_DES_CBC_SHA 0x001e
+#define OP_PCL_SSL30_DES_CBC_SHA_2 0x0009
+#define OP_PCL_SSL30_DES_CBC_SHA_3 0x000c
+#define OP_PCL_SSL30_DES_CBC_SHA_4 0x000f
+#define OP_PCL_SSL30_DES_CBC_SHA_5 0x0012
+#define OP_PCL_SSL30_DES_CBC_SHA_6 0x0015
+#define OP_PCL_SSL30_DES_CBC_SHA_7 0x001a
+
+#define OP_PCL_SSL30_RC4_128_MD5 0x0024
+#define OP_PCL_SSL30_RC4_128_MD5_2 0x0004
+#define OP_PCL_SSL30_RC4_128_MD5_3 0x0018
+
+#define OP_PCL_SSL30_RC4_40_MD5 0x002b
+#define OP_PCL_SSL30_RC4_40_MD5_2 0x0003
+#define OP_PCL_SSL30_RC4_40_MD5_3 0x0017
+
+#define OP_PCL_SSL30_RC4_128_SHA 0x0020
+#define OP_PCL_SSL30_RC4_128_SHA_2 0x008a
+#define OP_PCL_SSL30_RC4_128_SHA_3 0x008e
+#define OP_PCL_SSL30_RC4_128_SHA_4 0x0092
+#define OP_PCL_SSL30_RC4_128_SHA_5 0x0005
+#define OP_PCL_SSL30_RC4_128_SHA_6 0xc002
+#define OP_PCL_SSL30_RC4_128_SHA_7 0xc007
+#define OP_PCL_SSL30_RC4_128_SHA_8 0xc00c
+#define OP_PCL_SSL30_RC4_128_SHA_9 0xc011
+#define OP_PCL_SSL30_RC4_128_SHA_10 0xc016
+
+#define OP_PCL_SSL30_RC4_40_SHA 0x0028
+
+
+/* For TLS 1.0 - OP_PCLID_TLS10 */
+#define OP_PCL_TLS10_AES_128_CBC_SHA 0x002f
+#define OP_PCL_TLS10_AES_128_CBC_SHA_2 0x0030
+#define OP_PCL_TLS10_AES_128_CBC_SHA_3 0x0031
+#define OP_PCL_TLS10_AES_128_CBC_SHA_4 0x0032
+#define OP_PCL_TLS10_AES_128_CBC_SHA_5 0x0033
+#define OP_PCL_TLS10_AES_128_CBC_SHA_6 0x0034
+#define OP_PCL_TLS10_AES_128_CBC_SHA_7 0x008c
+#define OP_PCL_TLS10_AES_128_CBC_SHA_8 0x0090
+#define OP_PCL_TLS10_AES_128_CBC_SHA_9 0x0094
+#define OP_PCL_TLS10_AES_128_CBC_SHA_10 0xc004
+#define OP_PCL_TLS10_AES_128_CBC_SHA_11 0xc009
+#define OP_PCL_TLS10_AES_128_CBC_SHA_12 0xc00e
+#define OP_PCL_TLS10_AES_128_CBC_SHA_13 0xc013
+#define OP_PCL_TLS10_AES_128_CBC_SHA_14 0xc018
+#define OP_PCL_TLS10_AES_128_CBC_SHA_15 0xc01d
+#define OP_PCL_TLS10_AES_128_CBC_SHA_16 0xc01e
+#define OP_PCL_TLS10_AES_128_CBC_SHA_17 0xc01f
+
+#define OP_PCL_TLS10_AES_256_CBC_SHA 0x0035
+#define OP_PCL_TLS10_AES_256_CBC_SHA_2 0x0036
+#define OP_PCL_TLS10_AES_256_CBC_SHA_3 0x0037
+#define OP_PCL_TLS10_AES_256_CBC_SHA_4 0x0038
+#define OP_PCL_TLS10_AES_256_CBC_SHA_5 0x0039
+#define OP_PCL_TLS10_AES_256_CBC_SHA_6 0x003a
+#define OP_PCL_TLS10_AES_256_CBC_SHA_7 0x008d
+#define OP_PCL_TLS10_AES_256_CBC_SHA_8 0x0091
+#define OP_PCL_TLS10_AES_256_CBC_SHA_9 0x0095
+#define OP_PCL_TLS10_AES_256_CBC_SHA_10 0xc005
+#define OP_PCL_TLS10_AES_256_CBC_SHA_11 0xc00a
+#define OP_PCL_TLS10_AES_256_CBC_SHA_12 0xc00f
+#define OP_PCL_TLS10_AES_256_CBC_SHA_13 0xc014
+#define OP_PCL_TLS10_AES_256_CBC_SHA_14 0xc019
+#define OP_PCL_TLS10_AES_256_CBC_SHA_15 0xc020
+#define OP_PCL_TLS10_AES_256_CBC_SHA_16 0xc021
+#define OP_PCL_TLS10_AES_256_CBC_SHA_17 0xc022
+
+/* #define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0x0023 */
+
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA 0x001f
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_2 0x008b
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_3 0x008f
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_4 0x0093
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_5 0x000a
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_6 0x000d
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_7 0x0010
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_8 0x0013
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_9 0x0016
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_10 0x001b
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_11 0xc003
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_12 0xc008
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_13 0xc00d
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_14 0xc012
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_15 0xc017
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_16 0xc01a
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_17 0xc01b
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA_18 0xc01c
+
+#define OP_PCL_TLS10_DES40_CBC_MD5 0x0029
+
+#define OP_PCL_TLS10_DES_CBC_MD5 0x0022
+
+#define OP_PCL_TLS10_DES40_CBC_SHA 0x0008
+#define OP_PCL_TLS10_DES40_CBC_SHA_2 0x000b
+#define OP_PCL_TLS10_DES40_CBC_SHA_3 0x000e
+#define OP_PCL_TLS10_DES40_CBC_SHA_4 0x0011
+#define OP_PCL_TLS10_DES40_CBC_SHA_5 0x0014
+#define OP_PCL_TLS10_DES40_CBC_SHA_6 0x0019
+#define OP_PCL_TLS10_DES40_CBC_SHA_7 0x0026
+
+
+#define OP_PCL_TLS10_DES_CBC_SHA 0x001e
+#define OP_PCL_TLS10_DES_CBC_SHA_2 0x0009
+#define OP_PCL_TLS10_DES_CBC_SHA_3 0x000c
+#define OP_PCL_TLS10_DES_CBC_SHA_4 0x000f
+#define OP_PCL_TLS10_DES_CBC_SHA_5 0x0012
+#define OP_PCL_TLS10_DES_CBC_SHA_6 0x0015
+#define OP_PCL_TLS10_DES_CBC_SHA_7 0x001a
+
+#define OP_PCL_TLS10_RC4_128_MD5 0x0024
+#define OP_PCL_TLS10_RC4_128_MD5_2 0x0004
+#define OP_PCL_TLS10_RC4_128_MD5_3 0x0018
+
+#define OP_PCL_TLS10_RC4_40_MD5 0x002b
+#define OP_PCL_TLS10_RC4_40_MD5_2 0x0003
+#define OP_PCL_TLS10_RC4_40_MD5_3 0x0017
+
+#define OP_PCL_TLS10_RC4_128_SHA 0x0020
+#define OP_PCL_TLS10_RC4_128_SHA_2 0x008a
+#define OP_PCL_TLS10_RC4_128_SHA_3 0x008e
+#define OP_PCL_TLS10_RC4_128_SHA_4 0x0092
+#define OP_PCL_TLS10_RC4_128_SHA_5 0x0005
+#define OP_PCL_TLS10_RC4_128_SHA_6 0xc002
+#define OP_PCL_TLS10_RC4_128_SHA_7 0xc007
+#define OP_PCL_TLS10_RC4_128_SHA_8 0xc00c
+#define OP_PCL_TLS10_RC4_128_SHA_9 0xc011
+#define OP_PCL_TLS10_RC4_128_SHA_10 0xc016
+
+#define OP_PCL_TLS10_RC4_40_SHA 0x0028
+
+#define OP_PCL_TLS10_3DES_EDE_CBC_MD5 0xff23
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA160 0xff30
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA224 0xff34
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA256 0xff36
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA384 0xff33
+#define OP_PCL_TLS10_3DES_EDE_CBC_SHA512 0xff35
+#define OP_PCL_TLS10_AES_128_CBC_SHA160 0xff80
+#define OP_PCL_TLS10_AES_128_CBC_SHA224 0xff84
+#define OP_PCL_TLS10_AES_128_CBC_SHA256 0xff86
+#define OP_PCL_TLS10_AES_128_CBC_SHA384 0xff83
+#define OP_PCL_TLS10_AES_128_CBC_SHA512 0xff85
+#define OP_PCL_TLS10_AES_192_CBC_SHA160 0xff20
+#define OP_PCL_TLS10_AES_192_CBC_SHA224 0xff24
+#define OP_PCL_TLS10_AES_192_CBC_SHA256 0xff26
+#define OP_PCL_TLS10_AES_192_CBC_SHA384 0xff23
+#define OP_PCL_TLS10_AES_192_CBC_SHA512 0xff25
+#define OP_PCL_TLS10_AES_256_CBC_SHA160 0xff60
+#define OP_PCL_TLS10_AES_256_CBC_SHA224 0xff64
+#define OP_PCL_TLS10_AES_256_CBC_SHA256 0xff66
+#define OP_PCL_TLS10_AES_256_CBC_SHA384 0xff63
+#define OP_PCL_TLS10_AES_256_CBC_SHA512 0xff65
+
+
+
+/* For TLS 1.1 - OP_PCLID_TLS11 */
+#define OP_PCL_TLS11_AES_128_CBC_SHA 0x002f
+#define OP_PCL_TLS11_AES_128_CBC_SHA_2 0x0030
+#define OP_PCL_TLS11_AES_128_CBC_SHA_3 0x0031
+#define OP_PCL_TLS11_AES_128_CBC_SHA_4 0x0032
+#define OP_PCL_TLS11_AES_128_CBC_SHA_5 0x0033
+#define OP_PCL_TLS11_AES_128_CBC_SHA_6 0x0034
+#define OP_PCL_TLS11_AES_128_CBC_SHA_7 0x008c
+#define OP_PCL_TLS11_AES_128_CBC_SHA_8 0x0090
+#define OP_PCL_TLS11_AES_128_CBC_SHA_9 0x0094
+#define OP_PCL_TLS11_AES_128_CBC_SHA_10 0xc004
+#define OP_PCL_TLS11_AES_128_CBC_SHA_11 0xc009
+#define OP_PCL_TLS11_AES_128_CBC_SHA_12 0xc00e
+#define OP_PCL_TLS11_AES_128_CBC_SHA_13 0xc013
+#define OP_PCL_TLS11_AES_128_CBC_SHA_14 0xc018
+#define OP_PCL_TLS11_AES_128_CBC_SHA_15 0xc01d
+#define OP_PCL_TLS11_AES_128_CBC_SHA_16 0xc01e
+#define OP_PCL_TLS11_AES_128_CBC_SHA_17 0xc01f
+
+#define OP_PCL_TLS11_AES_256_CBC_SHA 0x0035
+#define OP_PCL_TLS11_AES_256_CBC_SHA_2 0x0036
+#define OP_PCL_TLS11_AES_256_CBC_SHA_3 0x0037
+#define OP_PCL_TLS11_AES_256_CBC_SHA_4 0x0038
+#define OP_PCL_TLS11_AES_256_CBC_SHA_5 0x0039
+#define OP_PCL_TLS11_AES_256_CBC_SHA_6 0x003a
+#define OP_PCL_TLS11_AES_256_CBC_SHA_7 0x008d
+#define OP_PCL_TLS11_AES_256_CBC_SHA_8 0x0091
+#define OP_PCL_TLS11_AES_256_CBC_SHA_9 0x0095
+#define OP_PCL_TLS11_AES_256_CBC_SHA_10 0xc005
+#define OP_PCL_TLS11_AES_256_CBC_SHA_11 0xc00a
+#define OP_PCL_TLS11_AES_256_CBC_SHA_12 0xc00f
+#define OP_PCL_TLS11_AES_256_CBC_SHA_13 0xc014
+#define OP_PCL_TLS11_AES_256_CBC_SHA_14 0xc019
+#define OP_PCL_TLS11_AES_256_CBC_SHA_15 0xc020
+#define OP_PCL_TLS11_AES_256_CBC_SHA_16 0xc021
+#define OP_PCL_TLS11_AES_256_CBC_SHA_17 0xc022
+
+/* #define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0x0023 */
+
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA 0x001f
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_2 0x008b
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_3 0x008f
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_4 0x0093
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_5 0x000a
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_6 0x000d
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_7 0x0010
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_8 0x0013
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_9 0x0016
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_10 0x001b
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_11 0xc003
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_12 0xc008
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_13 0xc00d
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_14 0xc012
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_15 0xc017
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_16 0xc01a
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_17 0xc01b
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA_18 0xc01c
+
+#define OP_PCL_TLS11_DES40_CBC_MD5 0x0029
+
+#define OP_PCL_TLS11_DES_CBC_MD5 0x0022
+
+#define OP_PCL_TLS11_DES40_CBC_SHA 0x0008
+#define OP_PCL_TLS11_DES40_CBC_SHA_2 0x000b
+#define OP_PCL_TLS11_DES40_CBC_SHA_3 0x000e
+#define OP_PCL_TLS11_DES40_CBC_SHA_4 0x0011
+#define OP_PCL_TLS11_DES40_CBC_SHA_5 0x0014
+#define OP_PCL_TLS11_DES40_CBC_SHA_6 0x0019
+#define OP_PCL_TLS11_DES40_CBC_SHA_7 0x0026
+
+#define OP_PCL_TLS11_DES_CBC_SHA 0x001e
+#define OP_PCL_TLS11_DES_CBC_SHA_2 0x0009
+#define OP_PCL_TLS11_DES_CBC_SHA_3 0x000c
+#define OP_PCL_TLS11_DES_CBC_SHA_4 0x000f
+#define OP_PCL_TLS11_DES_CBC_SHA_5 0x0012
+#define OP_PCL_TLS11_DES_CBC_SHA_6 0x0015
+#define OP_PCL_TLS11_DES_CBC_SHA_7 0x001a
+
+#define OP_PCL_TLS11_RC4_128_MD5 0x0024
+#define OP_PCL_TLS11_RC4_128_MD5_2 0x0004
+#define OP_PCL_TLS11_RC4_128_MD5_3 0x0018
+
+#define OP_PCL_TLS11_RC4_40_MD5 0x002b
+#define OP_PCL_TLS11_RC4_40_MD5_2 0x0003
+#define OP_PCL_TLS11_RC4_40_MD5_3 0x0017
+
+#define OP_PCL_TLS11_RC4_128_SHA 0x0020
+#define OP_PCL_TLS11_RC4_128_SHA_2 0x008a
+#define OP_PCL_TLS11_RC4_128_SHA_3 0x008e
+#define OP_PCL_TLS11_RC4_128_SHA_4 0x0092
+#define OP_PCL_TLS11_RC4_128_SHA_5 0x0005
+#define OP_PCL_TLS11_RC4_128_SHA_6 0xc002
+#define OP_PCL_TLS11_RC4_128_SHA_7 0xc007
+#define OP_PCL_TLS11_RC4_128_SHA_8 0xc00c
+#define OP_PCL_TLS11_RC4_128_SHA_9 0xc011
+#define OP_PCL_TLS11_RC4_128_SHA_10 0xc016
+
+#define OP_PCL_TLS11_RC4_40_SHA 0x0028
+
+#define OP_PCL_TLS11_3DES_EDE_CBC_MD5 0xff23
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA160 0xff30
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA224 0xff34
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA256 0xff36
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA384 0xff33
+#define OP_PCL_TLS11_3DES_EDE_CBC_SHA512 0xff35
+#define OP_PCL_TLS11_AES_128_CBC_SHA160 0xff80
+#define OP_PCL_TLS11_AES_128_CBC_SHA224 0xff84
+#define OP_PCL_TLS11_AES_128_CBC_SHA256 0xff86
+#define OP_PCL_TLS11_AES_128_CBC_SHA384 0xff83
+#define OP_PCL_TLS11_AES_128_CBC_SHA512 0xff85
+#define OP_PCL_TLS11_AES_192_CBC_SHA160 0xff20
+#define OP_PCL_TLS11_AES_192_CBC_SHA224 0xff24
+#define OP_PCL_TLS11_AES_192_CBC_SHA256 0xff26
+#define OP_PCL_TLS11_AES_192_CBC_SHA384 0xff23
+#define OP_PCL_TLS11_AES_192_CBC_SHA512 0xff25
+#define OP_PCL_TLS11_AES_256_CBC_SHA160 0xff60
+#define OP_PCL_TLS11_AES_256_CBC_SHA224 0xff64
+#define OP_PCL_TLS11_AES_256_CBC_SHA256 0xff66
+#define OP_PCL_TLS11_AES_256_CBC_SHA384 0xff63
+#define OP_PCL_TLS11_AES_256_CBC_SHA512 0xff65
+
+
+/* For TLS 1.2 - OP_PCLID_TLS12 */
+#define OP_PCL_TLS12_AES_128_CBC_SHA 0x002f
+#define OP_PCL_TLS12_AES_128_CBC_SHA_2 0x0030
+#define OP_PCL_TLS12_AES_128_CBC_SHA_3 0x0031
+#define OP_PCL_TLS12_AES_128_CBC_SHA_4 0x0032
+#define OP_PCL_TLS12_AES_128_CBC_SHA_5 0x0033
+#define OP_PCL_TLS12_AES_128_CBC_SHA_6 0x0034
+#define OP_PCL_TLS12_AES_128_CBC_SHA_7 0x008c
+#define OP_PCL_TLS12_AES_128_CBC_SHA_8 0x0090
+#define OP_PCL_TLS12_AES_128_CBC_SHA_9 0x0094
+#define OP_PCL_TLS12_AES_128_CBC_SHA_10 0xc004
+#define OP_PCL_TLS12_AES_128_CBC_SHA_11 0xc009
+#define OP_PCL_TLS12_AES_128_CBC_SHA_12 0xc00e
+#define OP_PCL_TLS12_AES_128_CBC_SHA_13 0xc013
+#define OP_PCL_TLS12_AES_128_CBC_SHA_14 0xc018
+#define OP_PCL_TLS12_AES_128_CBC_SHA_15 0xc01d
+#define OP_PCL_TLS12_AES_128_CBC_SHA_16 0xc01e
+#define OP_PCL_TLS12_AES_128_CBC_SHA_17 0xc01f
+
+#define OP_PCL_TLS12_AES_256_CBC_SHA 0x0035
+#define OP_PCL_TLS12_AES_256_CBC_SHA_2 0x0036
+#define OP_PCL_TLS12_AES_256_CBC_SHA_3 0x0037
+#define OP_PCL_TLS12_AES_256_CBC_SHA_4 0x0038
+#define OP_PCL_TLS12_AES_256_CBC_SHA_5 0x0039
+#define OP_PCL_TLS12_AES_256_CBC_SHA_6 0x003a
+#define OP_PCL_TLS12_AES_256_CBC_SHA_7 0x008d
+#define OP_PCL_TLS12_AES_256_CBC_SHA_8 0x0091
+#define OP_PCL_TLS12_AES_256_CBC_SHA_9 0x0095
+#define OP_PCL_TLS12_AES_256_CBC_SHA_10 0xc005
+#define OP_PCL_TLS12_AES_256_CBC_SHA_11 0xc00a
+#define OP_PCL_TLS12_AES_256_CBC_SHA_12 0xc00f
+#define OP_PCL_TLS12_AES_256_CBC_SHA_13 0xc014
+#define OP_PCL_TLS12_AES_256_CBC_SHA_14 0xc019
+#define OP_PCL_TLS12_AES_256_CBC_SHA_15 0xc020
+#define OP_PCL_TLS12_AES_256_CBC_SHA_16 0xc021
+#define OP_PCL_TLS12_AES_256_CBC_SHA_17 0xc022
+
+/* #define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0x0023 */
+
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA 0x001f
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_2 0x008b
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_3 0x008f
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_4 0x0093
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_5 0x000a
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_6 0x000d
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_7 0x0010
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_8 0x0013
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_9 0x0016
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_10 0x001b
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_11 0xc003
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_12 0xc008
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_13 0xc00d
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_14 0xc012
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_15 0xc017
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_16 0xc01a
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_17 0xc01b
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA_18 0xc01c
+
+#define OP_PCL_TLS12_DES40_CBC_MD5 0x0029
+
+#define OP_PCL_TLS12_DES_CBC_MD5 0x0022
+
+#define OP_PCL_TLS12_DES40_CBC_SHA 0x0008
+#define OP_PCL_TLS12_DES40_CBC_SHA_2 0x000b
+#define OP_PCL_TLS12_DES40_CBC_SHA_3 0x000e
+#define OP_PCL_TLS12_DES40_CBC_SHA_4 0x0011
+#define OP_PCL_TLS12_DES40_CBC_SHA_5 0x0014
+#define OP_PCL_TLS12_DES40_CBC_SHA_6 0x0019
+#define OP_PCL_TLS12_DES40_CBC_SHA_7 0x0026
+
+#define OP_PCL_TLS12_DES_CBC_SHA 0x001e
+#define OP_PCL_TLS12_DES_CBC_SHA_2 0x0009
+#define OP_PCL_TLS12_DES_CBC_SHA_3 0x000c
+#define OP_PCL_TLS12_DES_CBC_SHA_4 0x000f
+#define OP_PCL_TLS12_DES_CBC_SHA_5 0x0012
+#define OP_PCL_TLS12_DES_CBC_SHA_6 0x0015
+#define OP_PCL_TLS12_DES_CBC_SHA_7 0x001a
+
+#define OP_PCL_TLS12_RC4_128_MD5 0x0024
+#define OP_PCL_TLS12_RC4_128_MD5_2 0x0004
+#define OP_PCL_TLS12_RC4_128_MD5_3 0x0018
+
+#define OP_PCL_TLS12_RC4_40_MD5 0x002b
+#define OP_PCL_TLS12_RC4_40_MD5_2 0x0003
+#define OP_PCL_TLS12_RC4_40_MD5_3 0x0017
+
+#define OP_PCL_TLS12_RC4_128_SHA 0x0020
+#define OP_PCL_TLS12_RC4_128_SHA_2 0x008a
+#define OP_PCL_TLS12_RC4_128_SHA_3 0x008e
+#define OP_PCL_TLS12_RC4_128_SHA_4 0x0092
+#define OP_PCL_TLS12_RC4_128_SHA_5 0x0005
+#define OP_PCL_TLS12_RC4_128_SHA_6 0xc002
+#define OP_PCL_TLS12_RC4_128_SHA_7 0xc007
+#define OP_PCL_TLS12_RC4_128_SHA_8 0xc00c
+#define OP_PCL_TLS12_RC4_128_SHA_9 0xc011
+#define OP_PCL_TLS12_RC4_128_SHA_10 0xc016
+
+#define OP_PCL_TLS12_RC4_40_SHA 0x0028
+
+/* #define OP_PCL_TLS12_AES_128_CBC_SHA256 0x003c */
+#define OP_PCL_TLS12_AES_128_CBC_SHA256_2 0x003e
+#define OP_PCL_TLS12_AES_128_CBC_SHA256_3 0x003f
+#define OP_PCL_TLS12_AES_128_CBC_SHA256_4 0x0040
+#define OP_PCL_TLS12_AES_128_CBC_SHA256_5 0x0067
+#define OP_PCL_TLS12_AES_128_CBC_SHA256_6 0x006c
+
+/* #define OP_PCL_TLS12_AES_256_CBC_SHA256 0x003d */
+#define OP_PCL_TLS12_AES_256_CBC_SHA256_2 0x0068
+#define OP_PCL_TLS12_AES_256_CBC_SHA256_3 0x0069
+#define OP_PCL_TLS12_AES_256_CBC_SHA256_4 0x006a
+#define OP_PCL_TLS12_AES_256_CBC_SHA256_5 0x006b
+#define OP_PCL_TLS12_AES_256_CBC_SHA256_6 0x006d
+
+/* AEAD_AES_xxx_CCM/GCM remain to be defined... */
+
+#define OP_PCL_TLS12_3DES_EDE_CBC_MD5 0xff23
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA160 0xff30
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA224 0xff34
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA256 0xff36
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA384 0xff33
+#define OP_PCL_TLS12_3DES_EDE_CBC_SHA512 0xff35
+#define OP_PCL_TLS12_AES_128_CBC_SHA160 0xff80
+#define OP_PCL_TLS12_AES_128_CBC_SHA224 0xff84
+#define OP_PCL_TLS12_AES_128_CBC_SHA256 0xff86
+#define OP_PCL_TLS12_AES_128_CBC_SHA384 0xff83
+#define OP_PCL_TLS12_AES_128_CBC_SHA512 0xff85
+#define OP_PCL_TLS12_AES_192_CBC_SHA160 0xff20
+#define OP_PCL_TLS12_AES_192_CBC_SHA224 0xff24
+#define OP_PCL_TLS12_AES_192_CBC_SHA256 0xff26
+#define OP_PCL_TLS12_AES_192_CBC_SHA384 0xff23
+#define OP_PCL_TLS12_AES_192_CBC_SHA512 0xff25
+#define OP_PCL_TLS12_AES_256_CBC_SHA160 0xff60
+#define OP_PCL_TLS12_AES_256_CBC_SHA224 0xff64
+#define OP_PCL_TLS12_AES_256_CBC_SHA256 0xff66
+#define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63
+#define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65
+
+/* For DTLS - OP_PCLID_DTLS */
+
+#define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f
+#define OP_PCL_DTLS_AES_128_CBC_SHA_2 0x0030
+#define OP_PCL_DTLS_AES_128_CBC_SHA_3 0x0031
+#define OP_PCL_DTLS_AES_128_CBC_SHA_4 0x0032
+#define OP_PCL_DTLS_AES_128_CBC_SHA_5 0x0033
+#define OP_PCL_DTLS_AES_128_CBC_SHA_6 0x0034
+#define OP_PCL_DTLS_AES_128_CBC_SHA_7 0x008c
+#define OP_PCL_DTLS_AES_128_CBC_SHA_8 0x0090
+#define OP_PCL_DTLS_AES_128_CBC_SHA_9 0x0094
+#define OP_PCL_DTLS_AES_128_CBC_SHA_10 0xc004
+#define OP_PCL_DTLS_AES_128_CBC_SHA_11 0xc009
+#define OP_PCL_DTLS_AES_128_CBC_SHA_12 0xc00e
+#define OP_PCL_DTLS_AES_128_CBC_SHA_13 0xc013
+#define OP_PCL_DTLS_AES_128_CBC_SHA_14 0xc018
+#define OP_PCL_DTLS_AES_128_CBC_SHA_15 0xc01d
+#define OP_PCL_DTLS_AES_128_CBC_SHA_16 0xc01e
+#define OP_PCL_DTLS_AES_128_CBC_SHA_17 0xc01f
+
+#define OP_PCL_DTLS_AES_256_CBC_SHA 0x0035
+#define OP_PCL_DTLS_AES_256_CBC_SHA_2 0x0036
+#define OP_PCL_DTLS_AES_256_CBC_SHA_3 0x0037
+#define OP_PCL_DTLS_AES_256_CBC_SHA_4 0x0038
+#define OP_PCL_DTLS_AES_256_CBC_SHA_5 0x0039
+#define OP_PCL_DTLS_AES_256_CBC_SHA_6 0x003a
+#define OP_PCL_DTLS_AES_256_CBC_SHA_7 0x008d
+#define OP_PCL_DTLS_AES_256_CBC_SHA_8 0x0091
+#define OP_PCL_DTLS_AES_256_CBC_SHA_9 0x0095
+#define OP_PCL_DTLS_AES_256_CBC_SHA_10 0xc005
+#define OP_PCL_DTLS_AES_256_CBC_SHA_11 0xc00a
+#define OP_PCL_DTLS_AES_256_CBC_SHA_12 0xc00f
+#define OP_PCL_DTLS_AES_256_CBC_SHA_13 0xc014
+#define OP_PCL_DTLS_AES_256_CBC_SHA_14 0xc019
+#define OP_PCL_DTLS_AES_256_CBC_SHA_15 0xc020
+#define OP_PCL_DTLS_AES_256_CBC_SHA_16 0xc021
+#define OP_PCL_DTLS_AES_256_CBC_SHA_17 0xc022
+
+/* #define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0x0023 */
+
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA 0x001f
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_2 0x008b
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_3 0x008f
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_4 0x0093
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_5 0x000a
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_6 0x000d
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_7 0x0010
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_8 0x0013
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_9 0x0016
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_10 0x001b
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_11 0xc003
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_12 0xc008
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_13 0xc00d
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_14 0xc012
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_15 0xc017
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_16 0xc01a
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_17 0xc01b
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA_18 0xc01c
+
+#define OP_PCL_DTLS_DES40_CBC_MD5 0x0029
+
+#define OP_PCL_DTLS_DES_CBC_MD5 0x0022
+
+#define OP_PCL_DTLS_DES40_CBC_SHA 0x0008
+#define OP_PCL_DTLS_DES40_CBC_SHA_2 0x000b
+#define OP_PCL_DTLS_DES40_CBC_SHA_3 0x000e
+#define OP_PCL_DTLS_DES40_CBC_SHA_4 0x0011
+#define OP_PCL_DTLS_DES40_CBC_SHA_5 0x0014
+#define OP_PCL_DTLS_DES40_CBC_SHA_6 0x0019
+#define OP_PCL_DTLS_DES40_CBC_SHA_7 0x0026
+
+
+#define OP_PCL_DTLS_DES_CBC_SHA 0x001e
+#define OP_PCL_DTLS_DES_CBC_SHA_2 0x0009
+#define OP_PCL_DTLS_DES_CBC_SHA_3 0x000c
+#define OP_PCL_DTLS_DES_CBC_SHA_4 0x000f
+#define OP_PCL_DTLS_DES_CBC_SHA_5 0x0012
+#define OP_PCL_DTLS_DES_CBC_SHA_6 0x0015
+#define OP_PCL_DTLS_DES_CBC_SHA_7 0x001a
+
+
+#define OP_PCL_DTLS_3DES_EDE_CBC_MD5 0xff23
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA160 0xff30
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA224 0xff34
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA256 0xff36
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA384 0xff33
+#define OP_PCL_DTLS_3DES_EDE_CBC_SHA512 0xff35
+#define OP_PCL_DTLS_AES_128_CBC_SHA160 0xff80
+#define OP_PCL_DTLS_AES_128_CBC_SHA224 0xff84
+#define OP_PCL_DTLS_AES_128_CBC_SHA256 0xff86
+#define OP_PCL_DTLS_AES_128_CBC_SHA384 0xff83
+#define OP_PCL_DTLS_AES_128_CBC_SHA512 0xff85
+#define OP_PCL_DTLS_AES_192_CBC_SHA160 0xff20
+#define OP_PCL_DTLS_AES_192_CBC_SHA224 0xff24
+#define OP_PCL_DTLS_AES_192_CBC_SHA256 0xff26
+#define OP_PCL_DTLS_AES_192_CBC_SHA384 0xff23
+#define OP_PCL_DTLS_AES_192_CBC_SHA512 0xff25
+#define OP_PCL_DTLS_AES_256_CBC_SHA160 0xff60
+#define OP_PCL_DTLS_AES_256_CBC_SHA224 0xff64
+#define OP_PCL_DTLS_AES_256_CBC_SHA256 0xff66
+#define OP_PCL_DTLS_AES_256_CBC_SHA384 0xff63
+#define OP_PCL_DTLS_AES_256_CBC_SHA512 0xff65
+
+/* 802.16 WiMAX protinfos */
+#define OP_PCL_WIMAX_OFDM 0x0201
+#define OP_PCL_WIMAX_OFDMA 0x0231
+
+/* 802.11 WiFi protinfos */
+#define OP_PCL_WIFI 0xac04
+
+/* MacSec protinfos */
+#define OP_PCL_MACSEC 0x0001
+
+/* PKI unidirectional protocol protinfo bits */
+#define OP_PCL_PKPROT_TEST 0x0008
+#define OP_PCL_PKPROT_DECRYPT 0x0004
+#define OP_PCL_PKPROT_ECC 0x0002
+#define OP_PCL_PKPROT_F2M 0x0001
+
+/* For non-protocol/alg-only op commands */
+#define OP_ALG_TYPE_SHIFT 24
+#define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT)
+#define OP_ALG_TYPE_CLASS1 2
+#define OP_ALG_TYPE_CLASS2 4
+
+#define OP_ALG_ALGSEL_SHIFT 16
+#define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SUBMASK (0x0f << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_AES (0x10 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_DES (0x20 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_3DES (0x21 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_ARC4 (0x30 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_MD5 (0x40 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SHA1 (0x41 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SHA224 (0x42 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SHA256 (0x43 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SHA384 (0x44 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SHA512 (0x45 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_RNG (0x50 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SNOW (0x60 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SNOW_F8 (0x60 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_KASUMI (0x70 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_CRC (0x90 << OP_ALG_ALGSEL_SHIFT)
+#define OP_ALG_ALGSEL_SNOW_F9 (0xA0 << OP_ALG_ALGSEL_SHIFT)
+
+#define OP_ALG_AAI_SHIFT 4
+#define OP_ALG_AAI_MASK (0x1ff << OP_ALG_AAI_SHIFT)
+
+/* blockcipher AAI set */
+#define OP_ALG_AAI_CTR_MOD128 (0x00 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD8 (0x01 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD16 (0x02 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD24 (0x03 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD32 (0x04 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD40 (0x05 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD48 (0x06 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD56 (0x07 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD64 (0x08 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD72 (0x09 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD80 (0x0a << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD88 (0x0b << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD96 (0x0c << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD104 (0x0d << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD112 (0x0e << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_MOD120 (0x0f << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CBC (0x10 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_ECB (0x20 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CFB (0x30 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_OFB (0x40 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_XTS (0x50 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CMAC (0x60 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_XCBC_MAC (0x70 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CCM (0x80 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_GCM (0x90 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CBC_XCBCMAC (0xa0 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CTR_XCBCMAC (0xb0 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CHECKODD (0x80 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_DK (0x100 << OP_ALG_AAI_SHIFT)
+
+/* randomizer AAI set */
+#define OP_ALG_AAI_RNG (0x00 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_RNG_NOZERO (0x10 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_RNG_ODD (0x20 << OP_ALG_AAI_SHIFT)
+
+/* hmac/smac AAI set */
+#define OP_ALG_AAI_HASH (0x00 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_HMAC (0x01 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_SMAC (0x02 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_HMAC_PRECOMP (0x04 << OP_ALG_AAI_SHIFT)
+
+/* CRC AAI set*/
+#define OP_ALG_AAI_802 (0x01 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_3385 (0x02 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_CUST_POLY (0x04 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_DIS (0x10 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_DOS (0x20 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_DOC (0x40 << OP_ALG_AAI_SHIFT)
+
+/* Kasumi/SNOW AAI set */
+#define OP_ALG_AAI_F8 (0xc0 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_F9 (0xc8 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_GSM (0x10 << OP_ALG_AAI_SHIFT)
+#define OP_ALG_AAI_EDGE (0x20 << OP_ALG_AAI_SHIFT)
+
+
+#define OP_ALG_AS_SHIFT 2
+#define OP_ALG_AS_MASK (0x3 << OP_ALG_AS_SHIFT)
+#define OP_ALG_AS_UPDATE (0 << OP_ALG_AS_SHIFT)
+#define OP_ALG_AS_INIT (1 << OP_ALG_AS_SHIFT)
+#define OP_ALG_AS_FINALIZE (2 << OP_ALG_AS_SHIFT)
+#define OP_ALG_AS_INITFINAL (3 << OP_ALG_AS_SHIFT)
+
+#define OP_ALG_ICV_SHIFT 1
+#define OP_ALG_ICV_MASK (1 << OP_ALG_ICV_SHIFT)
+#define OP_ALG_ICV_OFF (0 << OP_ALG_ICV_SHIFT)
+#define OP_ALG_ICV_ON (1 << OP_ALG_ICV_SHIFT)
+
+#define OP_ALG_DIR_SHIFT 0
+#define OP_ALG_DIR_MASK 1
+#define OP_ALG_DECRYPT 0
+#define OP_ALG_ENCRYPT 1
+
+/* PKHA algorithm type set */
+#define OP_ALG_PK 0x00800000
+#define OP_ALG_PK_FUN_MASK 0x3f /* clrmem, modmath, or cpymem */
+
+/* PKHA mode clear memory functions */
+#define OP_ALG_PKMODE_A_RAM 0x80000
+#define OP_ALG_PKMODE_B_RAM 0x40000
+#define OP_ALG_PKMODE_E_RAM 0x20000
+#define OP_ALG_PKMODE_N_RAM 0x10000
+#define OP_ALG_PKMODE_CLEARMEM 0x00001
+
+/* PKHA mode modular-arithmetic functions */
+#define OP_ALG_PKMODE_MOD_IN_MONTY 0x80000
+#define OP_ALG_PKMODE_MOD_OUT_MONTY 0x40000
+#define OP_ALG_PKMODE_MOD_F2M 0x20000
+#define OP_ALG_PKMODE_MOD_R2_IN 0x10000
+#define OP_ALG_PKMODE_PRJECTV 0x00800
+#define OP_ALG_PKMODE_TIME_EQ 0x400
+#define OP_ALG_PKMODE_OUT_B 0x000
+#define OP_ALG_PKMODE_OUT_A 0x100
+#define OP_ALG_PKMODE_MOD_ADD 0x002
+#define OP_ALG_PKMODE_MOD_SUB_AB 0x003
+#define OP_ALG_PKMODE_MOD_SUB_BA 0x004
+#define OP_ALG_PKMODE_MOD_MULT 0x005
+#define OP_ALG_PKMODE_MOD_EXPO 0x006
+#define OP_ALG_PKMODE_MOD_REDUCT 0x007
+#define OP_ALG_PKMODE_MOD_INV 0x008
+#define OP_ALG_PKMODE_MOD_ECC_ADD 0x009
+#define OP_ALG_PKMODE_MOD_ECC_DBL 0x00a
+#define OP_ALG_PKMODE_MOD_ECC_MULT 0x00b
+#define OP_ALG_PKMODE_MOD_MONT_CNST 0x00c
+#define OP_ALG_PKMODE_MOD_CRT_CNST 0x00d
+#define OP_ALG_PKMODE_MOD_GCD 0x00e
+#define OP_ALG_PKMODE_MOD_PRIMALITY 0x00f
+
+/* PKHA mode copy-memory functions */
+#define OP_ALG_PKMODE_SRC_REG_SHIFT 13
+#define OP_ALG_PKMODE_SRC_REG_MASK (7 << OP_ALG_PKMODE_SRC_REG_SHIFT)
+#define OP_ALG_PKMODE_DST_REG_SHIFT 10
+#define OP_ALG_PKMODE_DST_REG_MASK (7 << OP_ALG_PKMODE_DST_REG_SHIFT)
+#define OP_ALG_PKMODE_SRC_SEG_SHIFT 8
+#define OP_ALG_PKMODE_SRC_SEG_MASK (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT)
+#define OP_ALG_PKMODE_DST_SEG_SHIFT 6
+#define OP_ALG_PKMODE_DST_SEG_MASK (3 << OP_ALG_PKMODE_DST_SEG_SHIFT)
+
+#define OP_ALG_PKMODE_SRC_REG_A (0 << OP_ALG_PKMODE_SRC_REG_SHIFT)
+#define OP_ALG_PKMODE_SRC_REG_B (1 << OP_ALG_PKMODE_SRC_REG_SHIFT)
+#define OP_ALG_PKMODE_SRC_REG_N (3 << OP_ALG_PKMODE_SRC_REG_SHIFT)
+#define OP_ALG_PKMODE_DST_REG_A (0 << OP_ALG_PKMODE_DST_REG_SHIFT)
+#define OP_ALG_PKMODE_DST_REG_B (1 << OP_ALG_PKMODE_DST_REG_SHIFT)
+#define OP_ALG_PKMODE_DST_REG_E (2 << OP_ALG_PKMODE_DST_REG_SHIFT)
+#define OP_ALG_PKMODE_DST_REG_N (3 << OP_ALG_PKMODE_DST_REG_SHIFT)
+#define OP_ALG_PKMODE_SRC_SEG_0 (0 << OP_ALG_PKMODE_SRC_SEG_SHIFT)
+#define OP_ALG_PKMODE_SRC_SEG_1 (1 << OP_ALG_PKMODE_SRC_SEG_SHIFT)
+#define OP_ALG_PKMODE_SRC_SEG_2 (2 << OP_ALG_PKMODE_SRC_SEG_SHIFT)
+#define OP_ALG_PKMODE_SRC_SEG_3 (3 << OP_ALG_PKMODE_SRC_SEG_SHIFT)
+#define OP_ALG_PKMODE_DST_SEG_0 (0 << OP_ALG_PKMODE_DST_SEG_SHIFT)
+#define OP_ALG_PKMODE_DST_SEG_1 (1 << OP_ALG_PKMODE_DST_SEG_SHIFT)
+#define OP_ALG_PKMODE_DST_SEG_2 (2 << OP_ALG_PKMODE_DST_SEG_SHIFT)
+#define OP_ALG_PKMODE_DST_SEG_3 (3 << OP_ALG_PKMODE_DST_SEG_SHIFT)
+#define OP_ALG_PKMODE_CPYMEM_N_SZ 0x80
+#define OP_ALG_PKMODE_CPYMEM_SRC_SZ 0x81
+
+/*
+ * SEQ_IN_PTR Command Constructs
+ */
+
+/* Release Buffers */
+#define SQIN_RBS 0x04000000
+
+/* Sequence pointer is really a descriptor */
+#define SQIN_INL 0x02000000
+
+/* Sequence pointer is a scatter-gather table */
+#define SQIN_SGF 0x01000000
+
+/* Appends to a previous pointer */
+#define SQIN_PRE 0x00800000
+
+/* Use extended length following pointer */
+#define SQIN_EXT 0x00400000
+
+/* Restore sequence with pointer/length */
+#define SQIN_RTO 0x00200000
+
+/* Replace job descriptor */
+#define SQIN_RJD 0x00100000
+
+#define SQIN_LEN_SHIFT 0
+#define SQIN_LEN_MASK (0xffff << SQIN_LEN_SHIFT)
+
+/*
+ * SEQ_OUT_PTR Command Constructs
+ */
+
+/* Sequence pointer is a scatter-gather table */
+#define SQOUT_SGF 0x01000000
+
+/* Appends to a previous pointer */
+#define SQOUT_PRE 0x00800000
+
+/* Restore sequence with pointer/length */
+#define SQOUT_RTO 0x00200000
+
+/* Use extended length following pointer */
+#define SQOUT_EXT 0x00400000
+
+#define SQOUT_LEN_SHIFT 0
+#define SQOUT_LEN_MASK (0xffff << SQOUT_LEN_SHIFT)
+
+
+/*
+ * SIGNATURE Command Constructs
+ */
+
+/* TYPE field is all that's relevant */
+#define SIGN_TYPE_SHIFT 16
+#define SIGN_TYPE_MASK (0x0f << SIGN_TYPE_SHIFT)
+
+#define SIGN_TYPE_FINAL (0x00 << SIGN_TYPE_SHIFT)
+#define SIGN_TYPE_FINAL_RESTORE (0x01 << SIGN_TYPE_SHIFT)
+#define SIGN_TYPE_FINAL_NONZERO (0x02 << SIGN_TYPE_SHIFT)
+#define SIGN_TYPE_IMM_2 (0x0a << SIGN_TYPE_SHIFT)
+#define SIGN_TYPE_IMM_3 (0x0b << SIGN_TYPE_SHIFT)
+#define SIGN_TYPE_IMM_4 (0x0c << SIGN_TYPE_SHIFT)
+
+/*
+ * MOVE Command Constructs
+ */
+
+#define MOVE_AUX_SHIFT 25
+#define MOVE_AUX_MASK (3 << MOVE_AUX_SHIFT)
+#define MOVE_AUX_MS (2 << MOVE_AUX_SHIFT)
+#define MOVE_AUX_LS (1 << MOVE_AUX_SHIFT)
+
+#define MOVE_WAITCOMP_SHIFT 24
+#define MOVE_WAITCOMP_MASK (1 << MOVE_WAITCOMP_SHIFT)
+#define MOVE_WAITCOMP (1 << MOVE_WAITCOMP_SHIFT)
+
+#define MOVE_SRC_SHIFT 20
+#define MOVE_SRC_MASK (0x0f << MOVE_SRC_SHIFT)
+#define MOVE_SRC_CLASS1CTX (0x00 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_CLASS2CTX (0x01 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_OUTFIFO (0x02 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_DESCBUF (0x03 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_MATH0 (0x04 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_MATH1 (0x05 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_MATH2 (0x06 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_MATH3 (0x07 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_INFIFO (0x08 << MOVE_SRC_SHIFT)
+#define MOVE_SRC_INFIFO_CL (0x09 << MOVE_SRC_SHIFT)
+
+#define MOVE_DEST_SHIFT 16
+#define MOVE_DEST_MASK (0x0f << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS1CTX (0x00 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS2CTX (0x01 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_OUTFIFO (0x02 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_DESCBUF (0x03 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_MATH0 (0x04 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_MATH1 (0x05 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_MATH2 (0x06 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_MATH3 (0x07 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS1INFIFO (0x08 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS2INFIFO (0x09 << MOVE_DEST_SHIFT)
+#define MOVE_DEST_PK_A (0x0c << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS1KEY (0x0d << MOVE_DEST_SHIFT)
+#define MOVE_DEST_CLASS2KEY (0x0e << MOVE_DEST_SHIFT)
+
+#define MOVE_OFFSET_SHIFT 8
+#define MOVE_OFFSET_MASK (0xff << MOVE_OFFSET_SHIFT)
+
+#define MOVE_LEN_SHIFT 0
+#define MOVE_LEN_MASK (0xff << MOVE_LEN_SHIFT)
+
+#define MOVELEN_MRSEL_SHIFT 0
+#define MOVELEN_MRSEL_MASK (0x3 << MOVE_LEN_SHIFT)
+
+/*
+ * MATH Command Constructs
+ */
+
+#define MATH_IFB_SHIFT 26
+#define MATH_IFB_MASK (1 << MATH_IFB_SHIFT)
+#define MATH_IFB (1 << MATH_IFB_SHIFT)
+
+#define MATH_NFU_SHIFT 25
+#define MATH_NFU_MASK (1 << MATH_NFU_SHIFT)
+#define MATH_NFU (1 << MATH_NFU_SHIFT)
+
+#define MATH_STL_SHIFT 24
+#define MATH_STL_MASK (1 << MATH_STL_SHIFT)
+#define MATH_STL (1 << MATH_STL_SHIFT)
+
+/* Function selectors */
+#define MATH_FUN_SHIFT 20
+#define MATH_FUN_MASK (0x0f << MATH_FUN_SHIFT)
+#define MATH_FUN_ADD (0x00 << MATH_FUN_SHIFT)
+#define MATH_FUN_ADDC (0x01 << MATH_FUN_SHIFT)
+#define MATH_FUN_SUB (0x02 << MATH_FUN_SHIFT)
+#define MATH_FUN_SUBB (0x03 << MATH_FUN_SHIFT)
+#define MATH_FUN_OR (0x04 << MATH_FUN_SHIFT)
+#define MATH_FUN_AND (0x05 << MATH_FUN_SHIFT)
+#define MATH_FUN_XOR (0x06 << MATH_FUN_SHIFT)
+#define MATH_FUN_LSHIFT (0x07 << MATH_FUN_SHIFT)
+#define MATH_FUN_RSHIFT (0x08 << MATH_FUN_SHIFT)
+#define MATH_FUN_SHLD (0x09 << MATH_FUN_SHIFT)
+#define MATH_FUN_ZBYT (0x0a << MATH_FUN_SHIFT)
+
+/* Source 0 selectors */
+#define MATH_SRC0_SHIFT 16
+#define MATH_SRC0_MASK (0x0f << MATH_SRC0_SHIFT)
+#define MATH_SRC0_REG0 (0x00 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_REG1 (0x01 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_REG2 (0x02 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_REG3 (0x03 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_IMM (0x04 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_SEQINLEN (0x08 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_SEQOUTLEN (0x09 << MATH_SRC0_SHIFT)
+#define MATH_SRC0_VARSEQINLEN (0x0a << MATH_SRC0_SHIFT)
+#define MATH_SRC0_VARSEQOUTLEN (0x0b << MATH_SRC0_SHIFT)
+#define MATH_SRC0_ZERO (0x0c << MATH_SRC0_SHIFT)
+
+/* Source 1 selectors */
+#define MATH_SRC1_SHIFT 12
+#define MATH_SRC1_MASK (0x0f << MATH_SRC1_SHIFT)
+#define MATH_SRC1_REG0 (0x00 << MATH_SRC1_SHIFT)
+#define MATH_SRC1_REG1 (0x01 << MATH_SRC1_SHIFT)
+#define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT)
+#define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT)
+#define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT)
+#define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT)
+#define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT)
+#define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT)
+
+/* Destination selectors */
+#define MATH_DEST_SHIFT 8
+#define MATH_DEST_MASK (0x0f << MATH_DEST_SHIFT)
+#define MATH_DEST_REG0 (0x00 << MATH_DEST_SHIFT)
+#define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT)
+#define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT)
+#define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT)
+#define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT)
+#define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT)
+#define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT)
+#define MATH_DEST_VARSEQOUTLEN (0x0b << MATH_DEST_SHIFT)
+#define MATH_DEST_NONE (0x0f << MATH_DEST_SHIFT)
+
+/* Length selectors */
+#define MATH_LEN_SHIFT 0
+#define MATH_LEN_MASK (0x0f << MATH_LEN_SHIFT)
+#define MATH_LEN_1BYTE 0x01
+#define MATH_LEN_2BYTE 0x02
+#define MATH_LEN_4BYTE 0x04
+#define MATH_LEN_8BYTE 0x08
+
+/*
+ * JUMP Command Constructs
+ */
+
+#define JUMP_CLASS_SHIFT 25
+#define JUMP_CLASS_MASK (3 << JUMP_CLASS_SHIFT)
+#define JUMP_CLASS_NONE 0
+#define JUMP_CLASS_CLASS1 (1 << JUMP_CLASS_SHIFT)
+#define JUMP_CLASS_CLASS2 (2 << JUMP_CLASS_SHIFT)
+#define JUMP_CLASS_BOTH (3 << JUMP_CLASS_SHIFT)
+
+#define JUMP_JSL_SHIFT 24
+#define JUMP_JSL_MASK (1 << JUMP_JSL_SHIFT)
+#define JUMP_JSL (1 << JUMP_JSL_SHIFT)
+
+#define JUMP_TYPE_SHIFT 22
+#define JUMP_TYPE_MASK (0x03 << JUMP_TYPE_SHIFT)
+#define JUMP_TYPE_LOCAL (0x00 << JUMP_TYPE_SHIFT)
+#define JUMP_TYPE_NONLOCAL (0x01 << JUMP_TYPE_SHIFT)
+#define JUMP_TYPE_HALT (0x02 << JUMP_TYPE_SHIFT)
+#define JUMP_TYPE_HALT_USER (0x03 << JUMP_TYPE_SHIFT)
+
+#define JUMP_TEST_SHIFT 16
+#define JUMP_TEST_MASK (0x03 << JUMP_TEST_SHIFT)
+#define JUMP_TEST_ALL (0x00 << JUMP_TEST_SHIFT)
+#define JUMP_TEST_INVALL (0x01 << JUMP_TEST_SHIFT)
+#define JUMP_TEST_ANY (0x02 << JUMP_TEST_SHIFT)
+#define JUMP_TEST_INVANY (0x03 << JUMP_TEST_SHIFT)
+
+/* Condition codes. JSL bit is factored in */
+#define JUMP_COND_SHIFT 8
+#define JUMP_COND_MASK (0x100ff << JUMP_COND_SHIFT)
+#define JUMP_COND_PK_0 (0x80 << JUMP_COND_SHIFT)
+#define JUMP_COND_PK_GCD_1 (0x40 << JUMP_COND_SHIFT)
+#define JUMP_COND_PK_PRIME (0x20 << JUMP_COND_SHIFT)
+#define JUMP_COND_MATH_N (0x08 << JUMP_COND_SHIFT)
+#define JUMP_COND_MATH_Z (0x04 << JUMP_COND_SHIFT)
+#define JUMP_COND_MATH_C (0x02 << JUMP_COND_SHIFT)
+#define JUMP_COND_MATH_NV (0x01 << JUMP_COND_SHIFT)
+
+#define JUMP_COND_JRP ((0x80 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_SHRD ((0x40 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_SELF ((0x20 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_CALM ((0x10 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_NIP ((0x08 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_NIFP ((0x04 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_NOP ((0x02 << JUMP_COND_SHIFT) | JUMP_JSL)
+#define JUMP_COND_NCP ((0x01 << JUMP_COND_SHIFT) | JUMP_JSL)
+
+#define JUMP_OFFSET_SHIFT 0
+#define JUMP_OFFSET_MASK (0xff << JUMP_OFFSET_SHIFT)
+
+/*
+ * NFIFO ENTRY
+ * Data Constructs
+ *
+ */
+#define NFIFOENTRY_DEST_SHIFT 30
+#define NFIFOENTRY_DEST_MASK (3 << NFIFOENTRY_DEST_SHIFT)
+#define NFIFOENTRY_DEST_DECO (0 << NFIFOENTRY_DEST_SHIFT)
+#define NFIFOENTRY_DEST_CLASS1 (1 << NFIFOENTRY_DEST_SHIFT)
+#define NFIFOENTRY_DEST_CLASS2 (2 << NFIFOENTRY_DEST_SHIFT)
+#define NFIFOENTRY_DEST_BOTH (3 << NFIFOENTRY_DEST_SHIFT)
+
+#define NFIFOENTRY_LC2_SHIFT 29
+#define NFIFOENTRY_LC2_MASK (1 << NFIFOENTRY_LC2_SHIFT)
+#define NFIFOENTRY_LC2 (1 << NFIFOENTRY_LC2_SHIFT)
+
+#define NFIFOENTRY_LC1_SHIFT 28
+#define NFIFOENTRY_LC1_MASK (1 << NFIFOENTRY_LC1_SHIFT)
+#define NFIFOENTRY_LC1 (1 << NFIFOENTRY_LC1_SHIFT)
+
+#define NFIFOENTRY_FC2_SHIFT 27
+#define NFIFOENTRY_FC2_MASK (1 << NFIFOENTRY_FC2_SHIFT)
+#define NFIFOENTRY_FC2 (1 << NFIFOENTRY_FC2_SHIFT)
+
+#define NFIFOENTRY_FC1_SHIFT 26
+#define NFIFOENTRY_FC1_MASK (1 << NFIFOENTRY_FC1_SHIFT)
+#define NFIFOENTRY_FC1 (1 << NFIFOENTRY_FC1_SHIFT)
+
+#define NFIFOENTRY_STYPE_SHIFT 24
+#define NFIFOENTRY_STYPE_MASK (3 << NFIFOENTRY_STYPE_SHIFT)
+#define NFIFOENTRY_STYPE_DFIFO (0 << NFIFOENTRY_STYPE_SHIFT)
+#define NFIFOENTRY_STYPE_OFIFO (1 << NFIFOENTRY_STYPE_SHIFT)
+#define NFIFOENTRY_STYPE_PAD (2 << NFIFOENTRY_STYPE_SHIFT)
+#define NFIFOENTRY_STYPE_SNOOP (3 << NFIFOENTRY_STYPE_SHIFT)
+
+#define NFIFOENTRY_DTYPE_SHIFT 20
+#define NFIFOENTRY_DTYPE_MASK (0xF << NFIFOENTRY_DTYPE_SHIFT)
+
+#define NFIFOENTRY_DTYPE_SBOX (0x0 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_AAD (0x1 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_IV (0x2 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_SAD (0x3 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_ICV (0xA << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_SKIP (0xE << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_MSG (0xF << NFIFOENTRY_DTYPE_SHIFT)
+
+#define NFIFOENTRY_DTYPE_PK_A0 (0x0 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_A1 (0x1 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_A2 (0x2 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_A3 (0x3 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_B0 (0x4 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_B1 (0x5 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_B2 (0x6 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_B3 (0x7 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_N (0x8 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_E (0x9 << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_A (0xC << NFIFOENTRY_DTYPE_SHIFT)
+#define NFIFOENTRY_DTYPE_PK_B (0xD << NFIFOENTRY_DTYPE_SHIFT)
+
+
+#define NFIFOENTRY_BND_SHIFT 19
+#define NFIFOENTRY_BND_MASK (1 << NFIFOENTRY_BND_SHIFT)
+#define NFIFOENTRY_BND (1 << NFIFOENTRY_BND_SHIFT)
+
+#define NFIFOENTRY_PTYPE_SHIFT 16
+#define NFIFOENTRY_PTYPE_MASK (0x7 << NFIFOENTRY_PTYPE_SHIFT)
+
+#define NFIFOENTRY_PTYPE_ZEROS (0x0 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_RND_NOZEROS (0x1 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_INCREMENT (0x2 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_RND (0x3 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_ZEROS_NZ (0x4 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_RND_NZ_LZ (0x5 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_N (0x6 << NFIFOENTRY_PTYPE_SHIFT)
+#define NFIFOENTRY_PTYPE_RND_NZ_N (0x7 << NFIFOENTRY_PTYPE_SHIFT)
+
+#define NFIFOENTRY_OC_SHIFT 15
+#define NFIFOENTRY_OC_MASK (1 << NFIFOENTRY_OC_SHIFT)
+#define NFIFOENTRY_OC (1 << NFIFOENTRY_OC_SHIFT)
+
+#define NFIFOENTRY_AST_SHIFT 14
+#define NFIFOENTRY_AST_MASK (1 << NFIFOENTRY_OC_SHIFT)
+#define NFIFOENTRY_AST (1 << NFIFOENTRY_OC_SHIFT)
+
+#define NFIFOENTRY_BM_SHIFT 11
+#define NFIFOENTRY_BM_MASK (1 << NFIFOENTRY_BM_SHIFT)
+#define NFIFOENTRY_BM (1 << NFIFOENTRY_BM_SHIFT)
+
+#define NFIFOENTRY_PS_SHIFT 10
+#define NFIFOENTRY_PS_MASK (1 << NFIFOENTRY_PS_SHIFT)
+#define NFIFOENTRY_PS (1 << NFIFOENTRY_PS_SHIFT)
+
+#define NFIFOENTRY_DLEN_SHIFT 0
+#define NFIFOENTRY_DLEN_MASK (0xFFF << NFIFOENTRY_DLEN_SHIFT)
+
+#define NFIFOENTRY_PLEN_SHIFT 0
+#define NFIFOENTRY_PLEN_MASK (0xFF << NFIFOENTRY_PLEN_SHIFT)
+
+/*
+ * PDB internal definitions
+ */
+
+/* IPSec ESP CBC Encap/Decap Options */
+#define PDBOPTS_ESPCBC_ARSNONE 0x00 /* no antireplay window */
+#define PDBOPTS_ESPCBC_ARS32 0x40 /* 32-entry antireplay window */
+#define PDBOPTS_ESPCBC_ARS64 0xc0 /* 64-entry antireplay window */
+#define PDBOPTS_ESPCBC_IVSRC 0x20 /* IV comes from internal random gen */
+#define PDBOPTS_ESPCBC_ESN 0x10 /* extended sequence included */
+#define PDBOPTS_ESPCBC_OUTFMT 0x08 /* output only decapsulation (decap) */
+#define PDBOPTS_ESPCBC_IPHDRSRC 0x08 /* IP header comes from PDB (encap) */
+#define PDBOPTS_ESPCBC_INCIPHDR 0x04 /* Prepend IP header to output frame */
+#define PDBOPTS_ESPCBC_IPVSN 0x02 /* process IPv6 header */
+#define PDBOPTS_ESPCBC_TUNNEL 0x01 /* tunnel mode next-header byte */
+
+#endif /* DESC_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc_constr.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc_constr.h
new file mode 100644
index 0000000..348b882
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/desc_constr.h
@@ -0,0 +1,262 @@
+/*
+ * caam descriptor construction helper functions
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#include "desc.h"
+
+#define IMMEDIATE (1 << 23)
+#define CAAM_CMD_SZ sizeof(u32)
+#define CAAM_PTR_SZ sizeof(dma_addr_t)
+#define CAAM_DESC_BYTES_MAX (CAAM_CMD_SZ * MAX_CAAM_DESCSIZE)
+
+#ifdef DEBUG
+#define PRINT_POS do { printk(KERN_DEBUG "%02d: %s\n", desc_len(desc),\
+ &__func__[sizeof("append")]); } while (0)
+#else
+#define PRINT_POS
+#endif
+
+#define SET_OK_NO_PROP_ERRORS (IMMEDIATE | LDST_CLASS_DECO | \
+ LDST_SRCDST_WORD_DECOCTRL | \
+ (LDOFF_CHG_SHARE_OK_NO_PROP << \
+ LDST_OFFSET_SHIFT))
+#define DISABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \
+ LDST_SRCDST_WORD_DECOCTRL | \
+ (LDOFF_DISABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
+#define ENABLE_AUTO_INFO_FIFO (IMMEDIATE | LDST_CLASS_DECO | \
+ LDST_SRCDST_WORD_DECOCTRL | \
+ (LDOFF_ENABLE_AUTO_NFIFO << LDST_OFFSET_SHIFT))
+
+static inline int desc_len(u32 *desc)
+{
+ return *desc & HDR_DESCLEN_MASK;
+}
+
+static inline int desc_bytes(void *desc)
+{
+ return desc_len(desc) * CAAM_CMD_SZ;
+}
+
+static inline u32 *desc_end(u32 *desc)
+{
+ return desc + desc_len(desc);
+}
+
+static inline void *sh_desc_pdb(u32 *desc)
+{
+ return desc + 1;
+}
+
+static inline void init_desc(u32 *desc, u32 options)
+{
+ *desc = options | HDR_ONE | 1;
+}
+
+static inline void init_sh_desc(u32 *desc, u32 options)
+{
+ PRINT_POS;
+ init_desc(desc, CMD_SHARED_DESC_HDR | options);
+}
+
+static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes)
+{
+ u32 pdb_len = pdb_bytes / CAAM_CMD_SZ + 1;
+
+ init_sh_desc(desc, ((pdb_len << HDR_START_IDX_SHIFT) + pdb_len) |
+ options);
+}
+
+static inline void init_job_desc(u32 *desc, u32 options)
+{
+ init_desc(desc, CMD_DESC_HDR | options);
+}
+
+static inline void append_ptr(u32 *desc, dma_addr_t ptr)
+{
+ dma_addr_t *offset = (dma_addr_t *)desc_end(desc);
+
+ *offset = ptr;
+
+ (*desc) += CAAM_PTR_SZ / CAAM_CMD_SZ;
+}
+
+static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len,
+ u32 options)
+{
+ PRINT_POS;
+ init_job_desc(desc, HDR_SHARED | options |
+ (len << HDR_START_IDX_SHIFT));
+ append_ptr(desc, ptr);
+}
+
+static inline void append_data(u32 *desc, void *data, int len)
+{
+ u32 *offset = desc_end(desc);
+
+ if (len) /* avoid sparse warning: memcpy with byte count of 0 */
+ memcpy(offset, data, len);
+
+ (*desc) += (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ;
+}
+
+static inline void append_cmd(u32 *desc, u32 command)
+{
+ u32 *cmd = desc_end(desc);
+
+ *cmd = command;
+
+ (*desc)++;
+}
+
+static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len,
+ u32 command)
+{
+ append_cmd(desc, command | len);
+ append_ptr(desc, ptr);
+}
+
+static inline void append_cmd_data(u32 *desc, void *data, int len,
+ u32 command)
+{
+ append_cmd(desc, command | IMMEDIATE | len);
+ append_data(desc, data, len);
+}
+
+static inline u32 *append_jump(u32 *desc, u32 options)
+{
+ u32 *cmd = desc_end(desc);
+
+ PRINT_POS;
+ append_cmd(desc, CMD_JUMP | options);
+
+ return cmd;
+}
+
+static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd)
+{
+ *jump_cmd = *jump_cmd | (desc_len(desc) - (jump_cmd - desc));
+}
+
+#define APPEND_CMD(cmd, op) \
+static inline void append_##cmd(u32 *desc, u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd(desc, CMD_##op | options); \
+}
+APPEND_CMD(operation, OPERATION)
+APPEND_CMD(move, MOVE)
+
+#define APPEND_CMD_LEN(cmd, op) \
+static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd(desc, CMD_##op | len | options); \
+}
+APPEND_CMD_LEN(seq_store, SEQ_STORE)
+APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_LOAD)
+APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE)
+
+#define APPEND_CMD_PTR(cmd, op) \
+static inline void append_##cmd(u32 *desc, dma_addr_t ptr, unsigned int len, \
+ u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd_ptr(desc, ptr, len, CMD_##op | options); \
+}
+APPEND_CMD_PTR(key, KEY)
+APPEND_CMD_PTR(seq_in_ptr, SEQ_IN_PTR)
+APPEND_CMD_PTR(seq_out_ptr, SEQ_OUT_PTR)
+APPEND_CMD_PTR(load, LOAD)
+APPEND_CMD_PTR(store, STORE)
+APPEND_CMD_PTR(fifo_load, FIFO_LOAD)
+APPEND_CMD_PTR(fifo_store, FIFO_STORE)
+
+#define APPEND_CMD_PTR_TO_IMM(cmd, op) \
+static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
+ unsigned int len, u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd_data(desc, data, len, CMD_##op | options); \
+}
+APPEND_CMD_PTR_TO_IMM(load, LOAD);
+APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD);
+
+/*
+ * 2nd variant for commands whose specified immediate length differs
+ * from length of immediate data provided, e.g., split keys
+ */
+#define APPEND_CMD_PTR_TO_IMM2(cmd, op) \
+static inline void append_##cmd##_as_imm(u32 *desc, void *data, \
+ unsigned int data_len, \
+ unsigned int len, u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd(desc, CMD_##op | IMMEDIATE | len | options); \
+ append_data(desc, data, data_len); \
+}
+APPEND_CMD_PTR_TO_IMM2(key, KEY);
+
+#define APPEND_CMD_RAW_IMM(cmd, op, type) \
+static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \
+ u32 options) \
+{ \
+ PRINT_POS; \
+ append_cmd(desc, CMD_##op | IMMEDIATE | options | sizeof(type)); \
+ append_cmd(desc, immediate); \
+}
+APPEND_CMD_RAW_IMM(load, LOAD, u32);
+
+/*
+ * Append math command. Only the last part of destination and source need to
+ * be specified
+ */
+#define APPEND_MATH(op, desc, dest, src_0, src_1, len) \
+append_cmd(desc, CMD_MATH | MATH_FUN_##op | MATH_DEST_##dest | \
+ MATH_SRC0_##src_0 | MATH_SRC1_##src_1 | (u32) (len & MATH_LEN_MASK));
+
+#define append_math_add(desc, dest, src0, src1, len) \
+ APPEND_MATH(ADD, desc, dest, src0, src1, len)
+#define append_math_sub(desc, dest, src0, src1, len) \
+ APPEND_MATH(SUB, desc, dest, src0, src1, len)
+#define append_math_add_c(desc, dest, src0, src1, len) \
+ APPEND_MATH(ADDC, desc, dest, src0, src1, len)
+#define append_math_sub_b(desc, dest, src0, src1, len) \
+ APPEND_MATH(SUBB, desc, dest, src0, src1, len)
+#define append_math_and(desc, dest, src0, src1, len) \
+ APPEND_MATH(AND, desc, dest, src0, src1, len)
+#define append_math_or(desc, dest, src0, src1, len) \
+ APPEND_MATH(OR, desc, dest, src0, src1, len)
+#define append_math_xor(desc, dest, src0, src1, len) \
+ APPEND_MATH(XOR, desc, dest, src0, src1, len)
+#define append_math_lshift(desc, dest, src0, src1, len) \
+ APPEND_MATH(LSHIFT, desc, dest, src0, src1, len)
+#define append_math_rshift(desc, dest, src0, src1, len) \
+ APPEND_MATH(RSHIFT, desc, dest, src0, src1, len)
+
+/* Exactly one source is IMM. Data is passed in as u32 value */
+#define APPEND_MATH_IMM_u32(op, desc, dest, src_0, src_1, data) \
+do { \
+ APPEND_MATH(op, desc, dest, src_0, src_1, CAAM_CMD_SZ); \
+ append_cmd(desc, data); \
+} while (0);
+
+#define append_math_add_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(ADD, desc, dest, src0, src1, data)
+#define append_math_sub_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(SUB, desc, dest, src0, src1, data)
+#define append_math_add_c_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(ADDC, desc, dest, src0, src1, data)
+#define append_math_sub_b_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(SUBB, desc, dest, src0, src1, data)
+#define append_math_and_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(AND, desc, dest, src0, src1, data)
+#define append_math_or_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(OR, desc, dest, src0, src1, data)
+#define append_math_xor_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(XOR, desc, dest, src0, src1, data)
+#define append_math_lshift_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(LSHIFT, desc, dest, src0, src1, data)
+#define append_math_rshift_imm_u32(desc, dest, src0, src1, data) \
+ APPEND_MATH_IMM_u32(RSHIFT, desc, dest, src0, src1, data)
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.c b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.c
new file mode 100644
index 0000000..9b8d231
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.c
@@ -0,0 +1,252 @@
+/*
+ * CAAM Error Reporting
+ *
+ * Copyright 2009-2011 Freescale Semiconductor, Inc.
+ */
+
+#include "compat.h"
+#include "regs.h"
+#include "intern.h"
+#include "desc.h"
+#include "jr.h"
+#include "error.h"
+
+#define SPRINTFCAT(str, format, param, max_alloc) \
+{ \
+ char *tmp; \
+ \
+ tmp = kmalloc(sizeof(format) + max_alloc, GFP_ATOMIC); \
+ if (likely(tmp)) { \
+ sprintf(tmp, format, param); \
+ strcat(str, tmp); \
+ kfree(tmp); \
+ } else { \
+ strcat(str, "kmalloc failure in SPRINTFCAT"); \
+ } \
+}
+
+static void report_jump_idx(u32 status, char *outstr)
+{
+ u8 idx = (status & JRSTA_DECOERR_INDEX_MASK) >>
+ JRSTA_DECOERR_INDEX_SHIFT;
+
+ if (status & JRSTA_DECOERR_JUMP)
+ strcat(outstr, "jump tgt desc idx ");
+ else
+ strcat(outstr, "desc idx ");
+
+ SPRINTFCAT(outstr, "%d: ", idx, sizeof("255"));
+}
+
+static void report_ccb_status(u32 status, char *outstr)
+{
+ char *cha_id_list[] = {
+ "",
+ "AES",
+ "DES, 3DES",
+ "ARC4",
+ "MD5, SHA-1, SH-224, SHA-256, SHA-384, SHA-512",
+ "RNG",
+ "SNOW f8",
+ "Kasumi f8, f9",
+ "All Public Key Algorithms",
+ "CRC",
+ "SNOW f9",
+ };
+ char *err_id_list[] = {
+ "None. No error.",
+ "Mode error.",
+ "Data size error.",
+ "Key size error.",
+ "PKHA A memory size error.",
+ "PKHA B memory size error.",
+ "Data arrived out of sequence error.",
+ "PKHA divide-by-zero error.",
+ "PKHA modulus even error.",
+ "DES key parity error.",
+ "ICV check failed.",
+ "Hardware error.",
+ "Unsupported CCM AAD size.",
+ "Class 1 CHA is not reset",
+ "Invalid CHA combination was selected",
+ "Invalid CHA selected.",
+ };
+ u8 cha_id = (status & JRSTA_CCBERR_CHAID_MASK) >>
+ JRSTA_CCBERR_CHAID_SHIFT;
+ u8 err_id = status & JRSTA_CCBERR_ERRID_MASK;
+
+ report_jump_idx(status, outstr);
+
+ if (cha_id < ARRAY_SIZE(cha_id_list)) {
+ SPRINTFCAT(outstr, "%s: ", cha_id_list[cha_id],
+ strlen(cha_id_list[cha_id]));
+ } else {
+ SPRINTFCAT(outstr, "unidentified cha_id value 0x%02x: ",
+ cha_id, sizeof("ff"));
+ }
+
+ if (err_id < ARRAY_SIZE(err_id_list)) {
+ SPRINTFCAT(outstr, "%s", err_id_list[err_id],
+ strlen(err_id_list[err_id]));
+ } else {
+ SPRINTFCAT(outstr, "unidentified err_id value 0x%02x",
+ err_id, sizeof("ff"));
+ }
+}
+
+static void report_jump_status(u32 status, char *outstr)
+{
+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__));
+}
+
+static void report_deco_status(u32 status, char *outstr)
+{
+ const struct {
+ u8 value;
+ char *error_text;
+ } desc_error_list[] = {
+ { 0x00, "None. No error." },
+ { 0x01, "SGT Length Error. The descriptor is trying to read "
+ "more data than is contained in the SGT table." },
+ { 0x02, "Reserved." },
+ { 0x03, "Job Ring Control Error. There is a bad value in the "
+ "Job Ring Control register." },
+ { 0x04, "Invalid Descriptor Command. The Descriptor Command "
+ "field is invalid." },
+ { 0x05, "Reserved." },
+ { 0x06, "Invalid KEY Command" },
+ { 0x07, "Invalid LOAD Command" },
+ { 0x08, "Invalid STORE Command" },
+ { 0x09, "Invalid OPERATION Command" },
+ { 0x0A, "Invalid FIFO LOAD Command" },
+ { 0x0B, "Invalid FIFO STORE Command" },
+ { 0x0C, "Invalid MOVE Command" },
+ { 0x0D, "Invalid JUMP Command. A nonlocal JUMP Command is "
+ "invalid because the target is not a Job Header "
+ "Command, or the jump is from a Trusted Descriptor to "
+ "a Job Descriptor, or because the target Descriptor "
+ "contains a Shared Descriptor." },
+ { 0x0E, "Invalid MATH Command" },
+ { 0x0F, "Invalid SIGNATURE Command" },
+ { 0x10, "Invalid Sequence Command. A SEQ IN PTR OR SEQ OUT PTR "
+ "Command is invalid or a SEQ KEY, SEQ LOAD, SEQ FIFO "
+ "LOAD, or SEQ FIFO STORE decremented the input or "
+ "output sequence length below 0. This error may result "
+ "if a built-in PROTOCOL Command has encountered a "
+ "malformed PDU." },
+ { 0x11, "Skip data type invalid. The type must be 0xE or 0xF."},
+ { 0x12, "Shared Descriptor Header Error" },
+ { 0x13, "Header Error. Invalid length or parity, or certain "
+ "other problems." },
+ { 0x14, "Burster Error. Burster has gotten to an illegal "
+ "state" },
+ { 0x15, "Context Register Length Error. The descriptor is "
+ "trying to read or write past the end of the Context "
+ "Register. A SEQ LOAD or SEQ STORE with the VLF bit "
+ "set was executed with too large a length in the "
+ "variable length register (VSOL for SEQ STORE or VSIL "
+ "for SEQ LOAD)." },
+ { 0x16, "DMA Error" },
+ { 0x17, "Reserved." },
+ { 0x1A, "Job failed due to JR reset" },
+ { 0x1B, "Job failed due to Fail Mode" },
+ { 0x1C, "DECO Watchdog timer timeout error" },
+ { 0x1D, "DECO tried to copy a key from another DECO but the "
+ "other DECO's Key Registers were locked" },
+ { 0x1E, "DECO attempted to copy data from a DECO that had an "
+ "unmasked Descriptor error" },
+ { 0x1F, "LIODN error. DECO was trying to share from itself or "
+ "from another DECO but the two Non-SEQ LIODN values "
+ "didn't match or the 'shared from' DECO's Descriptor "
+ "required that the SEQ LIODNs be the same and they "
+ "aren't." },
+ { 0x20, "DECO has completed a reset initiated via the DRR "
+ "register" },
+ { 0x21, "Nonce error. When using EKT (CCM) key encryption "
+ "option in the FIFO STORE Command, the Nonce counter "
+ "reached its maximum value and this encryption mode "
+ "can no longer be used." },
+ { 0x22, "Meta data is too large (> 511 bytes) for TLS decap "
+ "(input frame; block ciphers) and IPsec decap (output "
+ "frame, when doing the next header byte update) and "
+ "DCRC (output frame)." },
+ { 0x80, "DNR (do not run) error" },
+ { 0x81, "undefined protocol command" },
+ { 0x82, "invalid setting in PDB" },
+ { 0x83, "Anti-replay LATE error" },
+ { 0x84, "Anti-replay REPLAY error" },
+ { 0x85, "Sequence number overflow" },
+ { 0x86, "Sigver invalid signature" },
+ { 0x87, "DSA Sign Illegal test descriptor" },
+ { 0x88, "Protocol Format Error - A protocol has seen an error "
+ "in the format of data received. When running RSA, "
+ "this means that formatting with random padding was "
+ "used, and did not follow the form: 0x00, 0x02, 8-to-N "
+ "bytes of non-zero pad, 0x00, F data." },
+ { 0x89, "Protocol Size Error - A protocol has seen an error in "
+ "size. When running RSA, pdb size N < (size of F) when "
+ "no formatting is used; or pdb size N < (F + 11) when "
+ "formatting is used." },
+ { 0xC1, "Blob Command error: Undefined mode" },
+ { 0xC2, "Blob Command error: Secure Memory Blob mode error" },
+ { 0xC4, "Blob Command error: Black Blob key or input size "
+ "error" },
+ { 0xC5, "Blob Command error: Invalid key destination" },
+ { 0xC8, "Blob Command error: Trusted/Secure mode error" },
+ { 0xF0, "IPsec TTL or hop limit field either came in as 0, "
+ "or was decremented to 0" },
+ { 0xF1, "3GPP HFN matches or exceeds the Threshold" },
+ };
+ u8 desc_error = status & JRSTA_DECOERR_ERROR_MASK;
+ int i;
+
+ report_jump_idx(status, outstr);
+
+ for (i = 0; i < ARRAY_SIZE(desc_error_list); i++)
+ if (desc_error_list[i].value == desc_error)
+ break;
+
+ if (i != ARRAY_SIZE(desc_error_list) && desc_error_list[i].error_text) {
+ SPRINTFCAT(outstr, "%s", desc_error_list[i].error_text,
+ strlen(desc_error_list[i].error_text));
+ } else {
+ SPRINTFCAT(outstr, "unidentified error value 0x%02x",
+ desc_error, sizeof("ff"));
+ }
+}
+
+static void report_jr_status(u32 status, char *outstr)
+{
+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__));
+}
+
+static void report_cond_code_status(u32 status, char *outstr)
+{
+ SPRINTFCAT(outstr, "%s() not implemented", __func__, sizeof(__func__));
+}
+
+char *caam_jr_strstatus(char *outstr, u32 status)
+{
+ struct stat_src {
+ void (*report_ssed)(u32 status, char *outstr);
+ char *error;
+ } status_src[] = {
+ { NULL, "No error" },
+ { NULL, NULL },
+ { report_ccb_status, "CCB" },
+ { report_jump_status, "Jump" },
+ { report_deco_status, "DECO" },
+ { NULL, NULL },
+ { report_jr_status, "Job Ring" },
+ { report_cond_code_status, "Condition Code" },
+ };
+ u32 ssrc = status >> JRSTA_SSRC_SHIFT;
+
+ sprintf(outstr, "%s: ", status_src[ssrc].error);
+
+ if (status_src[ssrc].report_ssed)
+ status_src[ssrc].report_ssed(status, outstr);
+
+ return outstr;
+}
+EXPORT_SYMBOL(caam_jr_strstatus);
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.h
new file mode 100644
index 0000000..02c7baa
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/error.h
@@ -0,0 +1,11 @@
+/*
+ * CAAM Error Reporting code header
+ *
+ * Copyright 2009-2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef CAAM_ERROR_H
+#define CAAM_ERROR_H
+#define CAAM_ERROR_STR_MAX 302
+extern char *caam_jr_strstatus(char *outstr, u32 status);
+#endif /* CAAM_ERROR_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/intern.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/intern.h
new file mode 100644
index 0000000..a34be01
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/intern.h
@@ -0,0 +1,113 @@
+/*
+ * CAAM/SEC 4.x driver backend
+ * Private/internal definitions between modules
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ */
+
+#ifndef INTERN_H
+#define INTERN_H
+
+#define JOBR_UNASSIGNED 0
+#define JOBR_ASSIGNED 1
+
+/* Currently comes from Kconfig param as a ^2 (driver-required) */
+#define JOBR_DEPTH (1 << CONFIG_CRYPTO_DEV_FSL_CAAM_RINGSIZE)
+
+/* Kconfig params for interrupt coalescing if selected (else zero) */
+#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_INTC
+#define JOBR_INTC JRCFG_ICEN
+#define JOBR_INTC_TIME_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD
+#define JOBR_INTC_COUNT_THLD CONFIG_CRYPTO_DEV_FSL_CAAM_INTC_COUNT_THLD
+#else
+#define JOBR_INTC 0
+#define JOBR_INTC_TIME_THLD 0
+#define JOBR_INTC_COUNT_THLD 0
+#endif
+
+/*
+ * Storage for tracking each in-process entry moving across a ring
+ * Each entry on an output ring needs one of these
+ */
+struct caam_jrentry_info {
+ void (*callbk)(struct device *dev, u32 *desc, u32 status, void *arg);
+ void *cbkarg; /* Argument per ring entry */
+ u32 *desc_addr_virt; /* Stored virt addr for postprocessing */
+ dma_addr_t desc_addr_dma; /* Stored bus addr for done matching */
+ u32 desc_size; /* Stored size for postprocessing, header derived */
+};
+
+/* Private sub-storage for a single JobR */
+struct caam_drv_private_jr {
+ struct device *parentdev; /* points back to controller dev */
+ int ridx;
+ struct caam_job_ring __iomem *rregs; /* JobR's register space */
+ struct tasklet_struct irqtask[NR_CPUS];
+ int irq; /* One per queue */
+ int assign; /* busy/free */
+
+ /* Job ring info */
+ int ringsize; /* Size of rings (assume input = output) */
+ struct caam_jrentry_info *entinfo; /* Alloc'ed 1 per ring entry */
+ spinlock_t inplock ____cacheline_aligned; /* Input ring index lock */
+ int inp_ring_write_index; /* Input index "tail" */
+ int head; /* entinfo (s/w ring) head index */
+ dma_addr_t *inpring; /* Base of input ring, alloc DMA-safe */
+ spinlock_t outlock ____cacheline_aligned; /* Output ring index lock */
+ int out_ring_read_index; /* Output index "tail" */
+ int tail; /* entinfo (s/w ring) tail index */
+ struct jr_outentry *outring; /* Base of output ring, DMA-safe */
+};
+
+/*
+ * Driver-private storage for a single CAAM block instance
+ */
+struct caam_drv_private {
+
+ struct device *dev;
+ struct device **jrdev; /* Alloc'ed array per sub-device */
+ spinlock_t jr_alloc_lock;
+ struct platform_device *pdev;
+
+ /* Physical-presence section */
+ struct caam_ctrl *ctrl; /* controller region */
+ struct caam_deco **deco; /* DECO/CCB views */
+ struct caam_assurance *ac;
+ struct caam_queue_if *qi; /* QI control region */
+
+ /*
+ * Detected geometry block. Filled in from device tree if powerpc,
+ * or from register-based version detection code
+ */
+ u8 total_jobrs; /* Total Job Rings in device */
+ u8 qi_present; /* Nonzero if QI present in device */
+ int secvio_irq; /* Security violation interrupt number */
+
+ /* which jr allocated to scatterlist crypto */
+ atomic_t tfm_count ____cacheline_aligned;
+ int num_jrs_for_algapi;
+ struct device **algapi_jr;
+ /* list of registered crypto algorithms (mk generic context handle?) */
+ struct list_head alg_list;
+
+ /*
+ * debugfs entries for developer view into driver/device
+ * variables at runtime.
+ */
+#ifdef CONFIG_DEBUG_FS
+ struct dentry *dfs_root;
+ struct dentry *ctl; /* controller dir */
+ struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req;
+ struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes;
+ struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes;
+ struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus;
+
+ struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap;
+ struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk;
+#endif
+};
+
+void caam_jr_algapi_init(struct device *dev);
+void caam_jr_algapi_remove(struct device *dev);
+#endif /* INTERN_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.c b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.c
new file mode 100644
index 0000000..340fa32
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.c
@@ -0,0 +1,517 @@
+/*
+ * CAAM/SEC 4.x transport/backend driver
+ * JobR backend functionality
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#include "compat.h"
+#include "regs.h"
+#include "jr.h"
+#include "desc.h"
+#include "intern.h"
+
+/* Main per-ring interrupt handler */
+static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
+{
+ struct device *dev = st_dev;
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ u32 irqstate;
+
+ /*
+ * Check the output ring for ready responses, kick
+ * tasklet if jobs done.
+ */
+ irqstate = rd_reg32(&jrp->rregs->jrintstatus);
+ if (!irqstate)
+ return IRQ_NONE;
+
+ /*
+ * If JobR error, we got more development work to do
+ * Flag a bug now, but we really need to shut down and
+ * restart the queue (and fix code).
+ */
+ if (irqstate & JRINT_JR_ERROR) {
+ dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
+ BUG();
+ }
+
+ /* mask valid interrupts */
+ setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
+
+ /* Have valid interrupt at this point, just ACK and trigger */
+ wr_reg32(&jrp->rregs->jrintstatus, irqstate);
+
+ preempt_disable();
+ tasklet_schedule(&jrp->irqtask[smp_processor_id()]);
+ preempt_enable();
+
+ return IRQ_HANDLED;
+}
+
+/* Deferred service handler, run as interrupt-fired tasklet */
+static void caam_jr_dequeue(unsigned long devarg)
+{
+ int hw_idx, sw_idx, i, head, tail;
+ struct device *dev = (struct device *)devarg;
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
+ u32 *userdesc, userstatus;
+ void *userarg;
+ unsigned long flags;
+
+ spin_lock_irqsave(&jrp->outlock, flags);
+
+ head = ACCESS_ONCE(jrp->head);
+ sw_idx = tail = jrp->tail;
+
+ while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
+ rd_reg32(&jrp->rregs->outring_used)) {
+
+ hw_idx = jrp->out_ring_read_index;
+ for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
+ sw_idx = (tail + i) & (JOBR_DEPTH - 1);
+
+ smp_read_barrier_depends();
+
+ if (jrp->outring[hw_idx].desc ==
+ jrp->entinfo[sw_idx].desc_addr_dma)
+ break; /* found */
+ }
+ /* we should never fail to find a matching descriptor */
+ BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
+
+ /* Unmap just-run descriptor so we can post-process */
+ dma_unmap_single(dev, jrp->outring[hw_idx].desc,
+ jrp->entinfo[sw_idx].desc_size,
+ DMA_TO_DEVICE);
+
+ /* mark completed, avoid matching on a recycled desc addr */
+ jrp->entinfo[sw_idx].desc_addr_dma = 0;
+
+ /* Stash callback params for use outside of lock */
+ usercall = jrp->entinfo[sw_idx].callbk;
+ userarg = jrp->entinfo[sw_idx].cbkarg;
+ userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
+ userstatus = jrp->outring[hw_idx].jrstatus;
+
+ smp_mb();
+
+ jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
+ (JOBR_DEPTH - 1);
+
+ /*
+ * if this job completed out-of-order, do not increment
+ * the tail. Otherwise, increment tail by 1 plus the
+ * number of subsequent jobs already completed out-of-order
+ */
+ if (sw_idx == tail) {
+ do {
+ tail = (tail + 1) & (JOBR_DEPTH - 1);
+ smp_read_barrier_depends();
+ } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
+ jrp->entinfo[tail].desc_addr_dma == 0);
+
+ jrp->tail = tail;
+ }
+
+ /* set done */
+ wr_reg32(&jrp->rregs->outring_rmvd, 1);
+
+ spin_unlock_irqrestore(&jrp->outlock, flags);
+
+ /* Finally, execute user's callback */
+ usercall(dev, userdesc, userstatus, userarg);
+
+ spin_lock_irqsave(&jrp->outlock, flags);
+
+ head = ACCESS_ONCE(jrp->head);
+ sw_idx = tail = jrp->tail;
+ }
+
+ spin_unlock_irqrestore(&jrp->outlock, flags);
+
+ /* reenable / unmask IRQs */
+ clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
+}
+
+/**
+ * caam_jr_register() - Alloc a ring for someone to use as needed. Returns
+ * an ordinal of the rings allocated, else returns -ENODEV if no rings
+ * are available.
+ * @ctrldev: points to the controller level dev (parent) that
+ * owns rings available for use.
+ * @dev: points to where a pointer to the newly allocated queue's
+ * dev can be written to if successful.
+ **/
+int caam_jr_register(struct device *ctrldev, struct device **rdev)
+{
+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev);
+ struct caam_drv_private_jr *jrpriv = NULL;
+ unsigned long flags;
+ int ring;
+
+ /* Lock, if free ring - assign, unlock */
+ spin_lock_irqsave(&ctrlpriv->jr_alloc_lock, flags);
+ for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) {
+ jrpriv = dev_get_drvdata(ctrlpriv->jrdev[ring]);
+ if (jrpriv->assign == JOBR_UNASSIGNED) {
+ jrpriv->assign = JOBR_ASSIGNED;
+ *rdev = ctrlpriv->jrdev[ring];
+ spin_unlock_irqrestore(&ctrlpriv->jr_alloc_lock, flags);
+ return ring;
+ }
+ }
+
+ /* If assigned, write dev where caller needs it */
+ spin_unlock_irqrestore(&ctrlpriv->jr_alloc_lock, flags);
+ *rdev = NULL;
+
+ return -ENODEV;
+}
+EXPORT_SYMBOL(caam_jr_register);
+
+/**
+ * caam_jr_deregister() - Deregister an API and release the queue.
+ * Returns 0 if OK, -EBUSY if queue still contains pending entries
+ * or unprocessed results at the time of the call
+ * @dev - points to the dev that identifies the queue to
+ * be released.
+ **/
+int caam_jr_deregister(struct device *rdev)
+{
+ struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
+ struct caam_drv_private *ctrlpriv;
+ unsigned long flags;
+
+ /* Get the owning controller's private space */
+ ctrlpriv = dev_get_drvdata(jrpriv->parentdev);
+
+ /*
+ * Make sure ring empty before release
+ */
+ if (rd_reg32(&jrpriv->rregs->outring_used) ||
+ (rd_reg32(&jrpriv->rregs->inpring_avail) != JOBR_DEPTH))
+ return -EBUSY;
+
+ /* Release ring */
+ spin_lock_irqsave(&ctrlpriv->jr_alloc_lock, flags);
+ jrpriv->assign = JOBR_UNASSIGNED;
+ spin_unlock_irqrestore(&ctrlpriv->jr_alloc_lock, flags);
+
+ return 0;
+}
+EXPORT_SYMBOL(caam_jr_deregister);
+
+/**
+ * caam_jr_enqueue() - Enqueue a job descriptor head. Returns 0 if OK,
+ * -EBUSY if the queue is full, -EIO if it cannot map the caller's
+ * descriptor.
+ * @dev: device of the job ring to be used. This device should have
+ * been assigned prior by caam_jr_register().
+ * @desc: points to a job descriptor that execute our request. All
+ * descriptors (and all referenced data) must be in a DMAable
+ * region, and all data references must be physical addresses
+ * accessible to CAAM (i.e. within a PAMU window granted
+ * to it).
+ * @cbk: pointer to a callback function to be invoked upon completion
+ * of this request. This has the form:
+ * callback(struct device *dev, u32 *desc, u32 stat, void *arg)
+ * where:
+ * @dev: contains the job ring device that processed this
+ * response.
+ * @desc: descriptor that initiated the request, same as
+ * "desc" being argued to caam_jr_enqueue().
+ * @status: untranslated status received from CAAM. See the
+ * reference manual for a detailed description of
+ * error meaning, or see the JRSTA definitions in the
+ * register header file
+ * @areq: optional pointer to an argument passed with the
+ * original request
+ * @areq: optional pointer to a user argument for use at callback
+ * time.
+ **/
+int caam_jr_enqueue(struct device *dev, u32 *desc,
+ void (*cbk)(struct device *dev, u32 *desc,
+ u32 status, void *areq),
+ void *areq)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ struct caam_jrentry_info *head_entry;
+ unsigned long flags;
+ int head, tail, desc_size;
+ dma_addr_t desc_dma;
+
+ desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
+ desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, desc_dma)) {
+ dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
+ return -EIO;
+ }
+
+ spin_lock_irqsave(&jrp->inplock, flags);
+
+ head = jrp->head;
+ tail = ACCESS_ONCE(jrp->tail);
+
+ if (!rd_reg32(&jrp->rregs->inpring_avail) ||
+ CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
+ spin_unlock_irqrestore(&jrp->inplock, flags);
+ dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
+ return -EBUSY;
+ }
+
+ head_entry = &jrp->entinfo[head];
+ head_entry->desc_addr_virt = desc;
+ head_entry->desc_size = desc_size;
+ head_entry->callbk = (void *)cbk;
+ head_entry->cbkarg = areq;
+ head_entry->desc_addr_dma = desc_dma;
+
+ jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
+
+ smp_wmb();
+
+ jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
+ (JOBR_DEPTH - 1);
+ jrp->head = (head + 1) & (JOBR_DEPTH - 1);
+
+ wmb();
+
+ wr_reg32(&jrp->rregs->inpring_jobadd, 1);
+
+ spin_unlock_irqrestore(&jrp->inplock, flags);
+
+ return 0;
+}
+EXPORT_SYMBOL(caam_jr_enqueue);
+
+static int caam_reset_hw_jr(struct device *dev)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ unsigned int timeout = 100000;
+
+ /*
+ * mask interrupts since we are going to poll
+ * for reset completion status
+ */
+ setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
+
+ /* initiate flush (required prior to reset) */
+ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
+ while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
+ JRINT_ERR_HALT_INPROGRESS) && --timeout)
+ cpu_relax();
+
+ if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
+ JRINT_ERR_HALT_COMPLETE || timeout == 0) {
+ dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
+ return -EIO;
+ }
+
+ /* initiate reset */
+ timeout = 100000;
+ wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
+ while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
+ cpu_relax();
+
+ if (timeout == 0) {
+ dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
+ return -EIO;
+ }
+
+ /* unmask interrupts */
+ clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
+
+ return 0;
+}
+
+/*
+ * Init JobR independent of platform property detection
+ */
+static int caam_jr_init(struct device *dev)
+{
+ struct caam_drv_private_jr *jrp;
+ dma_addr_t inpbusaddr, outbusaddr;
+ int i, error;
+
+ jrp = dev_get_drvdata(dev);
+
+ /* Connect job ring interrupt handler. */
+ for_each_possible_cpu(i)
+ tasklet_init(&jrp->irqtask[i], caam_jr_dequeue,
+ (unsigned long)dev);
+
+ error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
+ "caam-jobr", dev);
+ if (error) {
+ dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
+ jrp->ridx, jrp->irq);
+ irq_dispose_mapping(jrp->irq);
+ jrp->irq = 0;
+ return -EINVAL;
+ }
+
+ error = caam_reset_hw_jr(dev);
+ if (error)
+ return error;
+
+ jrp->inpring = kzalloc(sizeof(dma_addr_t) * JOBR_DEPTH,
+ GFP_KERNEL | GFP_DMA);
+ jrp->outring = kzalloc(sizeof(struct jr_outentry) *
+ JOBR_DEPTH, GFP_KERNEL | GFP_DMA);
+
+ jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
+ GFP_KERNEL);
+
+ if ((jrp->inpring == NULL) || (jrp->outring == NULL) ||
+ (jrp->entinfo == NULL)) {
+ dev_err(dev, "can't allocate job rings for %d\n",
+ jrp->ridx);
+ return -ENOMEM;
+ }
+
+ for (i = 0; i < JOBR_DEPTH; i++)
+ jrp->entinfo[i].desc_addr_dma = !0;
+
+ /* Setup rings */
+ inpbusaddr = dma_map_single(dev, jrp->inpring,
+ sizeof(u32 *) * JOBR_DEPTH,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, inpbusaddr)) {
+ dev_err(dev, "caam_jr_init(): can't map input ring\n");
+ kfree(jrp->inpring);
+ kfree(jrp->outring);
+ kfree(jrp->entinfo);
+ return -EIO;
+ }
+
+ outbusaddr = dma_map_single(dev, jrp->outring,
+ sizeof(struct jr_outentry) * JOBR_DEPTH,
+ DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(dev, outbusaddr)) {
+ dev_err(dev, "caam_jr_init(): can't map output ring\n");
+ dma_unmap_single(dev, inpbusaddr,
+ sizeof(u32 *) * JOBR_DEPTH,
+ DMA_BIDIRECTIONAL);
+ kfree(jrp->inpring);
+ kfree(jrp->outring);
+ kfree(jrp->entinfo);
+ return -EIO;
+ }
+
+ jrp->inp_ring_write_index = 0;
+ jrp->out_ring_read_index = 0;
+ jrp->head = 0;
+ jrp->tail = 0;
+
+ wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
+ wr_reg64(&jrp->rregs->outring_base, outbusaddr);
+ wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
+ wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
+
+ jrp->ringsize = JOBR_DEPTH;
+
+ spin_lock_init(&jrp->inplock);
+ spin_lock_init(&jrp->outlock);
+
+ /* Select interrupt coalescing parameters */
+ setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
+ (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
+ (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
+
+ jrp->assign = JOBR_UNASSIGNED;
+ return 0;
+}
+
+/*
+ * Shutdown JobR independent of platform property code
+ */
+int caam_jr_shutdown(struct device *dev)
+{
+ struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
+ dma_addr_t inpbusaddr, outbusaddr;
+ int ret, i;
+
+ ret = caam_reset_hw_jr(dev);
+
+ for_each_possible_cpu(i)
+ tasklet_kill(&jrp->irqtask[i]);
+
+ /* Release interrupt */
+ free_irq(jrp->irq, dev);
+
+ /* Free rings */
+ inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
+ outbusaddr = rd_reg64(&jrp->rregs->outring_base);
+ dma_unmap_single(dev, outbusaddr,
+ sizeof(struct jr_outentry) * JOBR_DEPTH,
+ DMA_BIDIRECTIONAL);
+ dma_unmap_single(dev, inpbusaddr, sizeof(u32 *) * JOBR_DEPTH,
+ DMA_BIDIRECTIONAL);
+ kfree(jrp->outring);
+ kfree(jrp->inpring);
+ kfree(jrp->entinfo);
+
+ return ret;
+}
+
+/*
+ * Probe routine for each detected JobR subsystem. It assumes that
+ * property detection was picked up externally.
+ */
+int caam_jr_probe(struct platform_device *pdev, struct device_node *np,
+ int ring)
+{
+ struct device *ctrldev, *jrdev;
+ struct platform_device *jr_pdev;
+ struct caam_drv_private *ctrlpriv;
+ struct caam_drv_private_jr *jrpriv;
+ u32 *jroffset;
+ int error;
+
+ ctrldev = &pdev->dev;
+ ctrlpriv = dev_get_drvdata(ctrldev);
+
+ jrpriv = kmalloc(sizeof(struct caam_drv_private_jr),
+ GFP_KERNEL);
+ if (jrpriv == NULL) {
+ dev_err(ctrldev, "can't alloc private mem for job ring %d\n",
+ ring);
+ return -ENOMEM;
+ }
+ jrpriv->parentdev = ctrldev; /* point back to parent */
+ jrpriv->ridx = ring; /* save ring identity relative to detection */
+
+ /*
+ * Derive a pointer to the detected JobRs regs
+ * Driver has already iomapped the entire space, we just
+ * need to add in the offset to this JobR. Don't know if I
+ * like this long-term, but it'll run
+ */
+ jroffset = (u32 *)of_get_property(np, "reg", NULL);
+ jrpriv->rregs = (struct caam_job_ring __iomem *)((void *)ctrlpriv->ctrl
+ + *jroffset);
+
+ /* Build a local dev for each detected queue */
+ jr_pdev = of_platform_device_create(np, NULL, ctrldev);
+ if (jr_pdev == NULL) {
+ kfree(jrpriv);
+ return -EINVAL;
+ }
+ jrdev = &jr_pdev->dev;
+ dev_set_drvdata(jrdev, jrpriv);
+ ctrlpriv->jrdev[ring] = jrdev;
+
+ /* Identify the interrupt */
+ jrpriv->irq = of_irq_to_resource(np, 0, NULL);
+
+ /* Now do the platform independent part */
+ error = caam_jr_init(jrdev); /* now turn on hardware */
+ if (error) {
+ kfree(jrpriv);
+ return error;
+ }
+
+ return error;
+}
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.h
new file mode 100644
index 0000000..c23df39
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/jr.h
@@ -0,0 +1,21 @@
+/*
+ * CAAM public-level include definitions for the JobR backend
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef JR_H
+#define JR_H
+
+/* Prototypes for backend-level services exposed to APIs */
+int caam_jr_register(struct device *ctrldev, struct device **rdev);
+int caam_jr_deregister(struct device *rdev);
+int caam_jr_enqueue(struct device *dev, u32 *desc,
+ void (*cbk)(struct device *dev, u32 *desc, u32 status,
+ void *areq),
+ void *areq);
+
+extern int caam_jr_probe(struct platform_device *pdev, struct device_node *np,
+ int ring);
+extern int caam_jr_shutdown(struct device *dev);
+#endif /* JR_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/caam/regs.h b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/regs.h
new file mode 100644
index 0000000..e9f7a70
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/caam/regs.h
@@ -0,0 +1,662 @@
+/*
+ * CAAM hardware register-level view
+ *
+ * Copyright 2008-2011 Freescale Semiconductor, Inc.
+ */
+
+#ifndef REGS_H
+#define REGS_H
+
+#include <linux/types.h>
+#include <linux/io.h>
+
+/*
+ * Architecture-specific register access methods
+ *
+ * CAAM's bus-addressable registers are 64 bits internally.
+ * They have been wired to be safely accessible on 32-bit
+ * architectures, however. Registers were organized such
+ * that (a) they can be contained in 32 bits, (b) if not, then they
+ * can be treated as two 32-bit entities, or finally (c) if they
+ * must be treated as a single 64-bit value, then this can safely
+ * be done with two 32-bit cycles.
+ *
+ * For 32-bit operations on 64-bit values, CAAM follows the same
+ * 64-bit register access conventions as it's predecessors, in that
+ * writes are "triggered" by a write to the register at the numerically
+ * higher address, thus, a full 64-bit write cycle requires a write
+ * to the lower address, followed by a write to the higher address,
+ * which will latch/execute the write cycle.
+ *
+ * For example, let's assume a SW reset of CAAM through the master
+ * configuration register.
+ * - SWRST is in bit 31 of MCFG.
+ * - MCFG begins at base+0x0000.
+ * - Bits 63-32 are a 32-bit word at base+0x0000 (numerically-lower)
+ * - Bits 31-0 are a 32-bit word at base+0x0004 (numerically-higher)
+ *
+ * (and on Power, the convention is 0-31, 32-63, I know...)
+ *
+ * Assuming a 64-bit write to this MCFG to perform a software reset
+ * would then require a write of 0 to base+0x0000, followed by a
+ * write of 0x80000000 to base+0x0004, which would "execute" the
+ * reset.
+ *
+ * Of course, since MCFG 63-32 is all zero, we could cheat and simply
+ * write 0x8000000 to base+0x0004, and the reset would work fine.
+ * However, since CAAM does contain some write-and-read-intended
+ * 64-bit registers, this code defines 64-bit access methods for
+ * the sake of internal consistency and simplicity, and so that a
+ * clean transition to 64-bit is possible when it becomes necessary.
+ *
+ * There are limitations to this that the developer must recognize.
+ * 32-bit architectures cannot enforce an atomic-64 operation,
+ * Therefore:
+ *
+ * - On writes, since the HW is assumed to latch the cycle on the
+ * write of the higher-numeric-address word, then ordered
+ * writes work OK.
+ *
+ * - For reads, where a register contains a relevant value of more
+ * that 32 bits, the hardware employs logic to latch the other
+ * "half" of the data until read, ensuring an accurate value.
+ * This is of particular relevance when dealing with CAAM's
+ * performance counters.
+ *
+ */
+
+#ifdef __BIG_ENDIAN
+#define wr_reg32(reg, data) out_be32(reg, data)
+#define rd_reg32(reg) in_be32(reg)
+#ifdef CONFIG_64BIT
+#define wr_reg64(reg, data) out_be64(reg, data)
+#define rd_reg64(reg) in_be64(reg)
+#endif
+#else
+#ifdef __LITTLE_ENDIAN
+#define wr_reg32(reg, data) __raw_writel(reg, data)
+#define rd_reg32(reg) __raw_readl(reg)
+#ifdef CONFIG_64BIT
+#define wr_reg64(reg, data) __raw_writeq(reg, data)
+#define rd_reg64(reg) __raw_readq(reg)
+#endif
+#endif
+#endif
+
+#ifndef CONFIG_64BIT
+static inline void wr_reg64(u64 __iomem *reg, u64 data)
+{
+ wr_reg32((u32 __iomem *)reg, (data & 0xffffffff00000000ull) >> 32);
+ wr_reg32((u32 __iomem *)reg + 1, data & 0x00000000ffffffffull);
+}
+
+static inline u64 rd_reg64(u64 __iomem *reg)
+{
+ return (((u64)rd_reg32((u32 __iomem *)reg)) << 32) |
+ ((u64)rd_reg32((u32 __iomem *)reg + 1));
+}
+#endif
+
+/*
+ * jr_outentry
+ * Represents each entry in a JobR output ring
+ */
+struct jr_outentry {
+ dma_addr_t desc;/* Pointer to completed descriptor */
+ u32 jrstatus; /* Status for completed descriptor */
+} __packed;
+
+/*
+ * caam_perfmon - Performance Monitor/Secure Memory Status/
+ * CAAM Global Status/Component Version IDs
+ *
+ * Spans f00-fff wherever instantiated
+ */
+
+/* Number of DECOs */
+#define CHA_NUM_DECONUM_SHIFT 56
+#define CHA_NUM_DECONUM_MASK (0xfull << CHA_NUM_DECONUM_SHIFT)
+
+struct caam_perfmon {
+ /* Performance Monitor Registers f00-f9f */
+ u64 req_dequeued; /* PC_REQ_DEQ - Dequeued Requests */
+ u64 ob_enc_req; /* PC_OB_ENC_REQ - Outbound Encrypt Requests */
+ u64 ib_dec_req; /* PC_IB_DEC_REQ - Inbound Decrypt Requests */
+ u64 ob_enc_bytes; /* PC_OB_ENCRYPT - Outbound Bytes Encrypted */
+ u64 ob_prot_bytes; /* PC_OB_PROTECT - Outbound Bytes Protected */
+ u64 ib_dec_bytes; /* PC_IB_DECRYPT - Inbound Bytes Decrypted */
+ u64 ib_valid_bytes; /* PC_IB_VALIDATED Inbound Bytes Validated */
+ u64 rsvd[13];
+
+ /* CAAM Hardware Instantiation Parameters fa0-fbf */
+ u64 cha_rev; /* CRNR - CHA Revision Number */
+#define CTPR_QI_SHIFT 57
+#define CTPR_QI_MASK (0x1ull << CTPR_QI_SHIFT)
+ u64 comp_parms; /* CTPR - Compile Parameters Register */
+ u64 rsvd1[2];
+
+ /* CAAM Global Status fc0-fdf */
+ u64 faultaddr; /* FAR - Fault Address */
+ u32 faultliodn; /* FALR - Fault Address LIODN */
+ u32 faultdetail; /* FADR - Fault Addr Detail */
+ u32 rsvd2;
+ u32 status; /* CSTA - CAAM Status */
+ u64 rsvd3;
+
+ /* Component Instantiation Parameters fe0-fff */
+ u32 rtic_id; /* RVID - RTIC Version ID */
+ u32 ccb_id; /* CCBVID - CCB Version ID */
+ u64 cha_id; /* CHAVID - CHA Version ID */
+ u64 cha_num; /* CHANUM - CHA Number */
+ u64 caam_id; /* CAAMVID - CAAM Version ID */
+};
+
+/* LIODN programming for DMA configuration */
+#define MSTRID_LOCK_LIODN 0x80000000
+#define MSTRID_LOCK_MAKETRUSTED 0x00010000 /* only for JR masterid */
+
+#define MSTRID_LIODN_MASK 0x0fff
+struct masterid {
+ u32 liodn_ms; /* lock and make-trusted control bits */
+ u32 liodn_ls; /* LIODN for non-sequence and seq access */
+};
+
+/* Partition ID for DMA configuration */
+struct partid {
+ u32 rsvd1;
+ u32 pidr; /* partition ID, DECO */
+};
+
+/* RNG test mode (replicated twice in some configurations) */
+/* Padded out to 0x100 */
+struct rngtst {
+ u32 mode; /* RTSTMODEx - Test mode */
+ u32 rsvd1[3];
+ u32 reset; /* RTSTRESETx - Test reset control */
+ u32 rsvd2[3];
+ u32 status; /* RTSTSSTATUSx - Test status */
+ u32 rsvd3;
+ u32 errstat; /* RTSTERRSTATx - Test error status */
+ u32 rsvd4;
+ u32 errctl; /* RTSTERRCTLx - Test error control */
+ u32 rsvd5;
+ u32 entropy; /* RTSTENTROPYx - Test entropy */
+ u32 rsvd6[15];
+ u32 verifctl; /* RTSTVERIFCTLx - Test verification control */
+ u32 rsvd7;
+ u32 verifstat; /* RTSTVERIFSTATx - Test verification status */
+ u32 rsvd8;
+ u32 verifdata; /* RTSTVERIFDx - Test verification data */
+ u32 rsvd9;
+ u32 xkey; /* RTSTXKEYx - Test XKEY */
+ u32 rsvd10;
+ u32 oscctctl; /* RTSTOSCCTCTLx - Test osc. counter control */
+ u32 rsvd11;
+ u32 oscct; /* RTSTOSCCTx - Test oscillator counter */
+ u32 rsvd12;
+ u32 oscctstat; /* RTSTODCCTSTATx - Test osc counter status */
+ u32 rsvd13[2];
+ u32 ofifo[4]; /* RTSTOFIFOx - Test output FIFO */
+ u32 rsvd14[15];
+};
+
+/*
+ * caam_ctrl - basic core configuration
+ * starts base + 0x0000 padded out to 0x1000
+ */
+
+#define KEK_KEY_SIZE 8
+#define TKEK_KEY_SIZE 8
+#define TDSK_KEY_SIZE 8
+
+#define DECO_RESET 1 /* Use with DECO reset/availability regs */
+#define DECO_RESET_0 (DECO_RESET << 0)
+#define DECO_RESET_1 (DECO_RESET << 1)
+#define DECO_RESET_2 (DECO_RESET << 2)
+#define DECO_RESET_3 (DECO_RESET << 3)
+#define DECO_RESET_4 (DECO_RESET << 4)
+
+struct caam_ctrl {
+ /* Basic Configuration Section 000-01f */
+ /* Read/Writable */
+ u32 rsvd1;
+ u32 mcr; /* MCFG Master Config Register */
+ u32 rsvd2[2];
+
+ /* Bus Access Configuration Section 010-11f */
+ /* Read/Writable */
+ struct masterid jr_mid[4]; /* JRxLIODNR - JobR LIODN setup */
+ u32 rsvd3[12];
+ struct masterid rtic_mid[4]; /* RTICxLIODNR - RTIC LIODN setup */
+ u32 rsvd4[7];
+ u32 deco_rq; /* DECORR - DECO Request */
+ struct partid deco_mid[5]; /* DECOxLIODNR - 1 per DECO */
+ u32 rsvd5[22];
+
+ /* DECO Availability/Reset Section 120-3ff */
+ u32 deco_avail; /* DAR - DECO availability */
+ u32 deco_reset; /* DRR - DECO reset */
+ u32 rsvd6[182];
+
+ /* Key Encryption/Decryption Configuration 400-5ff */
+ /* Read/Writable only while in Non-secure mode */
+ u32 kek[KEK_KEY_SIZE]; /* JDKEKR - Key Encryption Key */
+ u32 tkek[TKEK_KEY_SIZE]; /* TDKEKR - Trusted Desc KEK */
+ u32 tdsk[TDSK_KEY_SIZE]; /* TDSKR - Trusted Desc Signing Key */
+ u32 rsvd7[32];
+ u64 sknonce; /* SKNR - Secure Key Nonce */
+ u32 rsvd8[70];
+
+ /* RNG Test/Verification/Debug Access 600-7ff */
+ /* (Useful in Test/Debug modes only...) */
+ struct rngtst rtst[2];
+
+ u32 rsvd9[448];
+
+ /* Performance Monitor f00-fff */
+ struct caam_perfmon perfmon;
+};
+
+/*
+ * Controller master config register defs
+ */
+#define MCFGR_SWRESET 0x80000000 /* software reset */
+#define MCFGR_WDENABLE 0x40000000 /* DECO watchdog enable */
+#define MCFGR_WDFAIL 0x20000000 /* DECO watchdog force-fail */
+#define MCFGR_DMA_RESET 0x10000000
+#define MCFGR_LONG_PTR 0x00010000 /* Use >32-bit desc addressing */
+
+/* AXI read cache control */
+#define MCFGR_ARCACHE_SHIFT 12
+#define MCFGR_ARCACHE_MASK (0xf << MCFGR_ARCACHE_SHIFT)
+
+/* AXI write cache control */
+#define MCFGR_AWCACHE_SHIFT 8
+#define MCFGR_AWCACHE_MASK (0xf << MCFGR_AWCACHE_SHIFT)
+
+/* AXI pipeline depth */
+#define MCFGR_AXIPIPE_SHIFT 4
+#define MCFGR_AXIPIPE_MASK (0xf << MCFGR_AXIPIPE_SHIFT)
+
+#define MCFGR_AXIPRI 0x00000008 /* Assert AXI priority sideband */
+#define MCFGR_BURST_64 0x00000001 /* Max burst size */
+
+/*
+ * caam_job_ring - direct job ring setup
+ * 1-4 possible per instantiation, base + 1000/2000/3000/4000
+ * Padded out to 0x1000
+ */
+struct caam_job_ring {
+ /* Input ring */
+ u64 inpring_base; /* IRBAx - Input desc ring baseaddr */
+ u32 rsvd1;
+ u32 inpring_size; /* IRSx - Input ring size */
+ u32 rsvd2;
+ u32 inpring_avail; /* IRSAx - Input ring room remaining */
+ u32 rsvd3;
+ u32 inpring_jobadd; /* IRJAx - Input ring jobs added */
+
+ /* Output Ring */
+ u64 outring_base; /* ORBAx - Output status ring base addr */
+ u32 rsvd4;
+ u32 outring_size; /* ORSx - Output ring size */
+ u32 rsvd5;
+ u32 outring_rmvd; /* ORJRx - Output ring jobs removed */
+ u32 rsvd6;
+ u32 outring_used; /* ORSFx - Output ring slots full */
+
+ /* Status/Configuration */
+ u32 rsvd7;
+ u32 jroutstatus; /* JRSTAx - JobR output status */
+ u32 rsvd8;
+ u32 jrintstatus; /* JRINTx - JobR interrupt status */
+ u32 rconfig_hi; /* JRxCFG - Ring configuration */
+ u32 rconfig_lo;
+
+ /* Indices. CAAM maintains as "heads" of each queue */
+ u32 rsvd9;
+ u32 inp_rdidx; /* IRRIx - Input ring read index */
+ u32 rsvd10;
+ u32 out_wtidx; /* ORWIx - Output ring write index */
+
+ /* Command/control */
+ u32 rsvd11;
+ u32 jrcommand; /* JRCRx - JobR command */
+
+ u32 rsvd12[932];
+
+ /* Performance Monitor f00-fff */
+ struct caam_perfmon perfmon;
+};
+
+#define JR_RINGSIZE_MASK 0x03ff
+/*
+ * jrstatus - Job Ring Output Status
+ * All values in lo word
+ * Also note, same values written out as status through QI
+ * in the command/status field of a frame descriptor
+ */
+#define JRSTA_SSRC_SHIFT 28
+#define JRSTA_SSRC_MASK 0xf0000000
+
+#define JRSTA_SSRC_NONE 0x00000000
+#define JRSTA_SSRC_CCB_ERROR 0x20000000
+#define JRSTA_SSRC_JUMP_HALT_USER 0x30000000
+#define JRSTA_SSRC_DECO 0x40000000
+#define JRSTA_SSRC_JRERROR 0x60000000
+#define JRSTA_SSRC_JUMP_HALT_CC 0x70000000
+
+#define JRSTA_DECOERR_JUMP 0x08000000
+#define JRSTA_DECOERR_INDEX_SHIFT 8
+#define JRSTA_DECOERR_INDEX_MASK 0xff00
+#define JRSTA_DECOERR_ERROR_MASK 0x00ff
+
+#define JRSTA_DECOERR_NONE 0x00
+#define JRSTA_DECOERR_LINKLEN 0x01
+#define JRSTA_DECOERR_LINKPTR 0x02
+#define JRSTA_DECOERR_JRCTRL 0x03
+#define JRSTA_DECOERR_DESCCMD 0x04
+#define JRSTA_DECOERR_ORDER 0x05
+#define JRSTA_DECOERR_KEYCMD 0x06
+#define JRSTA_DECOERR_LOADCMD 0x07
+#define JRSTA_DECOERR_STORECMD 0x08
+#define JRSTA_DECOERR_OPCMD 0x09
+#define JRSTA_DECOERR_FIFOLDCMD 0x0a
+#define JRSTA_DECOERR_FIFOSTCMD 0x0b
+#define JRSTA_DECOERR_MOVECMD 0x0c
+#define JRSTA_DECOERR_JUMPCMD 0x0d
+#define JRSTA_DECOERR_MATHCMD 0x0e
+#define JRSTA_DECOERR_SHASHCMD 0x0f
+#define JRSTA_DECOERR_SEQCMD 0x10
+#define JRSTA_DECOERR_DECOINTERNAL 0x11
+#define JRSTA_DECOERR_SHDESCHDR 0x12
+#define JRSTA_DECOERR_HDRLEN 0x13
+#define JRSTA_DECOERR_BURSTER 0x14
+#define JRSTA_DECOERR_DESCSIGNATURE 0x15
+#define JRSTA_DECOERR_DMA 0x16
+#define JRSTA_DECOERR_BURSTFIFO 0x17
+#define JRSTA_DECOERR_JRRESET 0x1a
+#define JRSTA_DECOERR_JOBFAIL 0x1b
+#define JRSTA_DECOERR_DNRERR 0x80
+#define JRSTA_DECOERR_UNDEFPCL 0x81
+#define JRSTA_DECOERR_PDBERR 0x82
+#define JRSTA_DECOERR_ANRPLY_LATE 0x83
+#define JRSTA_DECOERR_ANRPLY_REPLAY 0x84
+#define JRSTA_DECOERR_SEQOVF 0x85
+#define JRSTA_DECOERR_INVSIGN 0x86
+#define JRSTA_DECOERR_DSASIGN 0x87
+
+#define JRSTA_CCBERR_JUMP 0x08000000
+#define JRSTA_CCBERR_INDEX_MASK 0xff00
+#define JRSTA_CCBERR_INDEX_SHIFT 8
+#define JRSTA_CCBERR_CHAID_MASK 0x00f0
+#define JRSTA_CCBERR_CHAID_SHIFT 4
+#define JRSTA_CCBERR_ERRID_MASK 0x000f
+
+#define JRSTA_CCBERR_CHAID_AES (0x01 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_DES (0x02 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_ARC4 (0x03 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_MD (0x04 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_RNG (0x05 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_SNOW (0x06 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_KASUMI (0x07 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_PK (0x08 << JRSTA_CCBERR_CHAID_SHIFT)
+#define JRSTA_CCBERR_CHAID_CRC (0x09 << JRSTA_CCBERR_CHAID_SHIFT)
+
+#define JRSTA_CCBERR_ERRID_NONE 0x00
+#define JRSTA_CCBERR_ERRID_MODE 0x01
+#define JRSTA_CCBERR_ERRID_DATASIZ 0x02
+#define JRSTA_CCBERR_ERRID_KEYSIZ 0x03
+#define JRSTA_CCBERR_ERRID_PKAMEMSZ 0x04
+#define JRSTA_CCBERR_ERRID_PKBMEMSZ 0x05
+#define JRSTA_CCBERR_ERRID_SEQUENCE 0x06
+#define JRSTA_CCBERR_ERRID_PKDIVZRO 0x07
+#define JRSTA_CCBERR_ERRID_PKMODEVN 0x08
+#define JRSTA_CCBERR_ERRID_KEYPARIT 0x09
+#define JRSTA_CCBERR_ERRID_ICVCHK 0x0a
+#define JRSTA_CCBERR_ERRID_HARDWARE 0x0b
+#define JRSTA_CCBERR_ERRID_CCMAAD 0x0c
+#define JRSTA_CCBERR_ERRID_INVCHA 0x0f
+
+#define JRINT_ERR_INDEX_MASK 0x3fff0000
+#define JRINT_ERR_INDEX_SHIFT 16
+#define JRINT_ERR_TYPE_MASK 0xf00
+#define JRINT_ERR_TYPE_SHIFT 8
+#define JRINT_ERR_HALT_MASK 0xc
+#define JRINT_ERR_HALT_SHIFT 2
+#define JRINT_ERR_HALT_INPROGRESS 0x4
+#define JRINT_ERR_HALT_COMPLETE 0x8
+#define JRINT_JR_ERROR 0x02
+#define JRINT_JR_INT 0x01
+
+#define JRINT_ERR_TYPE_WRITE 1
+#define JRINT_ERR_TYPE_BAD_INPADDR 3
+#define JRINT_ERR_TYPE_BAD_OUTADDR 4
+#define JRINT_ERR_TYPE_INV_INPWRT 5
+#define JRINT_ERR_TYPE_INV_OUTWRT 6
+#define JRINT_ERR_TYPE_RESET 7
+#define JRINT_ERR_TYPE_REMOVE_OFL 8
+#define JRINT_ERR_TYPE_ADD_OFL 9
+
+#define JRCFG_SOE 0x04
+#define JRCFG_ICEN 0x02
+#define JRCFG_IMSK 0x01
+#define JRCFG_ICDCT_SHIFT 8
+#define JRCFG_ICTT_SHIFT 16
+
+#define JRCR_RESET 0x01
+
+/*
+ * caam_assurance - Assurance Controller View
+ * base + 0x6000 padded out to 0x1000
+ */
+
+struct rtic_element {
+ u64 address;
+ u32 rsvd;
+ u32 length;
+};
+
+struct rtic_block {
+ struct rtic_element element[2];
+};
+
+struct rtic_memhash {
+ u32 memhash_be[32];
+ u32 memhash_le[32];
+};
+
+struct caam_assurance {
+ /* Status/Command/Watchdog */
+ u32 rsvd1;
+ u32 status; /* RSTA - Status */
+ u32 rsvd2;
+ u32 cmd; /* RCMD - Command */
+ u32 rsvd3;
+ u32 ctrl; /* RCTL - Control */
+ u32 rsvd4;
+ u32 throttle; /* RTHR - Throttle */
+ u32 rsvd5[2];
+ u64 watchdog; /* RWDOG - Watchdog Timer */
+ u32 rsvd6;
+ u32 rend; /* REND - Endian corrections */
+ u32 rsvd7[50];
+
+ /* Block access/configuration @ 100/110/120/130 */
+ struct rtic_block memblk[4]; /* Memory Blocks A-D */
+ u32 rsvd8[32];
+
+ /* Block hashes @ 200/300/400/500 */
+ struct rtic_memhash hash[4]; /* Block hash values A-D */
+ u32 rsvd_3[640];
+};
+
+/*
+ * caam_queue_if - QI configuration and control
+ * starts base + 0x7000, padded out to 0x1000 long
+ */
+
+struct caam_queue_if {
+ u32 qi_control_hi; /* QICTL - QI Control */
+ u32 qi_control_lo;
+ u32 rsvd1;
+ u32 qi_status; /* QISTA - QI Status */
+ u32 qi_deq_cfg_hi; /* QIDQC - QI Dequeue Configuration */
+ u32 qi_deq_cfg_lo;
+ u32 qi_enq_cfg_hi; /* QISEQC - QI Enqueue Command */
+ u32 qi_enq_cfg_lo;
+ u32 rsvd2[1016];
+};
+
+/* QI control bits - low word */
+#define QICTL_DQEN 0x01 /* Enable frame pop */
+#define QICTL_STOP 0x02 /* Stop dequeue/enqueue */
+#define QICTL_SOE 0x04 /* Stop on error */
+
+/* QI control bits - high word */
+#define QICTL_MBSI 0x01
+#define QICTL_MHWSI 0x02
+#define QICTL_MWSI 0x04
+#define QICTL_MDWSI 0x08
+#define QICTL_CBSI 0x10 /* CtrlDataByteSwapInput */
+#define QICTL_CHWSI 0x20 /* CtrlDataHalfSwapInput */
+#define QICTL_CWSI 0x40 /* CtrlDataWordSwapInput */
+#define QICTL_CDWSI 0x80 /* CtrlDataDWordSwapInput */
+#define QICTL_MBSO 0x0100
+#define QICTL_MHWSO 0x0200
+#define QICTL_MWSO 0x0400
+#define QICTL_MDWSO 0x0800
+#define QICTL_CBSO 0x1000 /* CtrlDataByteSwapOutput */
+#define QICTL_CHWSO 0x2000 /* CtrlDataHalfSwapOutput */
+#define QICTL_CWSO 0x4000 /* CtrlDataWordSwapOutput */
+#define QICTL_CDWSO 0x8000 /* CtrlDataDWordSwapOutput */
+#define QICTL_DMBS 0x010000
+#define QICTL_EPO 0x020000
+
+/* QI status bits */
+#define QISTA_PHRDERR 0x01 /* PreHeader Read Error */
+#define QISTA_CFRDERR 0x02 /* Compound Frame Read Error */
+#define QISTA_OFWRERR 0x04 /* Output Frame Read Error */
+#define QISTA_BPDERR 0x08 /* Buffer Pool Depleted */
+#define QISTA_BTSERR 0x10 /* Buffer Undersize */
+#define QISTA_CFWRERR 0x20 /* Compound Frame Write Err */
+#define QISTA_STOPD 0x80000000 /* QI Stopped (see QICTL) */
+
+/* deco_sg_table - DECO view of scatter/gather table */
+struct deco_sg_table {
+ u64 addr; /* Segment Address */
+ u32 elen; /* E, F bits + 30-bit length */
+ u32 bpid_offset; /* Buffer Pool ID + 16-bit length */
+};
+
+/*
+ * caam_deco - descriptor controller - CHA cluster block
+ *
+ * Only accessible when direct DECO access is turned on
+ * (done in DECORR, via MID programmed in DECOxMID
+ *
+ * 5 typical, base + 0x8000/9000/a000/b000
+ * Padded out to 0x1000 long
+ */
+struct caam_deco {
+ u32 rsvd1;
+ u32 cls1_mode; /* CxC1MR - Class 1 Mode */
+ u32 rsvd2;
+ u32 cls1_keysize; /* CxC1KSR - Class 1 Key Size */
+ u32 cls1_datasize_hi; /* CxC1DSR - Class 1 Data Size */
+ u32 cls1_datasize_lo;
+ u32 rsvd3;
+ u32 cls1_icvsize; /* CxC1ICVSR - Class 1 ICV size */
+ u32 rsvd4[5];
+ u32 cha_ctrl; /* CCTLR - CHA control */
+ u32 rsvd5;
+ u32 irq_crtl; /* CxCIRQ - CCB interrupt done/error/clear */
+ u32 rsvd6;
+ u32 clr_written; /* CxCWR - Clear-Written */
+ u32 ccb_status_hi; /* CxCSTA - CCB Status/Error */
+ u32 ccb_status_lo;
+ u32 rsvd7[3];
+ u32 aad_size; /* CxAADSZR - Current AAD Size */
+ u32 rsvd8;
+ u32 cls1_iv_size; /* CxC1IVSZR - Current Class 1 IV Size */
+ u32 rsvd9[7];
+ u32 pkha_a_size; /* PKASZRx - Size of PKHA A */
+ u32 rsvd10;
+ u32 pkha_b_size; /* PKBSZRx - Size of PKHA B */
+ u32 rsvd11;
+ u32 pkha_n_size; /* PKNSZRx - Size of PKHA N */
+ u32 rsvd12;
+ u32 pkha_e_size; /* PKESZRx - Size of PKHA E */
+ u32 rsvd13[24];
+ u32 cls1_ctx[16]; /* CxC1CTXR - Class 1 Context @100 */
+ u32 rsvd14[48];
+ u32 cls1_key[8]; /* CxC1KEYR - Class 1 Key @200 */
+ u32 rsvd15[121];
+ u32 cls2_mode; /* CxC2MR - Class 2 Mode */
+ u32 rsvd16;
+ u32 cls2_keysize; /* CxX2KSR - Class 2 Key Size */
+ u32 cls2_datasize_hi; /* CxC2DSR - Class 2 Data Size */
+ u32 cls2_datasize_lo;
+ u32 rsvd17;
+ u32 cls2_icvsize; /* CxC2ICVSZR - Class 2 ICV Size */
+ u32 rsvd18[56];
+ u32 cls2_ctx[18]; /* CxC2CTXR - Class 2 Context @500 */
+ u32 rsvd19[46];
+ u32 cls2_key[32]; /* CxC2KEYR - Class2 Key @600 */
+ u32 rsvd20[84];
+ u32 inp_infofifo_hi; /* CxIFIFO - Input Info FIFO @7d0 */
+ u32 inp_infofifo_lo;
+ u32 rsvd21[2];
+ u64 inp_datafifo; /* CxDFIFO - Input Data FIFO */
+ u32 rsvd22[2];
+ u64 out_datafifo; /* CxOFIFO - Output Data FIFO */
+ u32 rsvd23[2];
+ u32 jr_ctl_hi; /* CxJRR - JobR Control Register @800 */
+ u32 jr_ctl_lo;
+ u64 jr_descaddr; /* CxDADR - JobR Descriptor Address */
+ u32 op_status_hi; /* DxOPSTA - DECO Operation Status */
+ u32 op_status_lo;
+ u32 rsvd24[2];
+ u32 liodn; /* DxLSR - DECO LIODN Status - non-seq */
+ u32 td_liodn; /* DxLSR - DECO LIODN Status - trustdesc */
+ u32 rsvd26[6];
+ u64 math[4]; /* DxMTH - Math register */
+ u32 rsvd27[8];
+ struct deco_sg_table gthr_tbl[4]; /* DxGTR - Gather Tables */
+ u32 rsvd28[16];
+ struct deco_sg_table sctr_tbl[4]; /* DxSTR - Scatter Tables */
+ u32 rsvd29[48];
+ u32 descbuf[64]; /* DxDESB - Descriptor buffer */
+ u32 rsvd30[320];
+};
+
+/*
+ * Current top-level view of memory map is:
+ *
+ * 0x0000 - 0x0fff - CAAM Top-Level Control
+ * 0x1000 - 0x1fff - Job Ring 0
+ * 0x2000 - 0x2fff - Job Ring 1
+ * 0x3000 - 0x3fff - Job Ring 2
+ * 0x4000 - 0x4fff - Job Ring 3
+ * 0x5000 - 0x5fff - (unused)
+ * 0x6000 - 0x6fff - Assurance Controller
+ * 0x7000 - 0x7fff - Queue Interface
+ * 0x8000 - 0x8fff - DECO-CCB 0
+ * 0x9000 - 0x9fff - DECO-CCB 1
+ * 0xa000 - 0xafff - DECO-CCB 2
+ * 0xb000 - 0xbfff - DECO-CCB 3
+ * 0xc000 - 0xcfff - DECO-CCB 4
+ *
+ * caam_full describes the full register view of CAAM if useful,
+ * although many configurations may choose to implement parts of
+ * the register map separately, in differing privilege regions
+ */
+struct caam_full {
+ struct caam_ctrl __iomem ctrl;
+ struct caam_job_ring jr[4];
+ u64 rsvd[512];
+ struct caam_assurance assure;
+ struct caam_queue_if qi;
+};
+
+#endif /* REGS_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.c b/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.c
new file mode 100644
index 0000000..f3e36c8
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.c
@@ -0,0 +1,608 @@
+ /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/pci_ids.h>
+#include <linux/crypto.h>
+#include <linux/spinlock.h>
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+
+#include <linux/io.h>
+#include <linux/delay.h>
+
+#include "geode-aes.h"
+
+/* Static structures */
+
+static void __iomem *_iobase;
+static spinlock_t lock;
+
+/* Write a 128 bit field (either a writable key or IV) */
+static inline void
+_writefield(u32 offset, void *value)
+{
+ int i;
+ for (i = 0; i < 4; i++)
+ iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
+}
+
+/* Read a 128 bit field (either a writable key or IV) */
+static inline void
+_readfield(u32 offset, void *value)
+{
+ int i;
+ for (i = 0; i < 4; i++)
+ ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
+}
+
+static int
+do_crypt(void *src, void *dst, int len, u32 flags)
+{
+ u32 status;
+ u32 counter = AES_OP_TIMEOUT;
+
+ iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
+ iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
+ iowrite32(len, _iobase + AES_LENA_REG);
+
+ /* Start the operation */
+ iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
+
+ do {
+ status = ioread32(_iobase + AES_INTR_REG);
+ cpu_relax();
+ } while (!(status & AES_INTRA_PENDING) && --counter);
+
+ /* Clear the event */
+ iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
+ return counter ? 0 : 1;
+}
+
+static unsigned int
+geode_aes_crypt(struct geode_aes_op *op)
+{
+ u32 flags = 0;
+ unsigned long iflags;
+ int ret;
+
+ if (op->len == 0)
+ return 0;
+
+ /* If the source and destination is the same, then
+ * we need to turn on the coherent flags, otherwise
+ * we don't need to worry
+ */
+
+ flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
+
+ if (op->dir == AES_DIR_ENCRYPT)
+ flags |= AES_CTRL_ENCRYPT;
+
+ /* Start the critical section */
+
+ spin_lock_irqsave(&lock, iflags);
+
+ if (op->mode == AES_MODE_CBC) {
+ flags |= AES_CTRL_CBC;
+ _writefield(AES_WRITEIV0_REG, op->iv);
+ }
+
+ if (!(op->flags & AES_FLAGS_HIDDENKEY)) {
+ flags |= AES_CTRL_WRKEY;
+ _writefield(AES_WRITEKEY0_REG, op->key);
+ }
+
+ ret = do_crypt(op->src, op->dst, op->len, flags);
+ BUG_ON(ret);
+
+ if (op->mode == AES_MODE_CBC)
+ _readfield(AES_WRITEIV0_REG, op->iv);
+
+ spin_unlock_irqrestore(&lock, iflags);
+
+ return op->len;
+}
+
+/* CRYPTO-API Functions */
+
+static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int len)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+ unsigned int ret;
+
+ op->keylen = len;
+
+ if (len == AES_KEYSIZE_128) {
+ memcpy(op->key, key, len);
+ return 0;
+ }
+
+ if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
+ /* not supported at all */
+ tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+
+ /*
+ * The requested key size is not supported by HW, do a fallback
+ */
+ op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+ op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
+
+ ret = crypto_cipher_setkey(op->fallback.cip, key, len);
+ if (ret) {
+ tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+ tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK);
+ }
+ return ret;
+}
+
+static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int len)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+ unsigned int ret;
+
+ op->keylen = len;
+
+ if (len == AES_KEYSIZE_128) {
+ memcpy(op->key, key, len);
+ return 0;
+ }
+
+ if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
+ /* not supported at all */
+ tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+
+ /*
+ * The requested key size is not supported by HW, do a fallback
+ */
+ op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+ op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
+
+ ret = crypto_blkcipher_setkey(op->fallback.blk, key, len);
+ if (ret) {
+ tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+ tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
+ }
+ return ret;
+}
+
+static int fallback_blk_dec(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ unsigned int ret;
+ struct crypto_blkcipher *tfm;
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+
+ tfm = desc->tfm;
+ desc->tfm = op->fallback.blk;
+
+ ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
+
+ desc->tfm = tfm;
+ return ret;
+}
+static int fallback_blk_enc(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ unsigned int ret;
+ struct crypto_blkcipher *tfm;
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+
+ tfm = desc->tfm;
+ desc->tfm = op->fallback.blk;
+
+ ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
+
+ desc->tfm = tfm;
+ return ret;
+}
+
+static void
+geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128)) {
+ crypto_cipher_encrypt_one(op->fallback.cip, out, in);
+ return;
+ }
+
+ op->src = (void *) in;
+ op->dst = (void *) out;
+ op->mode = AES_MODE_ECB;
+ op->flags = 0;
+ op->len = AES_MIN_BLOCK_SIZE;
+ op->dir = AES_DIR_ENCRYPT;
+
+ geode_aes_crypt(op);
+}
+
+
+static void
+geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128)) {
+ crypto_cipher_decrypt_one(op->fallback.cip, out, in);
+ return;
+ }
+
+ op->src = (void *) in;
+ op->dst = (void *) out;
+ op->mode = AES_MODE_ECB;
+ op->flags = 0;
+ op->len = AES_MIN_BLOCK_SIZE;
+ op->dir = AES_DIR_DECRYPT;
+
+ geode_aes_crypt(op);
+}
+
+static int fallback_init_cip(struct crypto_tfm *tfm)
+{
+ const char *name = tfm->__crt_alg->cra_name;
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ op->fallback.cip = crypto_alloc_cipher(name, 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+
+ if (IS_ERR(op->fallback.cip)) {
+ printk(KERN_ERR "Error allocating fallback algo %s\n", name);
+ return PTR_ERR(op->fallback.cip);
+ }
+
+ return 0;
+}
+
+static void fallback_exit_cip(struct crypto_tfm *tfm)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ crypto_free_cipher(op->fallback.cip);
+ op->fallback.cip = NULL;
+}
+
+static struct crypto_alg geode_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "geode-aes",
+ .cra_priority = 300,
+ .cra_alignmask = 15,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_init = fallback_init_cip,
+ .cra_exit = fallback_exit_cip,
+ .cra_blocksize = AES_MIN_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct geode_aes_op),
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(geode_alg.cra_list),
+ .cra_u = {
+ .cipher = {
+ .cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cia_setkey = geode_setkey_cip,
+ .cia_encrypt = geode_encrypt,
+ .cia_decrypt = geode_decrypt
+ }
+ }
+};
+
+static int
+geode_cbc_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err, ret;
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128))
+ return fallback_blk_dec(desc, dst, src, nbytes);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+ op->iv = walk.iv;
+
+ while ((nbytes = walk.nbytes)) {
+ op->src = walk.src.virt.addr,
+ op->dst = walk.dst.virt.addr;
+ op->mode = AES_MODE_CBC;
+ op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+ op->dir = AES_DIR_DECRYPT;
+
+ ret = geode_aes_crypt(op);
+
+ nbytes -= ret;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static int
+geode_cbc_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err, ret;
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128))
+ return fallback_blk_enc(desc, dst, src, nbytes);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+ op->iv = walk.iv;
+
+ while ((nbytes = walk.nbytes)) {
+ op->src = walk.src.virt.addr,
+ op->dst = walk.dst.virt.addr;
+ op->mode = AES_MODE_CBC;
+ op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+ op->dir = AES_DIR_ENCRYPT;
+
+ ret = geode_aes_crypt(op);
+ nbytes -= ret;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static int fallback_init_blk(struct crypto_tfm *tfm)
+{
+ const char *name = tfm->__crt_alg->cra_name;
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ op->fallback.blk = crypto_alloc_blkcipher(name, 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+
+ if (IS_ERR(op->fallback.blk)) {
+ printk(KERN_ERR "Error allocating fallback algo %s\n", name);
+ return PTR_ERR(op->fallback.blk);
+ }
+
+ return 0;
+}
+
+static void fallback_exit_blk(struct crypto_tfm *tfm)
+{
+ struct geode_aes_op *op = crypto_tfm_ctx(tfm);
+
+ crypto_free_blkcipher(op->fallback.blk);
+ op->fallback.blk = NULL;
+}
+
+static struct crypto_alg geode_cbc_alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-geode",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_init = fallback_init_blk,
+ .cra_exit = fallback_exit_blk,
+ .cra_blocksize = AES_MIN_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct geode_aes_op),
+ .cra_alignmask = 15,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(geode_cbc_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = geode_setkey_blk,
+ .encrypt = geode_cbc_encrypt,
+ .decrypt = geode_cbc_decrypt,
+ .ivsize = AES_IV_LENGTH,
+ }
+ }
+};
+
+static int
+geode_ecb_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err, ret;
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128))
+ return fallback_blk_dec(desc, dst, src, nbytes);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ op->src = walk.src.virt.addr,
+ op->dst = walk.dst.virt.addr;
+ op->mode = AES_MODE_ECB;
+ op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+ op->dir = AES_DIR_DECRYPT;
+
+ ret = geode_aes_crypt(op);
+ nbytes -= ret;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static int
+geode_ecb_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err, ret;
+
+ if (unlikely(op->keylen != AES_KEYSIZE_128))
+ return fallback_blk_enc(desc, dst, src, nbytes);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ op->src = walk.src.virt.addr,
+ op->dst = walk.dst.virt.addr;
+ op->mode = AES_MODE_ECB;
+ op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
+ op->dir = AES_DIR_ENCRYPT;
+
+ ret = geode_aes_crypt(op);
+ nbytes -= ret;
+ ret = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+
+static struct crypto_alg geode_ecb_alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-geode",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_init = fallback_init_blk,
+ .cra_exit = fallback_exit_blk,
+ .cra_blocksize = AES_MIN_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct geode_aes_op),
+ .cra_alignmask = 15,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(geode_ecb_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = geode_setkey_blk,
+ .encrypt = geode_ecb_encrypt,
+ .decrypt = geode_ecb_decrypt,
+ }
+ }
+};
+
+static void __devexit
+geode_aes_remove(struct pci_dev *dev)
+{
+ crypto_unregister_alg(&geode_alg);
+ crypto_unregister_alg(&geode_ecb_alg);
+ crypto_unregister_alg(&geode_cbc_alg);
+
+ pci_iounmap(dev, _iobase);
+ _iobase = NULL;
+
+ pci_release_regions(dev);
+ pci_disable_device(dev);
+}
+
+
+static int __devinit
+geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
+{
+ int ret;
+ ret = pci_enable_device(dev);
+ if (ret)
+ return ret;
+
+ ret = pci_request_regions(dev, "geode-aes");
+ if (ret)
+ goto eenable;
+
+ _iobase = pci_iomap(dev, 0, 0);
+
+ if (_iobase == NULL) {
+ ret = -ENOMEM;
+ goto erequest;
+ }
+
+ spin_lock_init(&lock);
+
+ /* Clear any pending activity */
+ iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
+
+ ret = crypto_register_alg(&geode_alg);
+ if (ret)
+ goto eiomap;
+
+ ret = crypto_register_alg(&geode_ecb_alg);
+ if (ret)
+ goto ealg;
+
+ ret = crypto_register_alg(&geode_cbc_alg);
+ if (ret)
+ goto eecb;
+
+ printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
+ return 0;
+
+ eecb:
+ crypto_unregister_alg(&geode_ecb_alg);
+
+ ealg:
+ crypto_unregister_alg(&geode_alg);
+
+ eiomap:
+ pci_iounmap(dev, _iobase);
+
+ erequest:
+ pci_release_regions(dev);
+
+ eenable:
+ pci_disable_device(dev);
+
+ printk(KERN_ERR "geode-aes: GEODE AES initialization failed.\n");
+ return ret;
+}
+
+static struct pci_device_id geode_aes_tbl[] = {
+ { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), } ,
+ { 0, }
+};
+
+MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
+
+static struct pci_driver geode_aes_driver = {
+ .name = "Geode LX AES",
+ .id_table = geode_aes_tbl,
+ .probe = geode_aes_probe,
+ .remove = __devexit_p(geode_aes_remove)
+};
+
+static int __init
+geode_aes_init(void)
+{
+ return pci_register_driver(&geode_aes_driver);
+}
+
+static void __exit
+geode_aes_exit(void)
+{
+ pci_unregister_driver(&geode_aes_driver);
+}
+
+MODULE_AUTHOR("Advanced Micro Devices, Inc.");
+MODULE_DESCRIPTION("Geode LX Hardware AES driver");
+MODULE_LICENSE("GPL");
+
+module_init(geode_aes_init);
+module_exit(geode_aes_exit);
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.h b/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.h
new file mode 100644
index 0000000..f1855b5
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/geode-aes.h
@@ -0,0 +1,77 @@
+/* Copyright (C) 2003-2006, Advanced Micro Devices, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#ifndef _GEODE_AES_H_
+#define _GEODE_AES_H_
+
+/* driver logic flags */
+#define AES_IV_LENGTH 16
+#define AES_KEY_LENGTH 16
+#define AES_MIN_BLOCK_SIZE 16
+
+#define AES_MODE_ECB 0
+#define AES_MODE_CBC 1
+
+#define AES_DIR_DECRYPT 0
+#define AES_DIR_ENCRYPT 1
+
+#define AES_FLAGS_HIDDENKEY (1 << 0)
+
+/* Register definitions */
+
+#define AES_CTRLA_REG 0x0000
+
+#define AES_CTRL_START 0x01
+#define AES_CTRL_DECRYPT 0x00
+#define AES_CTRL_ENCRYPT 0x02
+#define AES_CTRL_WRKEY 0x04
+#define AES_CTRL_DCA 0x08
+#define AES_CTRL_SCA 0x10
+#define AES_CTRL_CBC 0x20
+
+#define AES_INTR_REG 0x0008
+
+#define AES_INTRA_PENDING (1 << 16)
+#define AES_INTRB_PENDING (1 << 17)
+
+#define AES_INTR_PENDING (AES_INTRA_PENDING | AES_INTRB_PENDING)
+#define AES_INTR_MASK 0x07
+
+#define AES_SOURCEA_REG 0x0010
+#define AES_DSTA_REG 0x0014
+#define AES_LENA_REG 0x0018
+#define AES_WRITEKEY0_REG 0x0030
+#define AES_WRITEIV0_REG 0x0040
+
+/* A very large counter that is used to gracefully bail out of an
+ * operation in case of trouble
+ */
+
+#define AES_OP_TIMEOUT 0x50000
+
+struct geode_aes_op {
+
+ void *src;
+ void *dst;
+
+ u32 mode;
+ u32 dir;
+ u32 flags;
+ int len;
+
+ u8 key[AES_KEY_LENGTH];
+ u8 *iv;
+
+ union {
+ struct crypto_blkcipher *blk;
+ struct crypto_cipher *cip;
+ } fallback;
+ u32 keylen;
+};
+
+#endif
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/hifn_795x.c b/ap/os/linux/linux-3.4.x/drivers/crypto/hifn_795x.c
new file mode 100644
index 0000000..c9c4bef
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/hifn_795x.c
@@ -0,0 +1,2799 @@
+/*
+ * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/mod_devicetable.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
+#include <linux/highmem.h>
+#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/ktime.h>
+
+#include <crypto/algapi.h>
+#include <crypto/des.h>
+
+#include <asm/kmap_types.h>
+
+//#define HIFN_DEBUG
+
+#ifdef HIFN_DEBUG
+#define dprintk(f, a...) printk(f, ##a)
+#else
+#define dprintk(f, a...) do {} while (0)
+#endif
+
+static char hifn_pll_ref[sizeof("extNNN")] = "ext";
+module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
+MODULE_PARM_DESC(hifn_pll_ref,
+ "PLL reference clock (pci[freq] or ext[freq], default ext)");
+
+static atomic_t hifn_dev_number;
+
+#define ACRYPTO_OP_DECRYPT 0
+#define ACRYPTO_OP_ENCRYPT 1
+#define ACRYPTO_OP_HMAC 2
+#define ACRYPTO_OP_RNG 3
+
+#define ACRYPTO_MODE_ECB 0
+#define ACRYPTO_MODE_CBC 1
+#define ACRYPTO_MODE_CFB 2
+#define ACRYPTO_MODE_OFB 3
+
+#define ACRYPTO_TYPE_AES_128 0
+#define ACRYPTO_TYPE_AES_192 1
+#define ACRYPTO_TYPE_AES_256 2
+#define ACRYPTO_TYPE_3DES 3
+#define ACRYPTO_TYPE_DES 4
+
+#define PCI_VENDOR_ID_HIFN 0x13A3
+#define PCI_DEVICE_ID_HIFN_7955 0x0020
+#define PCI_DEVICE_ID_HIFN_7956 0x001d
+
+/* I/O region sizes */
+
+#define HIFN_BAR0_SIZE 0x1000
+#define HIFN_BAR1_SIZE 0x2000
+#define HIFN_BAR2_SIZE 0x8000
+
+/* DMA registres */
+
+#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */
+#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */
+#define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */
+#define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */
+#define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */
+#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */
+#define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */
+#define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */
+#define HIFN_CHIP_ID 0x98 /* Chip ID */
+
+/*
+ * Processing Unit Registers (offset from BASEREG0)
+ */
+#define HIFN_0_PUDATA 0x00 /* Processing Unit Data */
+#define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */
+#define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */
+#define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */
+#define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */
+#define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */
+#define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */
+#define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */
+#define HIFN_0_SPACESIZE 0x20 /* Register space size */
+
+/* Processing Unit Control Register (HIFN_0_PUCTRL) */
+#define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */
+#define HIFN_PUCTRL_STOP 0x0008 /* stop pu */
+#define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */
+#define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */
+#define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */
+
+/* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
+#define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */
+#define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */
+#define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
+#define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
+#define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */
+#define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */
+#define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */
+#define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */
+#define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */
+#define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */
+
+/* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
+#define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */
+#define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */
+#define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */
+#define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */
+#define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */
+#define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */
+#define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */
+#define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */
+#define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */
+#define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */
+#define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */
+#define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */
+#define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */
+#define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */
+#define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */
+#define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */
+#define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */
+#define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */
+#define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */
+#define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */
+#define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */
+#define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */
+#define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */
+
+/* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
+#define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */
+#define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */
+#define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
+#define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
+#define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */
+#define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */
+#define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */
+#define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */
+#define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */
+#define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */
+
+/* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
+#define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */
+#define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */
+#define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
+#define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
+#define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */
+#define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */
+#define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */
+#define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */
+#define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */
+#define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */
+#define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */
+#define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */
+#define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */
+#define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */
+#define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */
+#define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */
+#define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */
+
+/* FIFO Status Register (HIFN_0_FIFOSTAT) */
+#define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */
+#define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */
+
+/* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
+#define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */
+
+/*
+ * DMA Interface Registers (offset from BASEREG1)
+ */
+#define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */
+#define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */
+#define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */
+#define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */
+#define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */
+#define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */
+#define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */
+#define HIFN_1_PLL 0x4c /* 795x: PLL config */
+#define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */
+#define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */
+#define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */
+#define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */
+#define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */
+#define HIFN_1_REVID 0x98 /* Revision ID */
+#define HIFN_1_UNLOCK_SECRET1 0xf4
+#define HIFN_1_UNLOCK_SECRET2 0xfc
+#define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */
+#define HIFN_1_PUB_BASE 0x300 /* Public Base Address */
+#define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */
+#define HIFN_1_PUB_OP 0x308 /* Public Operand */
+#define HIFN_1_PUB_STATUS 0x30c /* Public Status */
+#define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */
+#define HIFN_1_RNG_CONFIG 0x314 /* RNG config */
+#define HIFN_1_RNG_DATA 0x318 /* RNG data */
+#define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */
+#define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */
+
+/* DMA Status and Control Register (HIFN_1_DMA_CSR) */
+#define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */
+#define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */
+#define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */
+#define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */
+#define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */
+#define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */
+#define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */
+#define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */
+#define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */
+#define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */
+#define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */
+#define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */
+#define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */
+#define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */
+#define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */
+#define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */
+#define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */
+#define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */
+#define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */
+#define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */
+#define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */
+#define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */
+#define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */
+#define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */
+#define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */
+#define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */
+#define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */
+#define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */
+#define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */
+#define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */
+#define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */
+#define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */
+#define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */
+#define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */
+#define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */
+#define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */
+#define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */
+#define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */
+
+/* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
+#define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */
+#define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */
+#define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */
+#define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */
+#define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */
+#define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */
+#define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */
+#define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */
+#define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */
+#define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */
+#define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */
+#define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */
+#define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */
+#define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */
+#define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */
+#define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */
+#define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */
+#define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */
+#define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */
+#define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */
+#define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */
+#define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */
+
+/* DMA Configuration Register (HIFN_1_DMA_CNFG) */
+#define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */
+#define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */
+#define HIFN_DMACNFG_UNLOCK 0x00000800
+#define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */
+#define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */
+#define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */
+#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
+#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
+
+/* PLL configuration register */
+#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
+#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
+#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
+#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
+#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
+#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
+#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
+#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
+#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
+#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
+#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
+#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
+#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
+#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
+#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
+#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
+#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
+
+#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
+
+/* Public key reset register (HIFN_1_PUB_RESET) */
+#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
+
+/* Public base address register (HIFN_1_PUB_BASE) */
+#define HIFN_PUBBASE_ADDR 0x00003fff /* base address */
+
+/* Public operand length register (HIFN_1_PUB_OPLEN) */
+#define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */
+#define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */
+#define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */
+#define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */
+#define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */
+#define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */
+
+/* Public operation register (HIFN_1_PUB_OP) */
+#define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */
+#define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */
+#define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */
+#define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */
+#define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */
+#define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */
+#define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */
+#define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */
+#define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */
+#define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */
+#define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */
+#define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */
+#define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */
+#define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */
+#define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */
+#define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */
+#define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */
+#define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */
+#define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */
+#define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */
+
+/* Public status register (HIFN_1_PUB_STATUS) */
+#define HIFN_PUBSTS_DONE 0x00000001 /* operation done */
+#define HIFN_PUBSTS_CARRY 0x00000002 /* carry */
+
+/* Public interrupt enable register (HIFN_1_PUB_IEN) */
+#define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */
+
+/* Random number generator config register (HIFN_1_RNG_CONFIG) */
+#define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */
+
+#define HIFN_NAMESIZE 32
+#define HIFN_MAX_RESULT_ORDER 5
+
+#define HIFN_D_CMD_RSIZE 24*1
+#define HIFN_D_SRC_RSIZE 80*1
+#define HIFN_D_DST_RSIZE 80*1
+#define HIFN_D_RES_RSIZE 24*1
+
+#define HIFN_D_DST_DALIGN 4
+
+#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)
+
+#define AES_MIN_KEY_SIZE 16
+#define AES_MAX_KEY_SIZE 32
+
+#define HIFN_DES_KEY_LENGTH 8
+#define HIFN_3DES_KEY_LENGTH 24
+#define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE
+#define HIFN_IV_LENGTH 8
+#define HIFN_AES_IV_LENGTH 16
+#define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH
+
+#define HIFN_MAC_KEY_LENGTH 64
+#define HIFN_MD5_LENGTH 16
+#define HIFN_SHA1_LENGTH 20
+#define HIFN_MAC_TRUNC_LENGTH 12
+
+#define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260)
+#define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4)
+#define HIFN_USED_RESULT 12
+
+struct hifn_desc
+{
+ volatile __le32 l;
+ volatile __le32 p;
+};
+
+struct hifn_dma {
+ struct hifn_desc cmdr[HIFN_D_CMD_RSIZE+1];
+ struct hifn_desc srcr[HIFN_D_SRC_RSIZE+1];
+ struct hifn_desc dstr[HIFN_D_DST_RSIZE+1];
+ struct hifn_desc resr[HIFN_D_RES_RSIZE+1];
+
+ u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
+ u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
+
+ /*
+ * Our current positions for insertion and removal from the descriptor
+ * rings.
+ */
+ volatile int cmdi, srci, dsti, resi;
+ volatile int cmdu, srcu, dstu, resu;
+ int cmdk, srck, dstk, resk;
+};
+
+#define HIFN_FLAG_CMD_BUSY (1<<0)
+#define HIFN_FLAG_SRC_BUSY (1<<1)
+#define HIFN_FLAG_DST_BUSY (1<<2)
+#define HIFN_FLAG_RES_BUSY (1<<3)
+#define HIFN_FLAG_OLD_KEY (1<<4)
+
+#define HIFN_DEFAULT_ACTIVE_NUM 5
+
+struct hifn_device
+{
+ char name[HIFN_NAMESIZE];
+
+ int irq;
+
+ struct pci_dev *pdev;
+ void __iomem *bar[3];
+
+ void *desc_virt;
+ dma_addr_t desc_dma;
+
+ u32 dmareg;
+
+ void *sa[HIFN_D_RES_RSIZE];
+
+ spinlock_t lock;
+
+ u32 flags;
+ int active, started;
+ struct delayed_work work;
+ unsigned long reset;
+ unsigned long success;
+ unsigned long prev_success;
+
+ u8 snum;
+
+ struct tasklet_struct tasklet;
+
+ struct crypto_queue queue;
+ struct list_head alg_list;
+
+ unsigned int pk_clk_freq;
+
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ unsigned int rng_wait_time;
+ ktime_t rngtime;
+ struct hwrng rng;
+#endif
+};
+
+#define HIFN_D_LENGTH 0x0000ffff
+#define HIFN_D_NOINVALID 0x01000000
+#define HIFN_D_MASKDONEIRQ 0x02000000
+#define HIFN_D_DESTOVER 0x04000000
+#define HIFN_D_OVER 0x08000000
+#define HIFN_D_LAST 0x20000000
+#define HIFN_D_JUMP 0x40000000
+#define HIFN_D_VALID 0x80000000
+
+struct hifn_base_command
+{
+ volatile __le16 masks;
+ volatile __le16 session_num;
+ volatile __le16 total_source_count;
+ volatile __le16 total_dest_count;
+};
+
+#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
+#define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */
+#define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */
+#define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */
+#define HIFN_BASE_CMD_DECODE 0x2000
+#define HIFN_BASE_CMD_SRCLEN_M 0xc000
+#define HIFN_BASE_CMD_SRCLEN_S 14
+#define HIFN_BASE_CMD_DSTLEN_M 0x3000
+#define HIFN_BASE_CMD_DSTLEN_S 12
+#define HIFN_BASE_CMD_LENMASK_HI 0x30000
+#define HIFN_BASE_CMD_LENMASK_LO 0x0ffff
+
+/*
+ * Structure to help build up the command data structure.
+ */
+struct hifn_crypt_command
+{
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
+};
+
+#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
+#define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */
+#define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */
+#define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */
+#define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */
+#define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */
+#define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */
+#define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */
+#define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */
+#define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */
+#define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */
+#define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */
+#define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */
+#define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */
+#define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */
+#define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */
+#define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */
+#define HIFN_CRYPT_CMD_SRCLEN_M 0xc000
+#define HIFN_CRYPT_CMD_SRCLEN_S 14
+
+/*
+ * Structure to help build up the command data structure.
+ */
+struct hifn_mac_command
+{
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
+};
+
+#define HIFN_MAC_CMD_ALG_MASK 0x0001
+#define HIFN_MAC_CMD_ALG_SHA1 0x0000
+#define HIFN_MAC_CMD_ALG_MD5 0x0001
+#define HIFN_MAC_CMD_MODE_MASK 0x000c
+#define HIFN_MAC_CMD_MODE_HMAC 0x0000
+#define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004
+#define HIFN_MAC_CMD_MODE_HASH 0x0008
+#define HIFN_MAC_CMD_MODE_FULL 0x0004
+#define HIFN_MAC_CMD_TRUNC 0x0010
+#define HIFN_MAC_CMD_RESULT 0x0020
+#define HIFN_MAC_CMD_APPEND 0x0040
+#define HIFN_MAC_CMD_SRCLEN_M 0xc000
+#define HIFN_MAC_CMD_SRCLEN_S 14
+
+/*
+ * MAC POS IPsec initiates authentication after encryption on encodes
+ * and before decryption on decodes.
+ */
+#define HIFN_MAC_CMD_POS_IPSEC 0x0200
+#define HIFN_MAC_CMD_NEW_KEY 0x0800
+
+struct hifn_comp_command
+{
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
+};
+
+#define HIFN_COMP_CMD_SRCLEN_M 0xc000
+#define HIFN_COMP_CMD_SRCLEN_S 14
+#define HIFN_COMP_CMD_ONE 0x0100 /* must be one */
+#define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */
+#define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */
+#define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */
+#define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */
+#define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */
+#define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */
+#define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */
+
+struct hifn_base_result
+{
+ volatile __le16 flags;
+ volatile __le16 session;
+ volatile __le16 src_cnt; /* 15:0 of source count */
+ volatile __le16 dst_cnt; /* 15:0 of dest count */
+};
+
+#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
+#define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */
+#define HIFN_BASE_RES_SRCLEN_S 14
+#define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */
+#define HIFN_BASE_RES_DSTLEN_S 12
+
+struct hifn_comp_result
+{
+ volatile __le16 flags;
+ volatile __le16 crc;
+};
+
+#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
+#define HIFN_COMP_RES_LCB_S 8
+#define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */
+#define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */
+#define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */
+
+struct hifn_mac_result
+{
+ volatile __le16 flags;
+ volatile __le16 reserved;
+ /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
+};
+
+#define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */
+#define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */
+
+struct hifn_crypt_result
+{
+ volatile __le16 flags;
+ volatile __le16 reserved;
+};
+
+#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
+
+#ifndef HIFN_POLL_FREQUENCY
+#define HIFN_POLL_FREQUENCY 0x1
+#endif
+
+#ifndef HIFN_POLL_SCALAR
+#define HIFN_POLL_SCALAR 0x0
+#endif
+
+#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */
+#define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */
+
+struct hifn_crypto_alg
+{
+ struct list_head entry;
+ struct crypto_alg alg;
+ struct hifn_device *dev;
+};
+
+#define ASYNC_SCATTERLIST_CACHE 16
+
+#define ASYNC_FLAGS_MISALIGNED (1<<0)
+
+struct hifn_cipher_walk
+{
+ struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];
+ u32 flags;
+ int num;
+};
+
+struct hifn_context
+{
+ u8 key[HIFN_MAX_CRYPT_KEY_LENGTH];
+ struct hifn_device *dev;
+ unsigned int keysize;
+};
+
+struct hifn_request_context
+{
+ u8 *iv;
+ unsigned int ivsize;
+ u8 op, type, mode, unused;
+ struct hifn_cipher_walk walk;
+};
+
+#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
+
+static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
+{
+ u32 ret;
+
+ ret = readl(dev->bar[0] + reg);
+
+ return ret;
+}
+
+static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
+{
+ u32 ret;
+
+ ret = readl(dev->bar[1] + reg);
+
+ return ret;
+}
+
+static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
+{
+ writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
+}
+
+static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
+{
+ writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
+}
+
+static void hifn_wait_puc(struct hifn_device *dev)
+{
+ int i;
+ u32 ret;
+
+ for (i=10000; i > 0; --i) {
+ ret = hifn_read_0(dev, HIFN_0_PUCTRL);
+ if (!(ret & HIFN_PUCTRL_RESET))
+ break;
+
+ udelay(1);
+ }
+
+ if (!i)
+ dprintk("%s: Failed to reset PUC unit.\n", dev->name);
+}
+
+static void hifn_reset_puc(struct hifn_device *dev)
+{
+ hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
+ hifn_wait_puc(dev);
+}
+
+static void hifn_stop_device(struct hifn_device *dev)
+{
+ hifn_write_1(dev, HIFN_1_DMA_CSR,
+ HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
+ HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
+ hifn_write_0(dev, HIFN_0_PUIER, 0);
+ hifn_write_1(dev, HIFN_1_DMA_IER, 0);
+}
+
+static void hifn_reset_dma(struct hifn_device *dev, int full)
+{
+ hifn_stop_device(dev);
+
+ /*
+ * Setting poll frequency and others to 0.
+ */
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
+ HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
+ mdelay(1);
+
+ /*
+ * Reset DMA.
+ */
+ if (full) {
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
+ mdelay(1);
+ } else {
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
+ HIFN_DMACNFG_MSTRESET);
+ hifn_reset_puc(dev);
+ }
+
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
+ HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
+
+ hifn_reset_puc(dev);
+}
+
+static u32 hifn_next_signature(u_int32_t a, u_int cnt)
+{
+ int i;
+ u32 v;
+
+ for (i = 0; i < cnt; i++) {
+
+ /* get the parity */
+ v = a & 0x80080125;
+ v ^= v >> 16;
+ v ^= v >> 8;
+ v ^= v >> 4;
+ v ^= v >> 2;
+ v ^= v >> 1;
+
+ a = (v & 1) ^ (a << 1);
+ }
+
+ return a;
+}
+
+static struct pci2id {
+ u_short pci_vendor;
+ u_short pci_prod;
+ char card_id[13];
+} pci2id[] = {
+ {
+ PCI_VENDOR_ID_HIFN,
+ PCI_DEVICE_ID_HIFN_7955,
+ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00 }
+ },
+ {
+ PCI_VENDOR_ID_HIFN,
+ PCI_DEVICE_ID_HIFN_7956,
+ { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00 }
+ }
+};
+
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+static int hifn_rng_data_present(struct hwrng *rng, int wait)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+ s64 nsec;
+
+ nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
+ nsec -= dev->rng_wait_time;
+ if (nsec <= 0)
+ return 1;
+ if (!wait)
+ return 0;
+ ndelay(nsec);
+ return 1;
+}
+
+static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+
+ *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+ return 4;
+}
+
+static int hifn_register_rng(struct hifn_device *dev)
+{
+ /*
+ * We must wait at least 256 Pk_clk cycles between two reads of the rng.
+ */
+ dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) *
+ 256;
+
+ dev->rng.name = dev->name;
+ dev->rng.data_present = hifn_rng_data_present,
+ dev->rng.data_read = hifn_rng_data_read,
+ dev->rng.priv = (unsigned long)dev;
+
+ return hwrng_register(&dev->rng);
+}
+
+static void hifn_unregister_rng(struct hifn_device *dev)
+{
+ hwrng_unregister(&dev->rng);
+}
+#else
+#define hifn_register_rng(dev) 0
+#define hifn_unregister_rng(dev)
+#endif
+
+static int hifn_init_pubrng(struct hifn_device *dev)
+{
+ int i;
+
+ hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
+ HIFN_PUBRST_RESET);
+
+ for (i=100; i > 0; --i) {
+ mdelay(1);
+
+ if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
+ break;
+ }
+
+ if (!i)
+ dprintk("Chip %s: Failed to initialise public key engine.\n",
+ dev->name);
+ else {
+ hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
+ dev->dmareg |= HIFN_DMAIER_PUBDONE;
+ hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
+
+ dprintk("Chip %s: Public key engine has been successfully "
+ "initialised.\n", dev->name);
+ }
+
+ /*
+ * Enable RNG engine.
+ */
+
+ hifn_write_1(dev, HIFN_1_RNG_CONFIG,
+ hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
+ dprintk("Chip %s: RNG engine has been successfully initialised.\n",
+ dev->name);
+
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ /* First value must be discarded */
+ hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+#endif
+ return 0;
+}
+
+static int hifn_enable_crypto(struct hifn_device *dev)
+{
+ u32 dmacfg, addr;
+ char *offtbl = NULL;
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
+ if (pci2id[i].pci_vendor == dev->pdev->vendor &&
+ pci2id[i].pci_prod == dev->pdev->device) {
+ offtbl = pci2id[i].card_id;
+ break;
+ }
+ }
+
+ if (offtbl == NULL) {
+ dprintk("Chip %s: Unknown card!\n", dev->name);
+ return -ENODEV;
+ }
+
+ dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
+
+ hifn_write_1(dev, HIFN_1_DMA_CNFG,
+ HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
+ HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
+ mdelay(1);
+ addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
+ mdelay(1);
+ hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
+ mdelay(1);
+
+ for (i=0; i<12; ++i) {
+ addr = hifn_next_signature(addr, offtbl[i] + 0x101);
+ hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
+
+ mdelay(1);
+ }
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
+
+ dprintk("Chip %s: %s.\n", dev->name, pci_name(dev->pdev));
+
+ return 0;
+}
+
+static void hifn_init_dma(struct hifn_device *dev)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ u32 dptr = dev->desc_dma;
+ int i;
+
+ for (i=0; i<HIFN_D_CMD_RSIZE; ++i)
+ dma->cmdr[i].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, command_bufs[i][0]));
+ for (i=0; i<HIFN_D_RES_RSIZE; ++i)
+ dma->resr[i].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, result_bufs[i][0]));
+
+ /*
+ * Setup LAST descriptors.
+ */
+ dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, cmdr[0]));
+ dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, srcr[0]));
+ dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, dstr[0]));
+ dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
+ offsetof(struct hifn_dma, resr[0]));
+
+ dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
+ dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
+ dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
+}
+
+/*
+ * Initialize the PLL. We need to know the frequency of the reference clock
+ * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
+ * allows us to operate without the risk of overclocking the chip. If it
+ * actually uses 33MHz, the chip will operate at half the speed, this can be
+ * overriden by specifying the frequency as module parameter (pci33).
+ *
+ * Unfortunately the PCI clock is not very suitable since the HIFN needs a
+ * stable clock and the PCI clock frequency may vary, so the default is the
+ * external clock. There is no way to find out its frequency, we default to
+ * 66MHz since according to Mike Ham of HiFn, almost every board in existence
+ * has an external crystal populated at 66MHz.
+ */
+static void hifn_init_pll(struct hifn_device *dev)
+{
+ unsigned int freq, m;
+ u32 pllcfg;
+
+ pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
+
+ if (strncmp(hifn_pll_ref, "ext", 3) == 0)
+ pllcfg |= HIFN_PLL_REF_CLK_PLL;
+ else
+ pllcfg |= HIFN_PLL_REF_CLK_HBI;
+
+ if (hifn_pll_ref[3] != '\0')
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ else {
+ freq = 66;
+ printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, "
+ "override with hifn_pll_ref=%.3s<frequency>\n",
+ freq, hifn_pll_ref);
+ }
+
+ m = HIFN_PLL_FCK_MAX / freq;
+
+ pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
+ if (m <= 8)
+ pllcfg |= HIFN_PLL_IS_1_8;
+ else
+ pllcfg |= HIFN_PLL_IS_9_12;
+
+ /* Select clock source and enable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
+
+ /* Let the chip lock to the input clock */
+ mdelay(10);
+
+ /* Disable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
+
+ /* Switch the engines to the PLL */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
+
+ /*
+ * The Fpk_clk runs at half the total speed. Its frequency is needed to
+ * calculate the minimum time between two reads of the rng. Since 33MHz
+ * is actually 33.333... we overestimate the frequency here, resulting
+ * in slightly larger intervals.
+ */
+ dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
+}
+
+static void hifn_init_registers(struct hifn_device *dev)
+{
+ u32 dptr = dev->desc_dma;
+
+ /* Initialization magic... */
+ hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
+ hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
+ hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
+
+ /* write all 4 ring address registers */
+ hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
+ offsetof(struct hifn_dma, cmdr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
+ offsetof(struct hifn_dma, srcr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
+ offsetof(struct hifn_dma, dstr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
+ offsetof(struct hifn_dma, resr[0]));
+
+ mdelay(2);
+#if 0
+ hifn_write_1(dev, HIFN_1_DMA_CSR,
+ HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
+ HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
+ HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
+ HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
+ HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
+ HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
+ HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
+ HIFN_DMACSR_S_WAIT |
+ HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
+ HIFN_DMACSR_C_WAIT |
+ HIFN_DMACSR_ENGINE |
+ HIFN_DMACSR_PUBDONE);
+#else
+ hifn_write_1(dev, HIFN_1_DMA_CSR,
+ HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
+ HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
+ HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
+ HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
+ HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
+ HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
+ HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
+ HIFN_DMACSR_S_WAIT |
+ HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
+ HIFN_DMACSR_C_WAIT |
+ HIFN_DMACSR_ENGINE |
+ HIFN_DMACSR_PUBDONE);
+#endif
+ hifn_read_1(dev, HIFN_1_DMA_CSR);
+
+ dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
+ HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
+ HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
+ HIFN_DMAIER_ENGINE;
+ dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
+
+ hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
+ hifn_read_1(dev, HIFN_1_DMA_IER);
+#if 0
+ hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
+ HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
+ HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
+ HIFN_PUCNFG_DRAM);
+#else
+ hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
+#endif
+ hifn_init_pll(dev);
+
+ hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
+ hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
+ HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
+ ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
+ ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
+}
+
+static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
+ unsigned dlen, unsigned slen, u16 mask, u8 snum)
+{
+ struct hifn_base_command *base_cmd;
+ u8 *buf_pos = buf;
+
+ base_cmd = (struct hifn_base_command *)buf_pos;
+ base_cmd->masks = __cpu_to_le16(mask);
+ base_cmd->total_source_count =
+ __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
+ base_cmd->total_dest_count =
+ __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
+
+ dlen >>= 16;
+ slen >>= 16;
+ base_cmd->session_num = __cpu_to_le16(snum |
+ ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
+ ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
+
+ return sizeof(struct hifn_base_command);
+}
+
+static int hifn_setup_crypto_command(struct hifn_device *dev,
+ u8 *buf, unsigned dlen, unsigned slen,
+ u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ struct hifn_crypt_command *cry_cmd;
+ u8 *buf_pos = buf;
+ u16 cmd_len;
+
+ cry_cmd = (struct hifn_crypt_command *)buf_pos;
+
+ cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
+ dlen >>= 16;
+ cry_cmd->masks = __cpu_to_le16(mode |
+ ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
+ HIFN_CRYPT_CMD_SRCLEN_M));
+ cry_cmd->header_skip = 0;
+ cry_cmd->reserved = 0;
+
+ buf_pos += sizeof(struct hifn_crypt_command);
+
+ dma->cmdu++;
+ if (dma->cmdu > 1) {
+ dev->dmareg |= HIFN_DMAIER_C_WAIT;
+ hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
+ }
+
+ if (keylen) {
+ memcpy(buf_pos, key, keylen);
+ buf_pos += keylen;
+ }
+ if (ivsize) {
+ memcpy(buf_pos, iv, ivsize);
+ buf_pos += ivsize;
+ }
+
+ cmd_len = buf_pos - buf;
+
+ return cmd_len;
+}
+
+static int hifn_setup_cmd_desc(struct hifn_device *dev,
+ struct hifn_context *ctx, struct hifn_request_context *rctx,
+ void *priv, unsigned int nbytes)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int cmd_len, sa_idx;
+ u8 *buf, *buf_pos;
+ u16 mask;
+
+ sa_idx = dma->cmdi;
+ buf_pos = buf = dma->command_bufs[dma->cmdi];
+
+ mask = 0;
+ switch (rctx->op) {
+ case ACRYPTO_OP_DECRYPT:
+ mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
+ break;
+ case ACRYPTO_OP_ENCRYPT:
+ mask = HIFN_BASE_CMD_CRYPT;
+ break;
+ case ACRYPTO_OP_HMAC:
+ mask = HIFN_BASE_CMD_MAC;
+ break;
+ default:
+ goto err_out;
+ }
+
+ buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
+ nbytes, mask, dev->snum);
+
+ if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
+ u16 md = 0;
+
+ if (ctx->keysize)
+ md |= HIFN_CRYPT_CMD_NEW_KEY;
+ if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
+ md |= HIFN_CRYPT_CMD_NEW_IV;
+
+ switch (rctx->mode) {
+ case ACRYPTO_MODE_ECB:
+ md |= HIFN_CRYPT_CMD_MODE_ECB;
+ break;
+ case ACRYPTO_MODE_CBC:
+ md |= HIFN_CRYPT_CMD_MODE_CBC;
+ break;
+ case ACRYPTO_MODE_CFB:
+ md |= HIFN_CRYPT_CMD_MODE_CFB;
+ break;
+ case ACRYPTO_MODE_OFB:
+ md |= HIFN_CRYPT_CMD_MODE_OFB;
+ break;
+ default:
+ goto err_out;
+ }
+
+ switch (rctx->type) {
+ case ACRYPTO_TYPE_AES_128:
+ if (ctx->keysize != 16)
+ goto err_out;
+ md |= HIFN_CRYPT_CMD_KSZ_128 |
+ HIFN_CRYPT_CMD_ALG_AES;
+ break;
+ case ACRYPTO_TYPE_AES_192:
+ if (ctx->keysize != 24)
+ goto err_out;
+ md |= HIFN_CRYPT_CMD_KSZ_192 |
+ HIFN_CRYPT_CMD_ALG_AES;
+ break;
+ case ACRYPTO_TYPE_AES_256:
+ if (ctx->keysize != 32)
+ goto err_out;
+ md |= HIFN_CRYPT_CMD_KSZ_256 |
+ HIFN_CRYPT_CMD_ALG_AES;
+ break;
+ case ACRYPTO_TYPE_3DES:
+ if (ctx->keysize != 24)
+ goto err_out;
+ md |= HIFN_CRYPT_CMD_ALG_3DES;
+ break;
+ case ACRYPTO_TYPE_DES:
+ if (ctx->keysize != 8)
+ goto err_out;
+ md |= HIFN_CRYPT_CMD_ALG_DES;
+ break;
+ default:
+ goto err_out;
+ }
+
+ buf_pos += hifn_setup_crypto_command(dev, buf_pos,
+ nbytes, nbytes, ctx->key, ctx->keysize,
+ rctx->iv, rctx->ivsize, md);
+ }
+
+ dev->sa[sa_idx] = priv;
+ dev->started++;
+
+ cmd_len = buf_pos - buf;
+ dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
+ HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
+
+ if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
+ dma->cmdr[dma->cmdi].l = __cpu_to_le32(
+ HIFN_D_VALID | HIFN_D_LAST |
+ HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
+ dma->cmdi = 0;
+ } else
+ dma->cmdr[dma->cmdi-1].l |= __cpu_to_le32(HIFN_D_VALID);
+
+ if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_CMD_BUSY;
+ }
+ return 0;
+
+err_out:
+ return -EINVAL;
+}
+
+static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
+ unsigned int offset, unsigned int size, int last)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
+
+ idx = dma->srci;
+
+ dma->srcr[idx].p = __cpu_to_le32(addr);
+ dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
+
+ if (++idx == HIFN_D_SRC_RSIZE) {
+ dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
+ (last ? HIFN_D_LAST : 0));
+ idx = 0;
+ }
+
+ dma->srci = idx;
+ dma->srcu++;
+
+ if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_SRC_BUSY;
+ }
+
+ return size;
+}
+
+static void hifn_setup_res_desc(struct hifn_device *dev)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
+ HIFN_D_VALID | HIFN_D_LAST);
+ /*
+ * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
+ * HIFN_D_LAST);
+ */
+
+ if (++dma->resi == HIFN_D_RES_RSIZE) {
+ dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+ dma->resi = 0;
+ }
+
+ dma->resu++;
+
+ if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_RES_BUSY;
+ }
+}
+
+static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
+ unsigned offset, unsigned size, int last)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
+
+ idx = dma->dsti;
+ dma->dstr[idx].p = __cpu_to_le32(addr);
+ dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
+
+ if (++idx == HIFN_D_DST_RSIZE) {
+ dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
+ (last ? HIFN_D_LAST : 0));
+ idx = 0;
+ }
+ dma->dsti = idx;
+ dma->dstu++;
+
+ if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_DST_BUSY;
+ }
+}
+
+static int hifn_setup_dma(struct hifn_device *dev,
+ struct hifn_context *ctx, struct hifn_request_context *rctx,
+ struct scatterlist *src, struct scatterlist *dst,
+ unsigned int nbytes, void *priv)
+{
+ struct scatterlist *t;
+ struct page *spage, *dpage;
+ unsigned int soff, doff;
+ unsigned int n, len;
+
+ n = nbytes;
+ while (n) {
+ spage = sg_page(src);
+ soff = src->offset;
+ len = min(src->length, n);
+
+ hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
+
+ src++;
+ n -= len;
+ }
+
+ t = &rctx->walk.cache[0];
+ n = nbytes;
+ while (n) {
+ if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
+ BUG_ON(!sg_page(t));
+ dpage = sg_page(t);
+ doff = 0;
+ len = t->length;
+ } else {
+ BUG_ON(!sg_page(dst));
+ dpage = sg_page(dst);
+ doff = dst->offset;
+ len = dst->length;
+ }
+ len = min(len, n);
+
+ hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
+
+ dst++;
+ t++;
+ n -= len;
+ }
+
+ hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
+ hifn_setup_res_desc(dev);
+ return 0;
+}
+
+static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
+ int num, gfp_t gfp_flags)
+{
+ int i;
+
+ num = min(ASYNC_SCATTERLIST_CACHE, num);
+ sg_init_table(w->cache, num);
+
+ w->num = 0;
+ for (i=0; i<num; ++i) {
+ struct page *page = alloc_page(gfp_flags);
+ struct scatterlist *s;
+
+ if (!page)
+ break;
+
+ s = &w->cache[i];
+
+ sg_set_page(s, page, PAGE_SIZE, 0);
+ w->num++;
+ }
+
+ return i;
+}
+
+static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
+{
+ int i;
+
+ for (i=0; i<w->num; ++i) {
+ struct scatterlist *s = &w->cache[i];
+
+ __free_page(sg_page(s));
+
+ s->length = 0;
+ }
+
+ w->num = 0;
+}
+
+static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst,
+ unsigned int size, unsigned int *nbytesp)
+{
+ unsigned int copy, drest = *drestp, nbytes = *nbytesp;
+ int idx = 0;
+
+ if (drest < size || size > nbytes)
+ return -EINVAL;
+
+ while (size) {
+ copy = min3(drest, size, dst->length);
+
+ size -= copy;
+ drest -= copy;
+ nbytes -= copy;
+
+ dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
+ __func__, copy, size, drest, nbytes);
+
+ dst++;
+ idx++;
+ }
+
+ *nbytesp = nbytes;
+ *drestp = drest;
+
+ return idx;
+}
+
+static int hifn_cipher_walk(struct ablkcipher_request *req,
+ struct hifn_cipher_walk *w)
+{
+ struct scatterlist *dst, *t;
+ unsigned int nbytes = req->nbytes, offset, copy, diff;
+ int idx, tidx, err;
+
+ tidx = idx = 0;
+ offset = 0;
+ while (nbytes) {
+ if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
+ return -EINVAL;
+
+ dst = &req->dst[idx];
+
+ dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
+ __func__, dst->length, dst->offset, offset, nbytes);
+
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
+ offset) {
+ unsigned slen = min(dst->length - offset, nbytes);
+ unsigned dlen = PAGE_SIZE;
+
+ t = &w->cache[idx];
+
+ err = ablkcipher_add(&dlen, dst, slen, &nbytes);
+ if (err < 0)
+ return err;
+
+ idx += err;
+
+ copy = slen & ~(HIFN_D_DST_DALIGN - 1);
+ diff = slen & (HIFN_D_DST_DALIGN - 1);
+
+ if (dlen < nbytes) {
+ /*
+ * Destination page does not have enough space
+ * to put there additional blocksized chunk,
+ * so we mark that page as containing only
+ * blocksize aligned chunks:
+ * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
+ * and increase number of bytes to be processed
+ * in next chunk:
+ * nbytes += diff;
+ */
+ nbytes += diff;
+
+ /*
+ * Temporary of course...
+ * Kick author if you will catch this one.
+ */
+ printk(KERN_ERR "%s: dlen: %u, nbytes: %u,"
+ "slen: %u, offset: %u.\n",
+ __func__, dlen, nbytes, slen, offset);
+ printk(KERN_ERR "%s: please contact author to fix this "
+ "issue, generally you should not catch "
+ "this path under any condition but who "
+ "knows how did you use crypto code.\n"
+ "Thank you.\n", __func__);
+ BUG();
+ } else {
+ copy += diff + nbytes;
+
+ dst = &req->dst[idx];
+
+ err = ablkcipher_add(&dlen, dst, nbytes, &nbytes);
+ if (err < 0)
+ return err;
+
+ idx += err;
+ }
+
+ t->length = copy;
+ t->offset = offset;
+ } else {
+ nbytes -= min(dst->length, nbytes);
+ idx++;
+ }
+
+ tidx++;
+ }
+
+ return tidx;
+}
+
+static int hifn_setup_session(struct ablkcipher_request *req)
+{
+ struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
+ struct hifn_device *dev = ctx->dev;
+ unsigned long dlen, flags;
+ unsigned int nbytes = req->nbytes, idx = 0;
+ int err = -EINVAL, sg_num;
+ struct scatterlist *dst;
+
+ if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
+ goto err_out_exit;
+
+ rctx->walk.flags = 0;
+
+ while (nbytes) {
+ dst = &req->dst[idx];
+ dlen = min(dst->length, nbytes);
+
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
+ rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
+
+ nbytes -= dlen;
+ idx++;
+ }
+
+ if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
+ err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
+ if (err < 0)
+ return err;
+ }
+
+ sg_num = hifn_cipher_walk(req, &rctx->walk);
+ if (sg_num < 0) {
+ err = sg_num;
+ goto err_out_exit;
+ }
+
+ spin_lock_irqsave(&dev->lock, flags);
+ if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
+ err = -EAGAIN;
+ goto err_out;
+ }
+
+ err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req);
+ if (err)
+ goto err_out;
+
+ dev->snum++;
+
+ dev->active = HIFN_DEFAULT_ACTIVE_NUM;
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ return 0;
+
+err_out:
+ spin_unlock_irqrestore(&dev->lock, flags);
+err_out_exit:
+ if (err) {
+ printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
+ "type: %u, err: %d.\n",
+ dev->name, rctx->iv, rctx->ivsize,
+ ctx->key, ctx->keysize,
+ rctx->mode, rctx->op, rctx->type, err);
+ }
+
+ return err;
+}
+
+static int hifn_test(struct hifn_device *dev, int encdec, u8 snum)
+{
+ int n, err;
+ u8 src[16];
+ struct hifn_context ctx;
+ struct hifn_request_context rctx;
+ u8 fips_aes_ecb_from_zero[16] = {
+ 0x66, 0xE9, 0x4B, 0xD4,
+ 0xEF, 0x8A, 0x2C, 0x3B,
+ 0x88, 0x4C, 0xFA, 0x59,
+ 0xCA, 0x34, 0x2B, 0x2E};
+ struct scatterlist sg;
+
+ memset(src, 0, sizeof(src));
+ memset(ctx.key, 0, sizeof(ctx.key));
+
+ ctx.dev = dev;
+ ctx.keysize = 16;
+ rctx.ivsize = 0;
+ rctx.iv = NULL;
+ rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
+ rctx.mode = ACRYPTO_MODE_ECB;
+ rctx.type = ACRYPTO_TYPE_AES_128;
+ rctx.walk.cache[0].length = 0;
+
+ sg_init_one(&sg, &src, sizeof(src));
+
+ err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL);
+ if (err)
+ goto err_out;
+
+ dev->started = 0;
+ msleep(200);
+
+ dprintk("%s: decoded: ", dev->name);
+ for (n=0; n<sizeof(src); ++n)
+ dprintk("%02x ", src[n]);
+ dprintk("\n");
+ dprintk("%s: FIPS : ", dev->name);
+ for (n=0; n<sizeof(fips_aes_ecb_from_zero); ++n)
+ dprintk("%02x ", fips_aes_ecb_from_zero[n]);
+ dprintk("\n");
+
+ if (!memcmp(src, fips_aes_ecb_from_zero, sizeof(fips_aes_ecb_from_zero))) {
+ printk(KERN_INFO "%s: AES 128 ECB test has been successfully "
+ "passed.\n", dev->name);
+ return 0;
+ }
+
+err_out:
+ printk(KERN_INFO "%s: AES 128 ECB test has been failed.\n", dev->name);
+ return -1;
+}
+
+static int hifn_start_device(struct hifn_device *dev)
+{
+ int err;
+
+ dev->started = dev->active = 0;
+ hifn_reset_dma(dev, 1);
+
+ err = hifn_enable_crypto(dev);
+ if (err)
+ return err;
+
+ hifn_reset_puc(dev);
+
+ hifn_init_dma(dev);
+
+ hifn_init_registers(dev);
+
+ hifn_init_pubrng(dev);
+
+ return 0;
+}
+
+static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
+ struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
+{
+ unsigned int srest = *srestp, nbytes = *nbytesp, copy;
+ void *daddr;
+ int idx = 0;
+
+ if (srest < size || size > nbytes)
+ return -EINVAL;
+
+ while (size) {
+ copy = min3(srest, dst->length, size);
+
+ daddr = kmap_atomic(sg_page(dst));
+ memcpy(daddr + dst->offset + offset, saddr, copy);
+ kunmap_atomic(daddr);
+
+ nbytes -= copy;
+ size -= copy;
+ srest -= copy;
+ saddr += copy;
+ offset = 0;
+
+ dprintk("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
+ __func__, copy, size, srest, nbytes);
+
+ dst++;
+ idx++;
+ }
+
+ *nbytesp = nbytes;
+ *srestp = srest;
+
+ return idx;
+}
+
+static inline void hifn_complete_sa(struct hifn_device *dev, int i)
+{
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ dev->sa[i] = NULL;
+ dev->started--;
+ if (dev->started < 0)
+ printk("%s: started: %d.\n", __func__, dev->started);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ BUG_ON(dev->started < 0);
+}
+
+static void hifn_process_ready(struct ablkcipher_request *req, int error)
+{
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
+
+ if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
+ unsigned int nbytes = req->nbytes;
+ int idx = 0, err;
+ struct scatterlist *dst, *t;
+ void *saddr;
+
+ while (nbytes) {
+ t = &rctx->walk.cache[idx];
+ dst = &req->dst[idx];
+
+ dprintk("\n%s: sg_page(t): %p, t->length: %u, "
+ "sg_page(dst): %p, dst->length: %u, "
+ "nbytes: %u.\n",
+ __func__, sg_page(t), t->length,
+ sg_page(dst), dst->length, nbytes);
+
+ if (!t->length) {
+ nbytes -= min(dst->length, nbytes);
+ idx++;
+ continue;
+ }
+
+ saddr = kmap_atomic(sg_page(t));
+
+ err = ablkcipher_get(saddr, &t->length, t->offset,
+ dst, nbytes, &nbytes);
+ if (err < 0) {
+ kunmap_atomic(saddr);
+ break;
+ }
+
+ idx += err;
+ kunmap_atomic(saddr);
+ }
+
+ hifn_cipher_walk_exit(&rctx->walk);
+ }
+
+ req->base.complete(&req->base, error);
+}
+
+static void hifn_clear_rings(struct hifn_device *dev, int error)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int i, u;
+
+ dprintk("%s: ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
+ "k: %d.%d.%d.%d.\n",
+ dev->name,
+ dma->cmdi, dma->srci, dma->dsti, dma->resi,
+ dma->cmdu, dma->srcu, dma->dstu, dma->resu,
+ dma->cmdk, dma->srck, dma->dstk, dma->resk);
+
+ i = dma->resk; u = dma->resu;
+ while (u != 0) {
+ if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
+ break;
+
+ if (dev->sa[i]) {
+ dev->success++;
+ dev->reset = 0;
+ hifn_process_ready(dev->sa[i], error);
+ hifn_complete_sa(dev, i);
+ }
+
+ if (++i == HIFN_D_RES_RSIZE)
+ i = 0;
+ u--;
+ }
+ dma->resk = i; dma->resu = u;
+
+ i = dma->srck; u = dma->srcu;
+ while (u != 0) {
+ if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
+ break;
+ if (++i == HIFN_D_SRC_RSIZE)
+ i = 0;
+ u--;
+ }
+ dma->srck = i; dma->srcu = u;
+
+ i = dma->cmdk; u = dma->cmdu;
+ while (u != 0) {
+ if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
+ break;
+ if (++i == HIFN_D_CMD_RSIZE)
+ i = 0;
+ u--;
+ }
+ dma->cmdk = i; dma->cmdu = u;
+
+ i = dma->dstk; u = dma->dstu;
+ while (u != 0) {
+ if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
+ break;
+ if (++i == HIFN_D_DST_RSIZE)
+ i = 0;
+ u--;
+ }
+ dma->dstk = i; dma->dstu = u;
+
+ dprintk("%s: ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
+ "k: %d.%d.%d.%d.\n",
+ dev->name,
+ dma->cmdi, dma->srci, dma->dsti, dma->resi,
+ dma->cmdu, dma->srcu, dma->dstu, dma->resu,
+ dma->cmdk, dma->srck, dma->dstk, dma->resk);
+}
+
+static void hifn_work(struct work_struct *work)
+{
+ struct delayed_work *dw = to_delayed_work(work);
+ struct hifn_device *dev = container_of(dw, struct hifn_device, work);
+ unsigned long flags;
+ int reset = 0;
+ u32 r = 0;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ if (dev->active == 0) {
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
+ dev->flags &= ~HIFN_FLAG_CMD_BUSY;
+ r |= HIFN_DMACSR_C_CTRL_DIS;
+ }
+ if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
+ dev->flags &= ~HIFN_FLAG_SRC_BUSY;
+ r |= HIFN_DMACSR_S_CTRL_DIS;
+ }
+ if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
+ dev->flags &= ~HIFN_FLAG_DST_BUSY;
+ r |= HIFN_DMACSR_D_CTRL_DIS;
+ }
+ if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
+ dev->flags &= ~HIFN_FLAG_RES_BUSY;
+ r |= HIFN_DMACSR_R_CTRL_DIS;
+ }
+ if (r)
+ hifn_write_1(dev, HIFN_1_DMA_CSR, r);
+ } else
+ dev->active--;
+
+ if ((dev->prev_success == dev->success) && dev->started)
+ reset = 1;
+ dev->prev_success = dev->success;
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ if (reset) {
+ if (++dev->reset >= 5) {
+ int i;
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ printk("%s: r: %08x, active: %d, started: %d, "
+ "success: %lu: qlen: %u/%u, reset: %d.\n",
+ dev->name, r, dev->active, dev->started,
+ dev->success, dev->queue.qlen, dev->queue.max_qlen,
+ reset);
+
+ printk("%s: res: ", __func__);
+ for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
+ printk("%x.%p ", dma->resr[i].l, dev->sa[i]);
+ if (dev->sa[i]) {
+ hifn_process_ready(dev->sa[i], -ENODEV);
+ hifn_complete_sa(dev, i);
+ }
+ }
+ printk("\n");
+
+ hifn_reset_dma(dev, 1);
+ hifn_stop_device(dev);
+ hifn_start_device(dev);
+ dev->reset = 0;
+ }
+
+ tasklet_schedule(&dev->tasklet);
+ }
+
+ schedule_delayed_work(&dev->work, HZ);
+}
+
+static irqreturn_t hifn_interrupt(int irq, void *data)
+{
+ struct hifn_device *dev = (struct hifn_device *)data;
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ u32 dmacsr, restart;
+
+ dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
+
+ dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
+ "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
+ dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
+ dma->cmdi, dma->srci, dma->dsti, dma->resi,
+ dma->cmdu, dma->srcu, dma->dstu, dma->resu);
+
+ if ((dmacsr & dev->dmareg) == 0)
+ return IRQ_NONE;
+
+ hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
+
+ if (dmacsr & HIFN_DMACSR_ENGINE)
+ hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
+ if (dmacsr & HIFN_DMACSR_PUBDONE)
+ hifn_write_1(dev, HIFN_1_PUB_STATUS,
+ hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
+
+ restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
+ if (restart) {
+ u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
+
+ printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
+ dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
+ !!(dmacsr & HIFN_DMACSR_D_OVER),
+ puisr, !!(puisr & HIFN_PUISR_DSTOVER));
+ if (!!(puisr & HIFN_PUISR_DSTOVER))
+ hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
+ hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
+ HIFN_DMACSR_D_OVER));
+ }
+
+ restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
+ HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
+ if (restart) {
+ printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
+ dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
+ !!(dmacsr & HIFN_DMACSR_S_ABORT),
+ !!(dmacsr & HIFN_DMACSR_D_ABORT),
+ !!(dmacsr & HIFN_DMACSR_R_ABORT));
+ hifn_reset_dma(dev, 1);
+ hifn_init_dma(dev);
+ hifn_init_registers(dev);
+ }
+
+ if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
+ dprintk("%s: wait on command.\n", dev->name);
+ dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
+ hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
+ }
+
+ tasklet_schedule(&dev->tasklet);
+
+ return IRQ_HANDLED;
+}
+
+static void hifn_flush(struct hifn_device *dev)
+{
+ unsigned long flags;
+ struct crypto_async_request *async_req;
+ struct ablkcipher_request *req;
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int i;
+
+ for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
+ struct hifn_desc *d = &dma->resr[i];
+
+ if (dev->sa[i]) {
+ hifn_process_ready(dev->sa[i],
+ (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0);
+ hifn_complete_sa(dev, i);
+ }
+ }
+
+ spin_lock_irqsave(&dev->lock, flags);
+ while ((async_req = crypto_dequeue_request(&dev->queue))) {
+ req = container_of(async_req, struct ablkcipher_request, base);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ hifn_process_ready(req, -ENODEV);
+
+ spin_lock_irqsave(&dev->lock, flags);
+ }
+ spin_unlock_irqrestore(&dev->lock, flags);
+}
+
+static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct hifn_context *ctx = crypto_tfm_ctx(tfm);
+ struct hifn_device *dev = ctx->dev;
+
+ if (len > HIFN_MAX_CRYPT_KEY_LENGTH) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -1;
+ }
+
+ if (len == HIFN_DES_KEY_LENGTH) {
+ u32 tmp[DES_EXPKEY_WORDS];
+ int ret = des_ekey(tmp, key);
+
+ if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
+ tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ return -EINVAL;
+ }
+ }
+
+ dev->flags &= ~HIFN_FLAG_OLD_KEY;
+
+ memcpy(ctx->key, key, len);
+ ctx->keysize = len;
+
+ return 0;
+}
+
+static int hifn_handle_req(struct ablkcipher_request *req)
+{
+ struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_device *dev = ctx->dev;
+ int err = -EAGAIN;
+
+ if (dev->started + DIV_ROUND_UP(req->nbytes, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
+ err = hifn_setup_session(req);
+
+ if (err == -EAGAIN) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ err = ablkcipher_enqueue_request(&dev->queue, req);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ }
+
+ return err;
+}
+
+static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op,
+ u8 type, u8 mode)
+{
+ struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
+ unsigned ivsize;
+
+ ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
+
+ if (req->info && mode != ACRYPTO_MODE_ECB) {
+ if (type == ACRYPTO_TYPE_AES_128)
+ ivsize = HIFN_AES_IV_LENGTH;
+ else if (type == ACRYPTO_TYPE_DES)
+ ivsize = HIFN_DES_KEY_LENGTH;
+ else if (type == ACRYPTO_TYPE_3DES)
+ ivsize = HIFN_3DES_KEY_LENGTH;
+ }
+
+ if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
+ if (ctx->keysize == 24)
+ type = ACRYPTO_TYPE_AES_192;
+ else if (ctx->keysize == 32)
+ type = ACRYPTO_TYPE_AES_256;
+ }
+
+ rctx->op = op;
+ rctx->mode = mode;
+ rctx->type = type;
+ rctx->iv = req->info;
+ rctx->ivsize = ivsize;
+
+ /*
+ * HEAVY TODO: needs to kick Herbert XU to write documentation.
+ * HEAVY TODO: needs to kick Herbert XU to write documentation.
+ * HEAVY TODO: needs to kick Herbert XU to write documentation.
+ */
+
+ return hifn_handle_req(req);
+}
+
+static int hifn_process_queue(struct hifn_device *dev)
+{
+ struct crypto_async_request *async_req, *backlog;
+ struct ablkcipher_request *req;
+ unsigned long flags;
+ int err = 0;
+
+ while (dev->started < HIFN_QUEUE_LENGTH) {
+ spin_lock_irqsave(&dev->lock, flags);
+ backlog = crypto_get_backlog(&dev->queue);
+ async_req = crypto_dequeue_request(&dev->queue);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ if (!async_req)
+ break;
+
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
+ req = container_of(async_req, struct ablkcipher_request, base);
+
+ err = hifn_handle_req(req);
+ if (err)
+ break;
+ }
+
+ return err;
+}
+
+static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op,
+ u8 type, u8 mode)
+{
+ int err;
+ struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_device *dev = ctx->dev;
+
+ err = hifn_setup_crypto_req(req, op, type, mode);
+ if (err)
+ return err;
+
+ if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
+ hifn_process_queue(dev);
+
+ return -EINPROGRESS;
+}
+
+/*
+ * AES ecryption functions.
+ */
+static inline int hifn_encrypt_aes_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_encrypt_aes_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_encrypt_aes_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_encrypt_aes_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
+}
+
+/*
+ * AES decryption functions.
+ */
+static inline int hifn_decrypt_aes_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_decrypt_aes_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_decrypt_aes_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_decrypt_aes_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
+}
+
+/*
+ * DES ecryption functions.
+ */
+static inline int hifn_encrypt_des_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_encrypt_des_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_encrypt_des_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_encrypt_des_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
+}
+
+/*
+ * DES decryption functions.
+ */
+static inline int hifn_decrypt_des_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_decrypt_des_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_decrypt_des_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_decrypt_des_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
+}
+
+/*
+ * 3DES ecryption functions.
+ */
+static inline int hifn_encrypt_3des_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_encrypt_3des_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_encrypt_3des_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
+}
+
+/*
+ * 3DES decryption functions.
+ */
+static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
+}
+static inline int hifn_decrypt_3des_cbc(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
+}
+static inline int hifn_decrypt_3des_cfb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
+}
+static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req)
+{
+ return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
+ ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
+}
+
+struct hifn_alg_template
+{
+ char name[CRYPTO_MAX_ALG_NAME];
+ char drv_name[CRYPTO_MAX_ALG_NAME];
+ unsigned int bsize;
+ struct ablkcipher_alg ablkcipher;
+};
+
+static struct hifn_alg_template hifn_alg_templates[] = {
+ /*
+ * 3DES ECB, CBC, CFB and OFB modes.
+ */
+ {
+ .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_3DES_KEY_LENGTH,
+ .max_keysize = HIFN_3DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_3des_cfb,
+ .decrypt = hifn_decrypt_3des_cfb,
+ },
+ },
+ {
+ .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_3DES_KEY_LENGTH,
+ .max_keysize = HIFN_3DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_3des_ofb,
+ .decrypt = hifn_decrypt_3des_ofb,
+ },
+ },
+ {
+ .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
+ .ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
+ .min_keysize = HIFN_3DES_KEY_LENGTH,
+ .max_keysize = HIFN_3DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_3des_cbc,
+ .decrypt = hifn_decrypt_3des_cbc,
+ },
+ },
+ {
+ .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_3DES_KEY_LENGTH,
+ .max_keysize = HIFN_3DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_3des_ecb,
+ .decrypt = hifn_decrypt_3des_ecb,
+ },
+ },
+
+ /*
+ * DES ECB, CBC, CFB and OFB modes.
+ */
+ {
+ .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_DES_KEY_LENGTH,
+ .max_keysize = HIFN_DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_des_cfb,
+ .decrypt = hifn_decrypt_des_cfb,
+ },
+ },
+ {
+ .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_DES_KEY_LENGTH,
+ .max_keysize = HIFN_DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_des_ofb,
+ .decrypt = hifn_decrypt_des_ofb,
+ },
+ },
+ {
+ .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
+ .ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
+ .min_keysize = HIFN_DES_KEY_LENGTH,
+ .max_keysize = HIFN_DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_des_cbc,
+ .decrypt = hifn_decrypt_des_cbc,
+ },
+ },
+ {
+ .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
+ .ablkcipher = {
+ .min_keysize = HIFN_DES_KEY_LENGTH,
+ .max_keysize = HIFN_DES_KEY_LENGTH,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_des_ecb,
+ .decrypt = hifn_decrypt_des_ecb,
+ },
+ },
+
+ /*
+ * AES ECB, CBC, CFB and OFB modes.
+ */
+ {
+ .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_aes_ecb,
+ .decrypt = hifn_decrypt_aes_ecb,
+ },
+ },
+ {
+ .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
+ .ablkcipher = {
+ .ivsize = HIFN_AES_IV_LENGTH,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_aes_cbc,
+ .decrypt = hifn_decrypt_aes_cbc,
+ },
+ },
+ {
+ .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_aes_cfb,
+ .decrypt = hifn_decrypt_aes_cfb,
+ },
+ },
+ {
+ .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = hifn_setkey,
+ .encrypt = hifn_encrypt_aes_ofb,
+ .decrypt = hifn_decrypt_aes_ofb,
+ },
+ },
+};
+
+static int hifn_cra_init(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
+ struct hifn_context *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->dev = ha->dev;
+ tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context);
+ return 0;
+}
+
+static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t)
+{
+ struct hifn_crypto_alg *alg;
+ int err;
+
+ alg = kzalloc(sizeof(struct hifn_crypto_alg), GFP_KERNEL);
+ if (!alg)
+ return -ENOMEM;
+
+ snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
+ snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
+ t->drv_name, dev->name);
+
+ alg->alg.cra_priority = 300;
+ alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
+ alg->alg.cra_blocksize = t->bsize;
+ alg->alg.cra_ctxsize = sizeof(struct hifn_context);
+ alg->alg.cra_alignmask = 0;
+ alg->alg.cra_type = &crypto_ablkcipher_type;
+ alg->alg.cra_module = THIS_MODULE;
+ alg->alg.cra_u.ablkcipher = t->ablkcipher;
+ alg->alg.cra_init = hifn_cra_init;
+
+ alg->dev = dev;
+
+ list_add_tail(&alg->entry, &dev->alg_list);
+
+ err = crypto_register_alg(&alg->alg);
+ if (err) {
+ list_del(&alg->entry);
+ kfree(alg);
+ }
+
+ return err;
+}
+
+static void hifn_unregister_alg(struct hifn_device *dev)
+{
+ struct hifn_crypto_alg *a, *n;
+
+ list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
+ list_del(&a->entry);
+ crypto_unregister_alg(&a->alg);
+ kfree(a);
+ }
+}
+
+static int hifn_register_alg(struct hifn_device *dev)
+{
+ int i, err;
+
+ for (i=0; i<ARRAY_SIZE(hifn_alg_templates); ++i) {
+ err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
+ if (err)
+ goto err_out_exit;
+ }
+
+ return 0;
+
+err_out_exit:
+ hifn_unregister_alg(dev);
+ return err;
+}
+
+static void hifn_tasklet_callback(unsigned long data)
+{
+ struct hifn_device *dev = (struct hifn_device *)data;
+
+ /*
+ * This is ok to call this without lock being held,
+ * althogh it modifies some parameters used in parallel,
+ * (like dev->success), but they are used in process
+ * context or update is atomic (like setting dev->sa[i] to NULL).
+ */
+ hifn_clear_rings(dev, 0);
+
+ if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
+ hifn_process_queue(dev);
+}
+
+static int __devinit hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+ int err, i;
+ struct hifn_device *dev;
+ char name[8];
+
+ err = pci_enable_device(pdev);
+ if (err)
+ return err;
+ pci_set_master(pdev);
+
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+ if (err)
+ goto err_out_disable_pci_device;
+
+ snprintf(name, sizeof(name), "hifn%d",
+ atomic_inc_return(&hifn_dev_number)-1);
+
+ err = pci_request_regions(pdev, name);
+ if (err)
+ goto err_out_disable_pci_device;
+
+ if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
+ pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
+ pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
+ dprintk("%s: Broken hardware - I/O regions are too small.\n",
+ pci_name(pdev));
+ err = -ENODEV;
+ goto err_out_free_regions;
+ }
+
+ dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
+ GFP_KERNEL);
+ if (!dev) {
+ err = -ENOMEM;
+ goto err_out_free_regions;
+ }
+
+ INIT_LIST_HEAD(&dev->alg_list);
+
+ snprintf(dev->name, sizeof(dev->name), "%s", name);
+ spin_lock_init(&dev->lock);
+
+ for (i=0; i<3; ++i) {
+ unsigned long addr, size;
+
+ addr = pci_resource_start(pdev, i);
+ size = pci_resource_len(pdev, i);
+
+ dev->bar[i] = ioremap_nocache(addr, size);
+ if (!dev->bar[i])
+ goto err_out_unmap_bars;
+ }
+
+ dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
+ &dev->desc_dma);
+ if (!dev->desc_virt) {
+ dprintk("Failed to allocate descriptor rings.\n");
+ goto err_out_unmap_bars;
+ }
+ memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
+
+ dev->pdev = pdev;
+ dev->irq = pdev->irq;
+
+ for (i=0; i<HIFN_D_RES_RSIZE; ++i)
+ dev->sa[i] = NULL;
+
+ pci_set_drvdata(pdev, dev);
+
+ tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
+
+ crypto_init_queue(&dev->queue, 1);
+
+ err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
+ if (err) {
+ dprintk("Failed to request IRQ%d: err: %d.\n", dev->irq, err);
+ dev->irq = 0;
+ goto err_out_free_desc;
+ }
+
+ err = hifn_start_device(dev);
+ if (err)
+ goto err_out_free_irq;
+
+ err = hifn_test(dev, 1, 0);
+ if (err)
+ goto err_out_stop_device;
+
+ err = hifn_register_rng(dev);
+ if (err)
+ goto err_out_stop_device;
+
+ err = hifn_register_alg(dev);
+ if (err)
+ goto err_out_unregister_rng;
+
+ INIT_DELAYED_WORK(&dev->work, hifn_work);
+ schedule_delayed_work(&dev->work, HZ);
+
+ dprintk("HIFN crypto accelerator card at %s has been "
+ "successfully registered as %s.\n",
+ pci_name(pdev), dev->name);
+
+ return 0;
+
+err_out_unregister_rng:
+ hifn_unregister_rng(dev);
+err_out_stop_device:
+ hifn_reset_dma(dev, 1);
+ hifn_stop_device(dev);
+err_out_free_irq:
+ free_irq(dev->irq, dev->name);
+ tasklet_kill(&dev->tasklet);
+err_out_free_desc:
+ pci_free_consistent(pdev, sizeof(struct hifn_dma),
+ dev->desc_virt, dev->desc_dma);
+
+err_out_unmap_bars:
+ for (i=0; i<3; ++i)
+ if (dev->bar[i])
+ iounmap(dev->bar[i]);
+
+err_out_free_regions:
+ pci_release_regions(pdev);
+
+err_out_disable_pci_device:
+ pci_disable_device(pdev);
+
+ return err;
+}
+
+static void __devexit hifn_remove(struct pci_dev *pdev)
+{
+ int i;
+ struct hifn_device *dev;
+
+ dev = pci_get_drvdata(pdev);
+
+ if (dev) {
+ cancel_delayed_work_sync(&dev->work);
+
+ hifn_unregister_rng(dev);
+ hifn_unregister_alg(dev);
+ hifn_reset_dma(dev, 1);
+ hifn_stop_device(dev);
+
+ free_irq(dev->irq, dev->name);
+ tasklet_kill(&dev->tasklet);
+
+ hifn_flush(dev);
+
+ pci_free_consistent(pdev, sizeof(struct hifn_dma),
+ dev->desc_virt, dev->desc_dma);
+ for (i=0; i<3; ++i)
+ if (dev->bar[i])
+ iounmap(dev->bar[i]);
+
+ kfree(dev);
+ }
+
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+}
+
+static struct pci_device_id hifn_pci_tbl[] = {
+ { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
+ { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
+ { 0 }
+};
+MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
+
+static struct pci_driver hifn_pci_driver = {
+ .name = "hifn795x",
+ .id_table = hifn_pci_tbl,
+ .probe = hifn_probe,
+ .remove = __devexit_p(hifn_remove),
+};
+
+static int __init hifn_init(void)
+{
+ unsigned int freq;
+ int err;
+
+ /* HIFN supports only 32-bit addresses */
+ BUILD_BUG_ON(sizeof(dma_addr_t) != 4);
+
+ if (strncmp(hifn_pll_ref, "ext", 3) &&
+ strncmp(hifn_pll_ref, "pci", 3)) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, "
+ "must be pci or ext");
+ return -EINVAL;
+ }
+
+ /*
+ * For the 7955/7956 the reference clock frequency must be in the
+ * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
+ * but this chip is currently not supported.
+ */
+ if (hifn_pll_ref[3] != '\0') {
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ if (freq < 20 || freq > 100) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref "
+ "frequency, must be in the range "
+ "of 20-100");
+ return -EINVAL;
+ }
+ }
+
+ err = pci_register_driver(&hifn_pci_driver);
+ if (err < 0) {
+ dprintk("Failed to register PCI driver for %s device.\n",
+ hifn_pci_driver.name);
+ return -ENODEV;
+ }
+
+ printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
+ "has been successfully registered.\n");
+
+ return 0;
+}
+
+static void __exit hifn_fini(void)
+{
+ pci_unregister_driver(&hifn_pci_driver);
+
+ printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip "
+ "has been successfully unregistered.\n");
+}
+
+module_init(hifn_init);
+module_exit(hifn_fini);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Evgeniy Polyakov <johnpol@2ka.mipt.ru>");
+MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/ixp4xx_crypto.c b/ap/os/linux/linux-3.4.x/drivers/crypto/ixp4xx_crypto.c
new file mode 100644
index 0000000..8f3f74c
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/ixp4xx_crypto.c
@@ -0,0 +1,1508 @@
+/*
+ * Intel IXP4xx NPE-C crypto driver
+ *
+ * Copyright (C) 2008 Christian Hohnstaedt <chohnstaedt@innominate.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of version 2 of the GNU General Public License
+ * as published by the Free Software Foundation.
+ *
+ */
+
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <linux/rtnetlink.h>
+#include <linux/interrupt.h>
+#include <linux/spinlock.h>
+#include <linux/gfp.h>
+#include <linux/module.h>
+
+#include <crypto/ctr.h>
+#include <crypto/des.h>
+#include <crypto/aes.h>
+#include <crypto/sha.h>
+#include <crypto/algapi.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/scatterwalk.h>
+
+#include <mach/npe.h>
+#include <mach/qmgr.h>
+
+#define MAX_KEYLEN 32
+
+/* hash: cfgword + 2 * digestlen; crypt: keylen + cfgword */
+#define NPE_CTX_LEN 80
+#define AES_BLOCK128 16
+
+#define NPE_OP_HASH_VERIFY 0x01
+#define NPE_OP_CCM_ENABLE 0x04
+#define NPE_OP_CRYPT_ENABLE 0x08
+#define NPE_OP_HASH_ENABLE 0x10
+#define NPE_OP_NOT_IN_PLACE 0x20
+#define NPE_OP_HMAC_DISABLE 0x40
+#define NPE_OP_CRYPT_ENCRYPT 0x80
+
+#define NPE_OP_CCM_GEN_MIC 0xcc
+#define NPE_OP_HASH_GEN_ICV 0x50
+#define NPE_OP_ENC_GEN_KEY 0xc9
+
+#define MOD_ECB 0x0000
+#define MOD_CTR 0x1000
+#define MOD_CBC_ENC 0x2000
+#define MOD_CBC_DEC 0x3000
+#define MOD_CCM_ENC 0x4000
+#define MOD_CCM_DEC 0x5000
+
+#define KEYLEN_128 4
+#define KEYLEN_192 6
+#define KEYLEN_256 8
+
+#define CIPH_DECR 0x0000
+#define CIPH_ENCR 0x0400
+
+#define MOD_DES 0x0000
+#define MOD_TDEA2 0x0100
+#define MOD_3DES 0x0200
+#define MOD_AES 0x0800
+#define MOD_AES128 (0x0800 | KEYLEN_128)
+#define MOD_AES192 (0x0900 | KEYLEN_192)
+#define MOD_AES256 (0x0a00 | KEYLEN_256)
+
+#define MAX_IVLEN 16
+#define NPE_ID 2 /* NPE C */
+#define NPE_QLEN 16
+/* Space for registering when the first
+ * NPE_QLEN crypt_ctl are busy */
+#define NPE_QLEN_TOTAL 64
+
+#define SEND_QID 29
+#define RECV_QID 30
+
+#define CTL_FLAG_UNUSED 0x0000
+#define CTL_FLAG_USED 0x1000
+#define CTL_FLAG_PERFORM_ABLK 0x0001
+#define CTL_FLAG_GEN_ICV 0x0002
+#define CTL_FLAG_GEN_REVAES 0x0004
+#define CTL_FLAG_PERFORM_AEAD 0x0008
+#define CTL_FLAG_MASK 0x000f
+
+#define HMAC_IPAD_VALUE 0x36
+#define HMAC_OPAD_VALUE 0x5C
+#define HMAC_PAD_BLOCKLEN SHA1_BLOCK_SIZE
+
+#define MD5_DIGEST_SIZE 16
+
+struct buffer_desc {
+ u32 phys_next;
+#ifdef __ARMEB__
+ u16 buf_len;
+ u16 pkt_len;
+#else
+ u16 pkt_len;
+ u16 buf_len;
+#endif
+ u32 phys_addr;
+ u32 __reserved[4];
+ struct buffer_desc *next;
+ enum dma_data_direction dir;
+};
+
+struct crypt_ctl {
+#ifdef __ARMEB__
+ u8 mode; /* NPE_OP_* operation mode */
+ u8 init_len;
+ u16 reserved;
+#else
+ u16 reserved;
+ u8 init_len;
+ u8 mode; /* NPE_OP_* operation mode */
+#endif
+ u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */
+ u32 icv_rev_aes; /* icv or rev aes */
+ u32 src_buf;
+ u32 dst_buf;
+#ifdef __ARMEB__
+ u16 auth_offs; /* Authentication start offset */
+ u16 auth_len; /* Authentication data length */
+ u16 crypt_offs; /* Cryption start offset */
+ u16 crypt_len; /* Cryption data length */
+#else
+ u16 auth_len; /* Authentication data length */
+ u16 auth_offs; /* Authentication start offset */
+ u16 crypt_len; /* Cryption data length */
+ u16 crypt_offs; /* Cryption start offset */
+#endif
+ u32 aadAddr; /* Additional Auth Data Addr for CCM mode */
+ u32 crypto_ctx; /* NPE Crypto Param structure address */
+
+ /* Used by Host: 4*4 bytes*/
+ unsigned ctl_flags;
+ union {
+ struct ablkcipher_request *ablk_req;
+ struct aead_request *aead_req;
+ struct crypto_tfm *tfm;
+ } data;
+ struct buffer_desc *regist_buf;
+ u8 *regist_ptr;
+};
+
+struct ablk_ctx {
+ struct buffer_desc *src;
+ struct buffer_desc *dst;
+};
+
+struct aead_ctx {
+ struct buffer_desc *buffer;
+ struct scatterlist ivlist;
+ /* used when the hmac is not on one sg entry */
+ u8 *hmac_virt;
+ int encrypt;
+};
+
+struct ix_hash_algo {
+ u32 cfgword;
+ unsigned char *icv;
+};
+
+struct ix_sa_dir {
+ unsigned char *npe_ctx;
+ dma_addr_t npe_ctx_phys;
+ int npe_ctx_idx;
+ u8 npe_mode;
+};
+
+struct ixp_ctx {
+ struct ix_sa_dir encrypt;
+ struct ix_sa_dir decrypt;
+ int authkey_len;
+ u8 authkey[MAX_KEYLEN];
+ int enckey_len;
+ u8 enckey[MAX_KEYLEN];
+ u8 salt[MAX_IVLEN];
+ u8 nonce[CTR_RFC3686_NONCE_SIZE];
+ unsigned salted;
+ atomic_t configuring;
+ struct completion completion;
+};
+
+struct ixp_alg {
+ struct crypto_alg crypto;
+ const struct ix_hash_algo *hash;
+ u32 cfg_enc;
+ u32 cfg_dec;
+
+ int registered;
+};
+
+static const struct ix_hash_algo hash_alg_md5 = {
+ .cfgword = 0xAA010004,
+ .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF"
+ "\xFE\xDC\xBA\x98\x76\x54\x32\x10",
+};
+static const struct ix_hash_algo hash_alg_sha1 = {
+ .cfgword = 0x00000005,
+ .icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA"
+ "\xDC\xFE\x10\x32\x54\x76\xC3\xD2\xE1\xF0",
+};
+
+static struct npe *npe_c;
+static struct dma_pool *buffer_pool = NULL;
+static struct dma_pool *ctx_pool = NULL;
+
+static struct crypt_ctl *crypt_virt = NULL;
+static dma_addr_t crypt_phys;
+
+static int support_aes = 1;
+
+static void dev_release(struct device *dev)
+{
+ return;
+}
+
+#define DRIVER_NAME "ixp4xx_crypto"
+static struct platform_device pseudo_dev = {
+ .name = DRIVER_NAME,
+ .id = 0,
+ .num_resources = 0,
+ .dev = {
+ .coherent_dma_mask = DMA_BIT_MASK(32),
+ .release = dev_release,
+ }
+};
+
+static struct device *dev = &pseudo_dev.dev;
+
+static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt)
+{
+ return crypt_phys + (virt - crypt_virt) * sizeof(struct crypt_ctl);
+}
+
+static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys)
+{
+ return crypt_virt + (phys - crypt_phys) / sizeof(struct crypt_ctl);
+}
+
+static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm)
+{
+ return container_of(tfm->__crt_alg, struct ixp_alg,crypto)->cfg_enc;
+}
+
+static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm)
+{
+ return container_of(tfm->__crt_alg, struct ixp_alg,crypto)->cfg_dec;
+}
+
+static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm)
+{
+ return container_of(tfm->__crt_alg, struct ixp_alg, crypto)->hash;
+}
+
+static int setup_crypt_desc(void)
+{
+ BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64);
+ crypt_virt = dma_alloc_coherent(dev,
+ NPE_QLEN * sizeof(struct crypt_ctl),
+ &crypt_phys, GFP_ATOMIC);
+ if (!crypt_virt)
+ return -ENOMEM;
+ memset(crypt_virt, 0, NPE_QLEN * sizeof(struct crypt_ctl));
+ return 0;
+}
+
+static spinlock_t desc_lock;
+static struct crypt_ctl *get_crypt_desc(void)
+{
+ int i;
+ static int idx = 0;
+ unsigned long flags;
+
+ spin_lock_irqsave(&desc_lock, flags);
+
+ if (unlikely(!crypt_virt))
+ setup_crypt_desc();
+ if (unlikely(!crypt_virt)) {
+ spin_unlock_irqrestore(&desc_lock, flags);
+ return NULL;
+ }
+ i = idx;
+ if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
+ if (++idx >= NPE_QLEN)
+ idx = 0;
+ crypt_virt[i].ctl_flags = CTL_FLAG_USED;
+ spin_unlock_irqrestore(&desc_lock, flags);
+ return crypt_virt +i;
+ } else {
+ spin_unlock_irqrestore(&desc_lock, flags);
+ return NULL;
+ }
+}
+
+static spinlock_t emerg_lock;
+static struct crypt_ctl *get_crypt_desc_emerg(void)
+{
+ int i;
+ static int idx = NPE_QLEN;
+ struct crypt_ctl *desc;
+ unsigned long flags;
+
+ desc = get_crypt_desc();
+ if (desc)
+ return desc;
+ if (unlikely(!crypt_virt))
+ return NULL;
+
+ spin_lock_irqsave(&emerg_lock, flags);
+ i = idx;
+ if (crypt_virt[i].ctl_flags == CTL_FLAG_UNUSED) {
+ if (++idx >= NPE_QLEN_TOTAL)
+ idx = NPE_QLEN;
+ crypt_virt[i].ctl_flags = CTL_FLAG_USED;
+ spin_unlock_irqrestore(&emerg_lock, flags);
+ return crypt_virt +i;
+ } else {
+ spin_unlock_irqrestore(&emerg_lock, flags);
+ return NULL;
+ }
+}
+
+static void free_buf_chain(struct device *dev, struct buffer_desc *buf,u32 phys)
+{
+ while (buf) {
+ struct buffer_desc *buf1;
+ u32 phys1;
+
+ buf1 = buf->next;
+ phys1 = buf->phys_next;
+ dma_unmap_single(dev, buf->phys_next, buf->buf_len, buf->dir);
+ dma_pool_free(buffer_pool, buf, phys);
+ buf = buf1;
+ phys = phys1;
+ }
+}
+
+static struct tasklet_struct crypto_done_tasklet;
+
+static void finish_scattered_hmac(struct crypt_ctl *crypt)
+{
+ struct aead_request *req = crypt->data.aead_req;
+ struct aead_ctx *req_ctx = aead_request_ctx(req);
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ int authsize = crypto_aead_authsize(tfm);
+ int decryptlen = req->cryptlen - authsize;
+
+ if (req_ctx->encrypt) {
+ scatterwalk_map_and_copy(req_ctx->hmac_virt,
+ req->src, decryptlen, authsize, 1);
+ }
+ dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes);
+}
+
+static void one_packet(dma_addr_t phys)
+{
+ struct crypt_ctl *crypt;
+ struct ixp_ctx *ctx;
+ int failed;
+
+ failed = phys & 0x1 ? -EBADMSG : 0;
+ phys &= ~0x3;
+ crypt = crypt_phys2virt(phys);
+
+ switch (crypt->ctl_flags & CTL_FLAG_MASK) {
+ case CTL_FLAG_PERFORM_AEAD: {
+ struct aead_request *req = crypt->data.aead_req;
+ struct aead_ctx *req_ctx = aead_request_ctx(req);
+
+ free_buf_chain(dev, req_ctx->buffer, crypt->src_buf);
+ if (req_ctx->hmac_virt) {
+ finish_scattered_hmac(crypt);
+ }
+ req->base.complete(&req->base, failed);
+ break;
+ }
+ case CTL_FLAG_PERFORM_ABLK: {
+ struct ablkcipher_request *req = crypt->data.ablk_req;
+ struct ablk_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ if (req_ctx->dst) {
+ free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
+ }
+ free_buf_chain(dev, req_ctx->src, crypt->src_buf);
+ req->base.complete(&req->base, failed);
+ break;
+ }
+ case CTL_FLAG_GEN_ICV:
+ ctx = crypto_tfm_ctx(crypt->data.tfm);
+ dma_pool_free(ctx_pool, crypt->regist_ptr,
+ crypt->regist_buf->phys_addr);
+ dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf);
+ if (atomic_dec_and_test(&ctx->configuring))
+ complete(&ctx->completion);
+ break;
+ case CTL_FLAG_GEN_REVAES:
+ ctx = crypto_tfm_ctx(crypt->data.tfm);
+ *(u32*)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR);
+ if (atomic_dec_and_test(&ctx->configuring))
+ complete(&ctx->completion);
+ break;
+ default:
+ BUG();
+ }
+ crypt->ctl_flags = CTL_FLAG_UNUSED;
+}
+
+static void irqhandler(void *_unused)
+{
+ tasklet_schedule(&crypto_done_tasklet);
+}
+
+static void crypto_done_action(unsigned long arg)
+{
+ int i;
+
+ for(i=0; i<4; i++) {
+ dma_addr_t phys = qmgr_get_entry(RECV_QID);
+ if (!phys)
+ return;
+ one_packet(phys);
+ }
+ tasklet_schedule(&crypto_done_tasklet);
+}
+
+static int init_ixp_crypto(void)
+{
+ int ret = -ENODEV;
+ u32 msg[2] = { 0, 0 };
+
+ if (! ( ~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH |
+ IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) {
+ printk(KERN_ERR "ixp_crypto: No HW crypto available\n");
+ return ret;
+ }
+ npe_c = npe_request(NPE_ID);
+ if (!npe_c)
+ return ret;
+
+ if (!npe_running(npe_c)) {
+ ret = npe_load_firmware(npe_c, npe_name(npe_c), dev);
+ if (ret) {
+ return ret;
+ }
+ if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
+ goto npe_error;
+ } else {
+ if (npe_send_message(npe_c, msg, "STATUS_MSG"))
+ goto npe_error;
+
+ if (npe_recv_message(npe_c, msg, "STATUS_MSG"))
+ goto npe_error;
+ }
+
+ switch ((msg[1]>>16) & 0xff) {
+ case 3:
+ printk(KERN_WARNING "Firmware of %s lacks AES support\n",
+ npe_name(npe_c));
+ support_aes = 0;
+ break;
+ case 4:
+ case 5:
+ support_aes = 1;
+ break;
+ default:
+ printk(KERN_ERR "Firmware of %s lacks crypto support\n",
+ npe_name(npe_c));
+ return -ENODEV;
+ }
+ /* buffer_pool will also be used to sometimes store the hmac,
+ * so assure it is large enough
+ */
+ BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc));
+ buffer_pool = dma_pool_create("buffer", dev,
+ sizeof(struct buffer_desc), 32, 0);
+ ret = -ENOMEM;
+ if (!buffer_pool) {
+ goto err;
+ }
+ ctx_pool = dma_pool_create("context", dev,
+ NPE_CTX_LEN, 16, 0);
+ if (!ctx_pool) {
+ goto err;
+ }
+ ret = qmgr_request_queue(SEND_QID, NPE_QLEN_TOTAL, 0, 0,
+ "ixp_crypto:out", NULL);
+ if (ret)
+ goto err;
+ ret = qmgr_request_queue(RECV_QID, NPE_QLEN, 0, 0,
+ "ixp_crypto:in", NULL);
+ if (ret) {
+ qmgr_release_queue(SEND_QID);
+ goto err;
+ }
+ qmgr_set_irq(RECV_QID, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL);
+ tasklet_init(&crypto_done_tasklet, crypto_done_action, 0);
+
+ qmgr_enable_irq(RECV_QID);
+ return 0;
+
+npe_error:
+ printk(KERN_ERR "%s not responding\n", npe_name(npe_c));
+ ret = -EIO;
+err:
+ if (ctx_pool)
+ dma_pool_destroy(ctx_pool);
+ if (buffer_pool)
+ dma_pool_destroy(buffer_pool);
+ npe_release(npe_c);
+ return ret;
+}
+
+static void release_ixp_crypto(void)
+{
+ qmgr_disable_irq(RECV_QID);
+ tasklet_kill(&crypto_done_tasklet);
+
+ qmgr_release_queue(SEND_QID);
+ qmgr_release_queue(RECV_QID);
+
+ dma_pool_destroy(ctx_pool);
+ dma_pool_destroy(buffer_pool);
+
+ npe_release(npe_c);
+
+ if (crypt_virt) {
+ dma_free_coherent(dev,
+ NPE_QLEN_TOTAL * sizeof( struct crypt_ctl),
+ crypt_virt, crypt_phys);
+ }
+ return;
+}
+
+static void reset_sa_dir(struct ix_sa_dir *dir)
+{
+ memset(dir->npe_ctx, 0, NPE_CTX_LEN);
+ dir->npe_ctx_idx = 0;
+ dir->npe_mode = 0;
+}
+
+static int init_sa_dir(struct ix_sa_dir *dir)
+{
+ dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys);
+ if (!dir->npe_ctx) {
+ return -ENOMEM;
+ }
+ reset_sa_dir(dir);
+ return 0;
+}
+
+static void free_sa_dir(struct ix_sa_dir *dir)
+{
+ memset(dir->npe_ctx, 0, NPE_CTX_LEN);
+ dma_pool_free(ctx_pool, dir->npe_ctx, dir->npe_ctx_phys);
+}
+
+static int init_tfm(struct crypto_tfm *tfm)
+{
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ int ret;
+
+ atomic_set(&ctx->configuring, 0);
+ ret = init_sa_dir(&ctx->encrypt);
+ if (ret)
+ return ret;
+ ret = init_sa_dir(&ctx->decrypt);
+ if (ret) {
+ free_sa_dir(&ctx->encrypt);
+ }
+ return ret;
+}
+
+static int init_tfm_ablk(struct crypto_tfm *tfm)
+{
+ tfm->crt_ablkcipher.reqsize = sizeof(struct ablk_ctx);
+ return init_tfm(tfm);
+}
+
+static int init_tfm_aead(struct crypto_tfm *tfm)
+{
+ tfm->crt_aead.reqsize = sizeof(struct aead_ctx);
+ return init_tfm(tfm);
+}
+
+static void exit_tfm(struct crypto_tfm *tfm)
+{
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ free_sa_dir(&ctx->encrypt);
+ free_sa_dir(&ctx->decrypt);
+}
+
+static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target,
+ int init_len, u32 ctx_addr, const u8 *key, int key_len)
+{
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypt_ctl *crypt;
+ struct buffer_desc *buf;
+ int i;
+ u8 *pad;
+ u32 pad_phys, buf_phys;
+
+ BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN);
+ pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys);
+ if (!pad)
+ return -ENOMEM;
+ buf = dma_pool_alloc(buffer_pool, GFP_KERNEL, &buf_phys);
+ if (!buf) {
+ dma_pool_free(ctx_pool, pad, pad_phys);
+ return -ENOMEM;
+ }
+ crypt = get_crypt_desc_emerg();
+ if (!crypt) {
+ dma_pool_free(ctx_pool, pad, pad_phys);
+ dma_pool_free(buffer_pool, buf, buf_phys);
+ return -EAGAIN;
+ }
+
+ memcpy(pad, key, key_len);
+ memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len);
+ for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) {
+ pad[i] ^= xpad;
+ }
+
+ crypt->data.tfm = tfm;
+ crypt->regist_ptr = pad;
+ crypt->regist_buf = buf;
+
+ crypt->auth_offs = 0;
+ crypt->auth_len = HMAC_PAD_BLOCKLEN;
+ crypt->crypto_ctx = ctx_addr;
+ crypt->src_buf = buf_phys;
+ crypt->icv_rev_aes = target;
+ crypt->mode = NPE_OP_HASH_GEN_ICV;
+ crypt->init_len = init_len;
+ crypt->ctl_flags |= CTL_FLAG_GEN_ICV;
+
+ buf->next = 0;
+ buf->buf_len = HMAC_PAD_BLOCKLEN;
+ buf->pkt_len = 0;
+ buf->phys_addr = pad_phys;
+
+ atomic_inc(&ctx->configuring);
+ qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
+ BUG_ON(qmgr_stat_overflow(SEND_QID));
+ return 0;
+}
+
+static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize,
+ const u8 *key, int key_len, unsigned digest_len)
+{
+ u32 itarget, otarget, npe_ctx_addr;
+ unsigned char *cinfo;
+ int init_len, ret = 0;
+ u32 cfgword;
+ struct ix_sa_dir *dir;
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ const struct ix_hash_algo *algo;
+
+ dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
+ cinfo = dir->npe_ctx + dir->npe_ctx_idx;
+ algo = ix_hash(tfm);
+
+ /* write cfg word to cryptinfo */
+ cfgword = algo->cfgword | ( authsize << 6); /* (authsize/4) << 8 */
+#ifndef __ARMEB__
+ cfgword ^= 0xAA000000; /* change the "byte swap" flags */
+#endif
+ *(u32*)cinfo = cpu_to_be32(cfgword);
+ cinfo += sizeof(cfgword);
+
+ /* write ICV to cryptinfo */
+ memcpy(cinfo, algo->icv, digest_len);
+ cinfo += digest_len;
+
+ itarget = dir->npe_ctx_phys + dir->npe_ctx_idx
+ + sizeof(algo->cfgword);
+ otarget = itarget + digest_len;
+ init_len = cinfo - (dir->npe_ctx + dir->npe_ctx_idx);
+ npe_ctx_addr = dir->npe_ctx_phys + dir->npe_ctx_idx;
+
+ dir->npe_ctx_idx += init_len;
+ dir->npe_mode |= NPE_OP_HASH_ENABLE;
+
+ if (!encrypt)
+ dir->npe_mode |= NPE_OP_HASH_VERIFY;
+
+ ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget,
+ init_len, npe_ctx_addr, key, key_len);
+ if (ret)
+ return ret;
+ return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget,
+ init_len, npe_ctx_addr, key, key_len);
+}
+
+static int gen_rev_aes_key(struct crypto_tfm *tfm)
+{
+ struct crypt_ctl *crypt;
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct ix_sa_dir *dir = &ctx->decrypt;
+
+ crypt = get_crypt_desc_emerg();
+ if (!crypt) {
+ return -EAGAIN;
+ }
+ *(u32*)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR);
+
+ crypt->data.tfm = tfm;
+ crypt->crypt_offs = 0;
+ crypt->crypt_len = AES_BLOCK128;
+ crypt->src_buf = 0;
+ crypt->crypto_ctx = dir->npe_ctx_phys;
+ crypt->icv_rev_aes = dir->npe_ctx_phys + sizeof(u32);
+ crypt->mode = NPE_OP_ENC_GEN_KEY;
+ crypt->init_len = dir->npe_ctx_idx;
+ crypt->ctl_flags |= CTL_FLAG_GEN_REVAES;
+
+ atomic_inc(&ctx->configuring);
+ qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
+ BUG_ON(qmgr_stat_overflow(SEND_QID));
+ return 0;
+}
+
+static int setup_cipher(struct crypto_tfm *tfm, int encrypt,
+ const u8 *key, int key_len)
+{
+ u8 *cinfo;
+ u32 cipher_cfg;
+ u32 keylen_cfg = 0;
+ struct ix_sa_dir *dir;
+ struct ixp_ctx *ctx = crypto_tfm_ctx(tfm);
+ u32 *flags = &tfm->crt_flags;
+
+ dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
+ cinfo = dir->npe_ctx;
+
+ if (encrypt) {
+ cipher_cfg = cipher_cfg_enc(tfm);
+ dir->npe_mode |= NPE_OP_CRYPT_ENCRYPT;
+ } else {
+ cipher_cfg = cipher_cfg_dec(tfm);
+ }
+ if (cipher_cfg & MOD_AES) {
+ switch (key_len) {
+ case 16: keylen_cfg = MOD_AES128 | KEYLEN_128; break;
+ case 24: keylen_cfg = MOD_AES192 | KEYLEN_192; break;
+ case 32: keylen_cfg = MOD_AES256 | KEYLEN_256; break;
+ default:
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+ cipher_cfg |= keylen_cfg;
+ } else if (cipher_cfg & MOD_3DES) {
+ const u32 *K = (const u32 *)key;
+ if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) ||
+ !((K[2] ^ K[4]) | (K[3] ^ K[5]))))
+ {
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_SCHED;
+ return -EINVAL;
+ }
+ } else {
+ u32 tmp[DES_EXPKEY_WORDS];
+ if (des_ekey(tmp, key) == 0) {
+ *flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ }
+ }
+ /* write cfg word to cryptinfo */
+ *(u32*)cinfo = cpu_to_be32(cipher_cfg);
+ cinfo += sizeof(cipher_cfg);
+
+ /* write cipher key to cryptinfo */
+ memcpy(cinfo, key, key_len);
+ /* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */
+ if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) {
+ memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE -key_len);
+ key_len = DES3_EDE_KEY_SIZE;
+ }
+ dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len;
+ dir->npe_mode |= NPE_OP_CRYPT_ENABLE;
+ if ((cipher_cfg & MOD_AES) && !encrypt) {
+ return gen_rev_aes_key(tfm);
+ }
+ return 0;
+}
+
+static struct buffer_desc *chainup_buffers(struct device *dev,
+ struct scatterlist *sg, unsigned nbytes,
+ struct buffer_desc *buf, gfp_t flags,
+ enum dma_data_direction dir)
+{
+ for (;nbytes > 0; sg = scatterwalk_sg_next(sg)) {
+ unsigned len = min(nbytes, sg->length);
+ struct buffer_desc *next_buf;
+ u32 next_buf_phys;
+ void *ptr;
+
+ nbytes -= len;
+ ptr = page_address(sg_page(sg)) + sg->offset;
+ next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys);
+ if (!next_buf) {
+ buf = NULL;
+ break;
+ }
+ sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir);
+ buf->next = next_buf;
+ buf->phys_next = next_buf_phys;
+ buf = next_buf;
+
+ buf->phys_addr = sg_dma_address(sg);
+ buf->buf_len = len;
+ buf->dir = dir;
+ }
+ buf->next = NULL;
+ buf->phys_next = 0;
+ return buf;
+}
+
+static int ablk_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int key_len)
+{
+ struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ u32 *flags = &tfm->base.crt_flags;
+ int ret;
+
+ init_completion(&ctx->completion);
+ atomic_inc(&ctx->configuring);
+
+ reset_sa_dir(&ctx->encrypt);
+ reset_sa_dir(&ctx->decrypt);
+
+ ctx->encrypt.npe_mode = NPE_OP_HMAC_DISABLE;
+ ctx->decrypt.npe_mode = NPE_OP_HMAC_DISABLE;
+
+ ret = setup_cipher(&tfm->base, 0, key, key_len);
+ if (ret)
+ goto out;
+ ret = setup_cipher(&tfm->base, 1, key, key_len);
+ if (ret)
+ goto out;
+
+ if (*flags & CRYPTO_TFM_RES_WEAK_KEY) {
+ if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) {
+ ret = -EINVAL;
+ } else {
+ *flags &= ~CRYPTO_TFM_RES_WEAK_KEY;
+ }
+ }
+out:
+ if (!atomic_dec_and_test(&ctx->configuring))
+ wait_for_completion(&ctx->completion);
+ return ret;
+}
+
+static int ablk_rfc3686_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int key_len)
+{
+ struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+ /* the nonce is stored in bytes at end of key */
+ if (key_len < CTR_RFC3686_NONCE_SIZE)
+ return -EINVAL;
+
+ memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE),
+ CTR_RFC3686_NONCE_SIZE);
+
+ key_len -= CTR_RFC3686_NONCE_SIZE;
+ return ablk_setkey(tfm, key, key_len);
+}
+
+static int ablk_perform(struct ablkcipher_request *req, int encrypt)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ unsigned ivsize = crypto_ablkcipher_ivsize(tfm);
+ struct ix_sa_dir *dir;
+ struct crypt_ctl *crypt;
+ unsigned int nbytes = req->nbytes;
+ enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
+ struct ablk_ctx *req_ctx = ablkcipher_request_ctx(req);
+ struct buffer_desc src_hook;
+ gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (qmgr_stat_full(SEND_QID))
+ return -EAGAIN;
+ if (atomic_read(&ctx->configuring))
+ return -EAGAIN;
+
+ dir = encrypt ? &ctx->encrypt : &ctx->decrypt;
+
+ crypt = get_crypt_desc();
+ if (!crypt)
+ return -ENOMEM;
+
+ crypt->data.ablk_req = req;
+ crypt->crypto_ctx = dir->npe_ctx_phys;
+ crypt->mode = dir->npe_mode;
+ crypt->init_len = dir->npe_ctx_idx;
+
+ crypt->crypt_offs = 0;
+ crypt->crypt_len = nbytes;
+
+ BUG_ON(ivsize && !req->info);
+ memcpy(crypt->iv, req->info, ivsize);
+ if (req->src != req->dst) {
+ struct buffer_desc dst_hook;
+ crypt->mode |= NPE_OP_NOT_IN_PLACE;
+ /* This was never tested by Intel
+ * for more than one dst buffer, I think. */
+ BUG_ON(req->dst->length < nbytes);
+ req_ctx->dst = NULL;
+ if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook,
+ flags, DMA_FROM_DEVICE))
+ goto free_buf_dest;
+ src_direction = DMA_TO_DEVICE;
+ req_ctx->dst = dst_hook.next;
+ crypt->dst_buf = dst_hook.phys_next;
+ } else {
+ req_ctx->dst = NULL;
+ }
+ req_ctx->src = NULL;
+ if (!chainup_buffers(dev, req->src, nbytes, &src_hook,
+ flags, src_direction))
+ goto free_buf_src;
+
+ req_ctx->src = src_hook.next;
+ crypt->src_buf = src_hook.phys_next;
+ crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
+ qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
+ BUG_ON(qmgr_stat_overflow(SEND_QID));
+ return -EINPROGRESS;
+
+free_buf_src:
+ free_buf_chain(dev, req_ctx->src, crypt->src_buf);
+free_buf_dest:
+ if (req->src != req->dst) {
+ free_buf_chain(dev, req_ctx->dst, crypt->dst_buf);
+ }
+ crypt->ctl_flags = CTL_FLAG_UNUSED;
+ return -ENOMEM;
+}
+
+static int ablk_encrypt(struct ablkcipher_request *req)
+{
+ return ablk_perform(req, 1);
+}
+
+static int ablk_decrypt(struct ablkcipher_request *req)
+{
+ return ablk_perform(req, 0);
+}
+
+static int ablk_rfc3686_crypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct ixp_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ u8 iv[CTR_RFC3686_BLOCK_SIZE];
+ u8 *info = req->info;
+ int ret;
+
+ /* set up counter block */
+ memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE);
+ memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE);
+
+ /* initialize counter portion of counter block */
+ *(__be32 *)(iv + CTR_RFC3686_NONCE_SIZE + CTR_RFC3686_IV_SIZE) =
+ cpu_to_be32(1);
+
+ req->info = iv;
+ ret = ablk_perform(req, 1);
+ req->info = info;
+ return ret;
+}
+
+static int hmac_inconsistent(struct scatterlist *sg, unsigned start,
+ unsigned int nbytes)
+{
+ int offset = 0;
+
+ if (!nbytes)
+ return 0;
+
+ for (;;) {
+ if (start < offset + sg->length)
+ break;
+
+ offset += sg->length;
+ sg = scatterwalk_sg_next(sg);
+ }
+ return (start + nbytes > offset + sg->length);
+}
+
+static int aead_perform(struct aead_request *req, int encrypt,
+ int cryptoffset, int eff_cryptlen, u8 *iv)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
+ unsigned ivsize = crypto_aead_ivsize(tfm);
+ unsigned authsize = crypto_aead_authsize(tfm);
+ struct ix_sa_dir *dir;
+ struct crypt_ctl *crypt;
+ unsigned int cryptlen;
+ struct buffer_desc *buf, src_hook;
+ struct aead_ctx *req_ctx = aead_request_ctx(req);
+ gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
+ GFP_KERNEL : GFP_ATOMIC;
+
+ if (qmgr_stat_full(SEND_QID))
+ return -EAGAIN;
+ if (atomic_read(&ctx->configuring))
+ return -EAGAIN;
+
+ if (encrypt) {
+ dir = &ctx->encrypt;
+ cryptlen = req->cryptlen;
+ } else {
+ dir = &ctx->decrypt;
+ /* req->cryptlen includes the authsize when decrypting */
+ cryptlen = req->cryptlen -authsize;
+ eff_cryptlen -= authsize;
+ }
+ crypt = get_crypt_desc();
+ if (!crypt)
+ return -ENOMEM;
+
+ crypt->data.aead_req = req;
+ crypt->crypto_ctx = dir->npe_ctx_phys;
+ crypt->mode = dir->npe_mode;
+ crypt->init_len = dir->npe_ctx_idx;
+
+ crypt->crypt_offs = cryptoffset;
+ crypt->crypt_len = eff_cryptlen;
+
+ crypt->auth_offs = 0;
+ crypt->auth_len = req->assoclen + ivsize + cryptlen;
+ BUG_ON(ivsize && !req->iv);
+ memcpy(crypt->iv, req->iv, ivsize);
+
+ if (req->src != req->dst) {
+ BUG(); /* -ENOTSUP because of my laziness */
+ }
+
+ /* ASSOC data */
+ buf = chainup_buffers(dev, req->assoc, req->assoclen, &src_hook,
+ flags, DMA_TO_DEVICE);
+ req_ctx->buffer = src_hook.next;
+ crypt->src_buf = src_hook.phys_next;
+ if (!buf)
+ goto out;
+ /* IV */
+ sg_init_table(&req_ctx->ivlist, 1);
+ sg_set_buf(&req_ctx->ivlist, iv, ivsize);
+ buf = chainup_buffers(dev, &req_ctx->ivlist, ivsize, buf, flags,
+ DMA_BIDIRECTIONAL);
+ if (!buf)
+ goto free_chain;
+ if (unlikely(hmac_inconsistent(req->src, cryptlen, authsize))) {
+ /* The 12 hmac bytes are scattered,
+ * we need to copy them into a safe buffer */
+ req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags,
+ &crypt->icv_rev_aes);
+ if (unlikely(!req_ctx->hmac_virt))
+ goto free_chain;
+ if (!encrypt) {
+ scatterwalk_map_and_copy(req_ctx->hmac_virt,
+ req->src, cryptlen, authsize, 0);
+ }
+ req_ctx->encrypt = encrypt;
+ } else {
+ req_ctx->hmac_virt = NULL;
+ }
+ /* Crypt */
+ buf = chainup_buffers(dev, req->src, cryptlen + authsize, buf, flags,
+ DMA_BIDIRECTIONAL);
+ if (!buf)
+ goto free_hmac_virt;
+ if (!req_ctx->hmac_virt) {
+ crypt->icv_rev_aes = buf->phys_addr + buf->buf_len - authsize;
+ }
+
+ crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD;
+ qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt));
+ BUG_ON(qmgr_stat_overflow(SEND_QID));
+ return -EINPROGRESS;
+free_hmac_virt:
+ if (req_ctx->hmac_virt) {
+ dma_pool_free(buffer_pool, req_ctx->hmac_virt,
+ crypt->icv_rev_aes);
+ }
+free_chain:
+ free_buf_chain(dev, req_ctx->buffer, crypt->src_buf);
+out:
+ crypt->ctl_flags = CTL_FLAG_UNUSED;
+ return -ENOMEM;
+}
+
+static int aead_setup(struct crypto_aead *tfm, unsigned int authsize)
+{
+ struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
+ u32 *flags = &tfm->base.crt_flags;
+ unsigned digest_len = crypto_aead_alg(tfm)->maxauthsize;
+ int ret;
+
+ if (!ctx->enckey_len && !ctx->authkey_len)
+ return 0;
+ init_completion(&ctx->completion);
+ atomic_inc(&ctx->configuring);
+
+ reset_sa_dir(&ctx->encrypt);
+ reset_sa_dir(&ctx->decrypt);
+
+ ret = setup_cipher(&tfm->base, 0, ctx->enckey, ctx->enckey_len);
+ if (ret)
+ goto out;
+ ret = setup_cipher(&tfm->base, 1, ctx->enckey, ctx->enckey_len);
+ if (ret)
+ goto out;
+ ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey,
+ ctx->authkey_len, digest_len);
+ if (ret)
+ goto out;
+ ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey,
+ ctx->authkey_len, digest_len);
+ if (ret)
+ goto out;
+
+ if (*flags & CRYPTO_TFM_RES_WEAK_KEY) {
+ if (*flags & CRYPTO_TFM_REQ_WEAK_KEY) {
+ ret = -EINVAL;
+ goto out;
+ } else {
+ *flags &= ~CRYPTO_TFM_RES_WEAK_KEY;
+ }
+ }
+out:
+ if (!atomic_dec_and_test(&ctx->configuring))
+ wait_for_completion(&ctx->completion);
+ return ret;
+}
+
+static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
+{
+ int max = crypto_aead_alg(tfm)->maxauthsize >> 2;
+
+ if ((authsize>>2) < 1 || (authsize>>2) > max || (authsize & 3))
+ return -EINVAL;
+ return aead_setup(tfm, authsize);
+}
+
+static int aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
+ struct rtattr *rta = (struct rtattr *)key;
+ struct crypto_authenc_key_param *param;
+
+ if (!RTA_OK(rta, keylen))
+ goto badkey;
+ if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
+ goto badkey;
+ if (RTA_PAYLOAD(rta) < sizeof(*param))
+ goto badkey;
+
+ param = RTA_DATA(rta);
+ ctx->enckey_len = be32_to_cpu(param->enckeylen);
+
+ key += RTA_ALIGN(rta->rta_len);
+ keylen -= RTA_ALIGN(rta->rta_len);
+
+ if (keylen < ctx->enckey_len)
+ goto badkey;
+
+ ctx->authkey_len = keylen - ctx->enckey_len;
+ memcpy(ctx->enckey, key + ctx->authkey_len, ctx->enckey_len);
+ memcpy(ctx->authkey, key, ctx->authkey_len);
+
+ return aead_setup(tfm, crypto_aead_authsize(tfm));
+badkey:
+ ctx->enckey_len = 0;
+ crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+}
+
+static int aead_encrypt(struct aead_request *req)
+{
+ unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
+ return aead_perform(req, 1, req->assoclen + ivsize,
+ req->cryptlen, req->iv);
+}
+
+static int aead_decrypt(struct aead_request *req)
+{
+ unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
+ return aead_perform(req, 0, req->assoclen + ivsize,
+ req->cryptlen, req->iv);
+}
+
+static int aead_givencrypt(struct aead_givcrypt_request *req)
+{
+ struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);
+ struct ixp_ctx *ctx = crypto_aead_ctx(tfm);
+ unsigned len, ivsize = crypto_aead_ivsize(tfm);
+ __be64 seq;
+
+ /* copied from eseqiv.c */
+ if (!ctx->salted) {
+ get_random_bytes(ctx->salt, ivsize);
+ ctx->salted = 1;
+ }
+ memcpy(req->areq.iv, ctx->salt, ivsize);
+ len = ivsize;
+ if (ivsize > sizeof(u64)) {
+ memset(req->giv, 0, ivsize - sizeof(u64));
+ len = sizeof(u64);
+ }
+ seq = cpu_to_be64(req->seq);
+ memcpy(req->giv + ivsize - len, &seq, len);
+ return aead_perform(&req->areq, 1, req->areq.assoclen,
+ req->areq.cryptlen +ivsize, req->giv);
+}
+
+static struct ixp_alg ixp4xx_algos[] = {
+{
+ .crypto = {
+ .cra_name = "cbc(des)",
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ .geniv = "eseqiv",
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
+
+}, {
+ .crypto = {
+ .cra_name = "ecb(des)",
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_DES | MOD_ECB | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_DES | MOD_ECB | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "cbc(des3_ede)",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .geniv = "eseqiv",
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "ecb(des3_ede)",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_ECB | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_3DES | MOD_ECB | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "cbc(aes)",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .geniv = "eseqiv",
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
+ .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
+}, {
+ .crypto = {
+ .cra_name = "ecb(aes)",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_ECB,
+ .cfg_dec = CIPH_DECR | MOD_AES | MOD_ECB,
+}, {
+ .crypto = {
+ .cra_name = "ctr(aes)",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .geniv = "eseqiv",
+ }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
+ .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
+}, {
+ .crypto = {
+ .cra_name = "rfc3686(ctr(aes))",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .geniv = "eseqiv",
+ .setkey = ablk_rfc3686_setkey,
+ .encrypt = ablk_rfc3686_crypt,
+ .decrypt = ablk_rfc3686_crypt }
+ }
+ },
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CTR,
+ .cfg_dec = CIPH_ENCR | MOD_AES | MOD_CTR,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(md5),cbc(des))",
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_md5,
+ .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_md5,
+ .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(sha1),cbc(des))",
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = DES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_sha1,
+ .cfg_enc = CIPH_ENCR | MOD_DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_DES | MOD_CBC_DEC | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_sha1,
+ .cfg_enc = CIPH_ENCR | MOD_3DES | MOD_CBC_ENC | KEYLEN_192,
+ .cfg_dec = CIPH_DECR | MOD_3DES | MOD_CBC_DEC | KEYLEN_192,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(md5),cbc(aes))",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_md5,
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
+ .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
+}, {
+ .crypto = {
+ .cra_name = "authenc(hmac(sha1),cbc(aes))",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_u = { .aead = {
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ }
+ }
+ },
+ .hash = &hash_alg_sha1,
+ .cfg_enc = CIPH_ENCR | MOD_AES | MOD_CBC_ENC,
+ .cfg_dec = CIPH_DECR | MOD_AES | MOD_CBC_DEC,
+} };
+
+#define IXP_POSTFIX "-ixp4xx"
+static int __init ixp_module_init(void)
+{
+ int num = ARRAY_SIZE(ixp4xx_algos);
+ int i,err ;
+
+ if (platform_device_register(&pseudo_dev))
+ return -ENODEV;
+
+ spin_lock_init(&desc_lock);
+ spin_lock_init(&emerg_lock);
+
+ err = init_ixp_crypto();
+ if (err) {
+ platform_device_unregister(&pseudo_dev);
+ return err;
+ }
+ for (i=0; i< num; i++) {
+ struct crypto_alg *cra = &ixp4xx_algos[i].crypto;
+
+ if (snprintf(cra->cra_driver_name, CRYPTO_MAX_ALG_NAME,
+ "%s"IXP_POSTFIX, cra->cra_name) >=
+ CRYPTO_MAX_ALG_NAME)
+ {
+ continue;
+ }
+ if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) {
+ continue;
+ }
+ if (!ixp4xx_algos[i].hash) {
+ /* block ciphers */
+ cra->cra_type = &crypto_ablkcipher_type;
+ cra->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ if (!cra->cra_ablkcipher.setkey)
+ cra->cra_ablkcipher.setkey = ablk_setkey;
+ if (!cra->cra_ablkcipher.encrypt)
+ cra->cra_ablkcipher.encrypt = ablk_encrypt;
+ if (!cra->cra_ablkcipher.decrypt)
+ cra->cra_ablkcipher.decrypt = ablk_decrypt;
+ cra->cra_init = init_tfm_ablk;
+ } else {
+ /* authenc */
+ cra->cra_type = &crypto_aead_type;
+ cra->cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC;
+ cra->cra_aead.setkey = aead_setkey;
+ cra->cra_aead.setauthsize = aead_setauthsize;
+ cra->cra_aead.encrypt = aead_encrypt;
+ cra->cra_aead.decrypt = aead_decrypt;
+ cra->cra_aead.givencrypt = aead_givencrypt;
+ cra->cra_init = init_tfm_aead;
+ }
+ cra->cra_ctxsize = sizeof(struct ixp_ctx);
+ cra->cra_module = THIS_MODULE;
+ cra->cra_alignmask = 3;
+ cra->cra_priority = 300;
+ cra->cra_exit = exit_tfm;
+ if (crypto_register_alg(cra))
+ printk(KERN_ERR "Failed to register '%s'\n",
+ cra->cra_name);
+ else
+ ixp4xx_algos[i].registered = 1;
+ }
+ return 0;
+}
+
+static void __exit ixp_module_exit(void)
+{
+ int num = ARRAY_SIZE(ixp4xx_algos);
+ int i;
+
+ for (i=0; i< num; i++) {
+ if (ixp4xx_algos[i].registered)
+ crypto_unregister_alg(&ixp4xx_algos[i].crypto);
+ }
+ release_ixp_crypto();
+ platform_device_unregister(&pseudo_dev);
+}
+
+module_init(ixp_module_init);
+module_exit(ixp_module_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Christian Hohnstaedt <chohnstaedt@innominate.com>");
+MODULE_DESCRIPTION("IXP4xx hardware crypto");
+
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.c b/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.c
new file mode 100644
index 0000000..e6ecc5f
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.c
@@ -0,0 +1,1140 @@
+/*
+ * Support for Marvell's crypto engine which can be found on some Orion5X
+ * boards.
+ *
+ * Author: Sebastian Andrzej Siewior < sebastian at breakpoint dot cc >
+ * License: GPLv2
+ *
+ */
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <linux/crypto.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kthread.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <crypto/internal/hash.h>
+#include <crypto/sha.h>
+
+#include "mv_cesa.h"
+
+#define MV_CESA "MV-CESA:"
+#define MAX_HW_HASH_SIZE 0xFFFF
+
+/*
+ * STM:
+ * /---------------------------------------\
+ * | | request complete
+ * \./ |
+ * IDLE -> new request -> BUSY -> done -> DEQUEUE
+ * /°\ |
+ * | | more scatter entries
+ * \________________/
+ */
+enum engine_status {
+ ENGINE_IDLE,
+ ENGINE_BUSY,
+ ENGINE_W_DEQUEUE,
+};
+
+/**
+ * struct req_progress - used for every crypt request
+ * @src_sg_it: sg iterator for src
+ * @dst_sg_it: sg iterator for dst
+ * @sg_src_left: bytes left in src to process (scatter list)
+ * @src_start: offset to add to src start position (scatter list)
+ * @crypt_len: length of current hw crypt/hash process
+ * @hw_nbytes: total bytes to process in hw for this request
+ * @copy_back: whether to copy data back (crypt) or not (hash)
+ * @sg_dst_left: bytes left dst to process in this scatter list
+ * @dst_start: offset to add to dst start position (scatter list)
+ * @hw_processed_bytes: number of bytes processed by hw (request).
+ *
+ * sg helper are used to iterate over the scatterlist. Since the size of the
+ * SRAM may be less than the scatter size, this struct struct is used to keep
+ * track of progress within current scatterlist.
+ */
+struct req_progress {
+ struct sg_mapping_iter src_sg_it;
+ struct sg_mapping_iter dst_sg_it;
+ void (*complete) (void);
+ void (*process) (int is_first);
+
+ /* src mostly */
+ int sg_src_left;
+ int src_start;
+ int crypt_len;
+ int hw_nbytes;
+ /* dst mostly */
+ int copy_back;
+ int sg_dst_left;
+ int dst_start;
+ int hw_processed_bytes;
+};
+
+struct crypto_priv {
+ void __iomem *reg;
+ void __iomem *sram;
+ int irq;
+ struct task_struct *queue_th;
+
+ /* the lock protects queue and eng_st */
+ spinlock_t lock;
+ struct crypto_queue queue;
+ enum engine_status eng_st;
+ struct crypto_async_request *cur_req;
+ struct req_progress p;
+ int max_req_size;
+ int sram_size;
+ int has_sha1;
+ int has_hmac_sha1;
+};
+
+static struct crypto_priv *cpg;
+
+struct mv_ctx {
+ u8 aes_enc_key[AES_KEY_LEN];
+ u32 aes_dec_key[8];
+ int key_len;
+ u32 need_calc_aes_dkey;
+};
+
+enum crypto_op {
+ COP_AES_ECB,
+ COP_AES_CBC,
+};
+
+struct mv_req_ctx {
+ enum crypto_op op;
+ int decrypt;
+};
+
+enum hash_op {
+ COP_SHA1,
+ COP_HMAC_SHA1
+};
+
+struct mv_tfm_hash_ctx {
+ struct crypto_shash *fallback;
+ struct crypto_shash *base_hash;
+ u32 ivs[2 * SHA1_DIGEST_SIZE / 4];
+ int count_add;
+ enum hash_op op;
+};
+
+struct mv_req_hash_ctx {
+ u64 count;
+ u32 state[SHA1_DIGEST_SIZE / 4];
+ u8 buffer[SHA1_BLOCK_SIZE];
+ int first_hash; /* marks that we don't have previous state */
+ int last_chunk; /* marks that this is the 'final' request */
+ int extra_bytes; /* unprocessed bytes in buffer */
+ enum hash_op op;
+ int count_add;
+};
+
+static void compute_aes_dec_key(struct mv_ctx *ctx)
+{
+ struct crypto_aes_ctx gen_aes_key;
+ int key_pos;
+
+ if (!ctx->need_calc_aes_dkey)
+ return;
+
+ crypto_aes_expand_key(&gen_aes_key, ctx->aes_enc_key, ctx->key_len);
+
+ key_pos = ctx->key_len + 24;
+ memcpy(ctx->aes_dec_key, &gen_aes_key.key_enc[key_pos], 4 * 4);
+ switch (ctx->key_len) {
+ case AES_KEYSIZE_256:
+ key_pos -= 2;
+ /* fall */
+ case AES_KEYSIZE_192:
+ key_pos -= 2;
+ memcpy(&ctx->aes_dec_key[4], &gen_aes_key.key_enc[key_pos],
+ 4 * 4);
+ break;
+ }
+ ctx->need_calc_aes_dkey = 0;
+}
+
+static int mv_setkey_aes(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct mv_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ switch (len) {
+ case AES_KEYSIZE_128:
+ case AES_KEYSIZE_192:
+ case AES_KEYSIZE_256:
+ break;
+ default:
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ ctx->key_len = len;
+ ctx->need_calc_aes_dkey = 1;
+
+ memcpy(ctx->aes_enc_key, key, AES_KEY_LEN);
+ return 0;
+}
+
+static void copy_src_to_buf(struct req_progress *p, char *dbuf, int len)
+{
+ int ret;
+ void *sbuf;
+ int copy_len;
+
+ while (len) {
+ if (!p->sg_src_left) {
+ ret = sg_miter_next(&p->src_sg_it);
+ BUG_ON(!ret);
+ p->sg_src_left = p->src_sg_it.length;
+ p->src_start = 0;
+ }
+
+ sbuf = p->src_sg_it.addr + p->src_start;
+
+ copy_len = min(p->sg_src_left, len);
+ memcpy(dbuf, sbuf, copy_len);
+
+ p->src_start += copy_len;
+ p->sg_src_left -= copy_len;
+
+ len -= copy_len;
+ dbuf += copy_len;
+ }
+}
+
+static void setup_data_in(void)
+{
+ struct req_progress *p = &cpg->p;
+ int data_in_sram =
+ min(p->hw_nbytes - p->hw_processed_bytes, cpg->max_req_size);
+ copy_src_to_buf(p, cpg->sram + SRAM_DATA_IN_START + p->crypt_len,
+ data_in_sram - p->crypt_len);
+ p->crypt_len = data_in_sram;
+}
+
+static void mv_process_current_q(int first_block)
+{
+ struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req);
+ struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+ struct sec_accel_config op;
+
+ switch (req_ctx->op) {
+ case COP_AES_ECB:
+ op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB;
+ break;
+ case COP_AES_CBC:
+ default:
+ op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC;
+ op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) |
+ ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF);
+ if (first_block)
+ memcpy(cpg->sram + SRAM_DATA_IV, req->info, 16);
+ break;
+ }
+ if (req_ctx->decrypt) {
+ op.config |= CFG_DIR_DEC;
+ memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_dec_key,
+ AES_KEY_LEN);
+ } else {
+ op.config |= CFG_DIR_ENC;
+ memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_enc_key,
+ AES_KEY_LEN);
+ }
+
+ switch (ctx->key_len) {
+ case AES_KEYSIZE_128:
+ op.config |= CFG_AES_LEN_128;
+ break;
+ case AES_KEYSIZE_192:
+ op.config |= CFG_AES_LEN_192;
+ break;
+ case AES_KEYSIZE_256:
+ op.config |= CFG_AES_LEN_256;
+ break;
+ }
+ op.enc_p = ENC_P_SRC(SRAM_DATA_IN_START) |
+ ENC_P_DST(SRAM_DATA_OUT_START);
+ op.enc_key_p = SRAM_DATA_KEY_P;
+
+ setup_data_in();
+ op.enc_len = cpg->p.crypt_len;
+ memcpy(cpg->sram + SRAM_CONFIG, &op,
+ sizeof(struct sec_accel_config));
+
+ /* GO */
+ writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD);
+
+ /*
+ * XXX: add timer if the interrupt does not occur for some mystery
+ * reason
+ */
+}
+
+static void mv_crypto_algo_completion(void)
+{
+ struct ablkcipher_request *req = ablkcipher_request_cast(cpg->cur_req);
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ sg_miter_stop(&cpg->p.src_sg_it);
+ sg_miter_stop(&cpg->p.dst_sg_it);
+
+ if (req_ctx->op != COP_AES_CBC)
+ return ;
+
+ memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16);
+}
+
+static void mv_process_hash_current(int first_block)
+{
+ struct ahash_request *req = ahash_request_cast(cpg->cur_req);
+ const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req);
+ struct req_progress *p = &cpg->p;
+ struct sec_accel_config op = { 0 };
+ int is_last;
+
+ switch (req_ctx->op) {
+ case COP_SHA1:
+ default:
+ op.config = CFG_OP_MAC_ONLY | CFG_MACM_SHA1;
+ break;
+ case COP_HMAC_SHA1:
+ op.config = CFG_OP_MAC_ONLY | CFG_MACM_HMAC_SHA1;
+ memcpy(cpg->sram + SRAM_HMAC_IV_IN,
+ tfm_ctx->ivs, sizeof(tfm_ctx->ivs));
+ break;
+ }
+
+ op.mac_src_p =
+ MAC_SRC_DATA_P(SRAM_DATA_IN_START) | MAC_SRC_TOTAL_LEN((u32)
+ req_ctx->
+ count);
+
+ setup_data_in();
+
+ op.mac_digest =
+ MAC_DIGEST_P(SRAM_DIGEST_BUF) | MAC_FRAG_LEN(p->crypt_len);
+ op.mac_iv =
+ MAC_INNER_IV_P(SRAM_HMAC_IV_IN) |
+ MAC_OUTER_IV_P(SRAM_HMAC_IV_OUT);
+
+ is_last = req_ctx->last_chunk
+ && (p->hw_processed_bytes + p->crypt_len >= p->hw_nbytes)
+ && (req_ctx->count <= MAX_HW_HASH_SIZE);
+ if (req_ctx->first_hash) {
+ if (is_last)
+ op.config |= CFG_NOT_FRAG;
+ else
+ op.config |= CFG_FIRST_FRAG;
+
+ req_ctx->first_hash = 0;
+ } else {
+ if (is_last)
+ op.config |= CFG_LAST_FRAG;
+ else
+ op.config |= CFG_MID_FRAG;
+
+ if (first_block) {
+ writel(req_ctx->state[0], cpg->reg + DIGEST_INITIAL_VAL_A);
+ writel(req_ctx->state[1], cpg->reg + DIGEST_INITIAL_VAL_B);
+ writel(req_ctx->state[2], cpg->reg + DIGEST_INITIAL_VAL_C);
+ writel(req_ctx->state[3], cpg->reg + DIGEST_INITIAL_VAL_D);
+ writel(req_ctx->state[4], cpg->reg + DIGEST_INITIAL_VAL_E);
+ }
+ }
+
+ memcpy(cpg->sram + SRAM_CONFIG, &op, sizeof(struct sec_accel_config));
+
+ /* GO */
+ writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD);
+
+ /*
+ * XXX: add timer if the interrupt does not occur for some mystery
+ * reason
+ */
+}
+
+static inline int mv_hash_import_sha1_ctx(const struct mv_req_hash_ctx *ctx,
+ struct shash_desc *desc)
+{
+ int i;
+ struct sha1_state shash_state;
+
+ shash_state.count = ctx->count + ctx->count_add;
+ for (i = 0; i < 5; i++)
+ shash_state.state[i] = ctx->state[i];
+ memcpy(shash_state.buffer, ctx->buffer, sizeof(shash_state.buffer));
+ return crypto_shash_import(desc, &shash_state);
+}
+
+static int mv_hash_final_fallback(struct ahash_request *req)
+{
+ const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
+ struct mv_req_hash_ctx *req_ctx = ahash_request_ctx(req);
+ struct {
+ struct shash_desc shash;
+ char ctx[crypto_shash_descsize(tfm_ctx->fallback)];
+ } desc;
+ int rc;
+
+ desc.shash.tfm = tfm_ctx->fallback;
+ desc.shash.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+ if (unlikely(req_ctx->first_hash)) {
+ crypto_shash_init(&desc.shash);
+ crypto_shash_update(&desc.shash, req_ctx->buffer,
+ req_ctx->extra_bytes);
+ } else {
+ /* only SHA1 for now....
+ */
+ rc = mv_hash_import_sha1_ctx(req_ctx, &desc.shash);
+ if (rc)
+ goto out;
+ }
+ rc = crypto_shash_final(&desc.shash, req->result);
+out:
+ return rc;
+}
+
+static void mv_hash_algo_completion(void)
+{
+ struct ahash_request *req = ahash_request_cast(cpg->cur_req);
+ struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
+
+ if (ctx->extra_bytes)
+ copy_src_to_buf(&cpg->p, ctx->buffer, ctx->extra_bytes);
+ sg_miter_stop(&cpg->p.src_sg_it);
+
+ if (likely(ctx->last_chunk)) {
+ if (likely(ctx->count <= MAX_HW_HASH_SIZE)) {
+ memcpy(req->result, cpg->sram + SRAM_DIGEST_BUF,
+ crypto_ahash_digestsize(crypto_ahash_reqtfm
+ (req)));
+ } else
+ mv_hash_final_fallback(req);
+ } else {
+ ctx->state[0] = readl(cpg->reg + DIGEST_INITIAL_VAL_A);
+ ctx->state[1] = readl(cpg->reg + DIGEST_INITIAL_VAL_B);
+ ctx->state[2] = readl(cpg->reg + DIGEST_INITIAL_VAL_C);
+ ctx->state[3] = readl(cpg->reg + DIGEST_INITIAL_VAL_D);
+ ctx->state[4] = readl(cpg->reg + DIGEST_INITIAL_VAL_E);
+ }
+}
+
+static void dequeue_complete_req(void)
+{
+ struct crypto_async_request *req = cpg->cur_req;
+ void *buf;
+ int ret;
+ cpg->p.hw_processed_bytes += cpg->p.crypt_len;
+ if (cpg->p.copy_back) {
+ int need_copy_len = cpg->p.crypt_len;
+ int sram_offset = 0;
+ do {
+ int dst_copy;
+
+ if (!cpg->p.sg_dst_left) {
+ ret = sg_miter_next(&cpg->p.dst_sg_it);
+ BUG_ON(!ret);
+ cpg->p.sg_dst_left = cpg->p.dst_sg_it.length;
+ cpg->p.dst_start = 0;
+ }
+
+ buf = cpg->p.dst_sg_it.addr;
+ buf += cpg->p.dst_start;
+
+ dst_copy = min(need_copy_len, cpg->p.sg_dst_left);
+
+ memcpy(buf,
+ cpg->sram + SRAM_DATA_OUT_START + sram_offset,
+ dst_copy);
+ sram_offset += dst_copy;
+ cpg->p.sg_dst_left -= dst_copy;
+ need_copy_len -= dst_copy;
+ cpg->p.dst_start += dst_copy;
+ } while (need_copy_len > 0);
+ }
+
+ cpg->p.crypt_len = 0;
+
+ BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE);
+ if (cpg->p.hw_processed_bytes < cpg->p.hw_nbytes) {
+ /* process next scatter list entry */
+ cpg->eng_st = ENGINE_BUSY;
+ cpg->p.process(0);
+ } else {
+ cpg->p.complete();
+ cpg->eng_st = ENGINE_IDLE;
+ local_bh_disable();
+ req->complete(req, 0);
+ local_bh_enable();
+ }
+}
+
+static int count_sgs(struct scatterlist *sl, unsigned int total_bytes)
+{
+ int i = 0;
+ size_t cur_len;
+
+ while (sl) {
+ cur_len = sl[i].length;
+ ++i;
+ if (total_bytes > cur_len)
+ total_bytes -= cur_len;
+ else
+ break;
+ }
+
+ return i;
+}
+
+static void mv_start_new_crypt_req(struct ablkcipher_request *req)
+{
+ struct req_progress *p = &cpg->p;
+ int num_sgs;
+
+ cpg->cur_req = &req->base;
+ memset(p, 0, sizeof(struct req_progress));
+ p->hw_nbytes = req->nbytes;
+ p->complete = mv_crypto_algo_completion;
+ p->process = mv_process_current_q;
+ p->copy_back = 1;
+
+ num_sgs = count_sgs(req->src, req->nbytes);
+ sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG);
+
+ num_sgs = count_sgs(req->dst, req->nbytes);
+ sg_miter_start(&p->dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG);
+
+ mv_process_current_q(1);
+}
+
+static void mv_start_new_hash_req(struct ahash_request *req)
+{
+ struct req_progress *p = &cpg->p;
+ struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
+ int num_sgs, hw_bytes, old_extra_bytes, rc;
+ cpg->cur_req = &req->base;
+ memset(p, 0, sizeof(struct req_progress));
+ hw_bytes = req->nbytes + ctx->extra_bytes;
+ old_extra_bytes = ctx->extra_bytes;
+
+ ctx->extra_bytes = hw_bytes % SHA1_BLOCK_SIZE;
+ if (ctx->extra_bytes != 0
+ && (!ctx->last_chunk || ctx->count > MAX_HW_HASH_SIZE))
+ hw_bytes -= ctx->extra_bytes;
+ else
+ ctx->extra_bytes = 0;
+
+ num_sgs = count_sgs(req->src, req->nbytes);
+ sg_miter_start(&p->src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG);
+
+ if (hw_bytes) {
+ p->hw_nbytes = hw_bytes;
+ p->complete = mv_hash_algo_completion;
+ p->process = mv_process_hash_current;
+
+ if (unlikely(old_extra_bytes)) {
+ memcpy(cpg->sram + SRAM_DATA_IN_START, ctx->buffer,
+ old_extra_bytes);
+ p->crypt_len = old_extra_bytes;
+ }
+
+ mv_process_hash_current(1);
+ } else {
+ copy_src_to_buf(p, ctx->buffer + old_extra_bytes,
+ ctx->extra_bytes - old_extra_bytes);
+ sg_miter_stop(&p->src_sg_it);
+ if (ctx->last_chunk)
+ rc = mv_hash_final_fallback(req);
+ else
+ rc = 0;
+ cpg->eng_st = ENGINE_IDLE;
+ local_bh_disable();
+ req->base.complete(&req->base, rc);
+ local_bh_enable();
+ }
+}
+
+static int queue_manag(void *data)
+{
+ cpg->eng_st = ENGINE_IDLE;
+ do {
+ struct crypto_async_request *async_req = NULL;
+ struct crypto_async_request *backlog;
+
+ __set_current_state(TASK_INTERRUPTIBLE);
+
+ if (cpg->eng_st == ENGINE_W_DEQUEUE)
+ dequeue_complete_req();
+
+ spin_lock_irq(&cpg->lock);
+ if (cpg->eng_st == ENGINE_IDLE) {
+ backlog = crypto_get_backlog(&cpg->queue);
+ async_req = crypto_dequeue_request(&cpg->queue);
+ if (async_req) {
+ BUG_ON(cpg->eng_st != ENGINE_IDLE);
+ cpg->eng_st = ENGINE_BUSY;
+ }
+ }
+ spin_unlock_irq(&cpg->lock);
+
+ if (backlog) {
+ backlog->complete(backlog, -EINPROGRESS);
+ backlog = NULL;
+ }
+
+ if (async_req) {
+ if (async_req->tfm->__crt_alg->cra_type !=
+ &crypto_ahash_type) {
+ struct ablkcipher_request *req =
+ ablkcipher_request_cast(async_req);
+ mv_start_new_crypt_req(req);
+ } else {
+ struct ahash_request *req =
+ ahash_request_cast(async_req);
+ mv_start_new_hash_req(req);
+ }
+ async_req = NULL;
+ }
+
+ schedule();
+
+ } while (!kthread_should_stop());
+ return 0;
+}
+
+static int mv_handle_req(struct crypto_async_request *req)
+{
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&cpg->lock, flags);
+ ret = crypto_enqueue_request(&cpg->queue, req);
+ spin_unlock_irqrestore(&cpg->lock, flags);
+ wake_up_process(cpg->queue_th);
+ return ret;
+}
+
+static int mv_enc_aes_ecb(struct ablkcipher_request *req)
+{
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ req_ctx->op = COP_AES_ECB;
+ req_ctx->decrypt = 0;
+
+ return mv_handle_req(&req->base);
+}
+
+static int mv_dec_aes_ecb(struct ablkcipher_request *req)
+{
+ struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ req_ctx->op = COP_AES_ECB;
+ req_ctx->decrypt = 1;
+
+ compute_aes_dec_key(ctx);
+ return mv_handle_req(&req->base);
+}
+
+static int mv_enc_aes_cbc(struct ablkcipher_request *req)
+{
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ req_ctx->op = COP_AES_CBC;
+ req_ctx->decrypt = 0;
+
+ return mv_handle_req(&req->base);
+}
+
+static int mv_dec_aes_cbc(struct ablkcipher_request *req)
+{
+ struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req);
+
+ req_ctx->op = COP_AES_CBC;
+ req_ctx->decrypt = 1;
+
+ compute_aes_dec_key(ctx);
+ return mv_handle_req(&req->base);
+}
+
+static int mv_cra_init(struct crypto_tfm *tfm)
+{
+ tfm->crt_ablkcipher.reqsize = sizeof(struct mv_req_ctx);
+ return 0;
+}
+
+static void mv_init_hash_req_ctx(struct mv_req_hash_ctx *ctx, int op,
+ int is_last, unsigned int req_len,
+ int count_add)
+{
+ memset(ctx, 0, sizeof(*ctx));
+ ctx->op = op;
+ ctx->count = req_len;
+ ctx->first_hash = 1;
+ ctx->last_chunk = is_last;
+ ctx->count_add = count_add;
+}
+
+static void mv_update_hash_req_ctx(struct mv_req_hash_ctx *ctx, int is_last,
+ unsigned req_len)
+{
+ ctx->last_chunk = is_last;
+ ctx->count += req_len;
+}
+
+static int mv_hash_init(struct ahash_request *req)
+{
+ const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
+ mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 0, 0,
+ tfm_ctx->count_add);
+ return 0;
+}
+
+static int mv_hash_update(struct ahash_request *req)
+{
+ if (!req->nbytes)
+ return 0;
+
+ mv_update_hash_req_ctx(ahash_request_ctx(req), 0, req->nbytes);
+ return mv_handle_req(&req->base);
+}
+
+static int mv_hash_final(struct ahash_request *req)
+{
+ struct mv_req_hash_ctx *ctx = ahash_request_ctx(req);
+
+ ahash_request_set_crypt(req, NULL, req->result, 0);
+ mv_update_hash_req_ctx(ctx, 1, 0);
+ return mv_handle_req(&req->base);
+}
+
+static int mv_hash_finup(struct ahash_request *req)
+{
+ mv_update_hash_req_ctx(ahash_request_ctx(req), 1, req->nbytes);
+ return mv_handle_req(&req->base);
+}
+
+static int mv_hash_digest(struct ahash_request *req)
+{
+ const struct mv_tfm_hash_ctx *tfm_ctx = crypto_tfm_ctx(req->base.tfm);
+ mv_init_hash_req_ctx(ahash_request_ctx(req), tfm_ctx->op, 1,
+ req->nbytes, tfm_ctx->count_add);
+ return mv_handle_req(&req->base);
+}
+
+static void mv_hash_init_ivs(struct mv_tfm_hash_ctx *ctx, const void *istate,
+ const void *ostate)
+{
+ const struct sha1_state *isha1_state = istate, *osha1_state = ostate;
+ int i;
+ for (i = 0; i < 5; i++) {
+ ctx->ivs[i] = cpu_to_be32(isha1_state->state[i]);
+ ctx->ivs[i + 5] = cpu_to_be32(osha1_state->state[i]);
+ }
+}
+
+static int mv_hash_setkey(struct crypto_ahash *tfm, const u8 * key,
+ unsigned int keylen)
+{
+ int rc;
+ struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(&tfm->base);
+ int bs, ds, ss;
+
+ if (!ctx->base_hash)
+ return 0;
+
+ rc = crypto_shash_setkey(ctx->fallback, key, keylen);
+ if (rc)
+ return rc;
+
+ /* Can't see a way to extract the ipad/opad from the fallback tfm
+ so I'm basically copying code from the hmac module */
+ bs = crypto_shash_blocksize(ctx->base_hash);
+ ds = crypto_shash_digestsize(ctx->base_hash);
+ ss = crypto_shash_statesize(ctx->base_hash);
+
+ {
+ struct {
+ struct shash_desc shash;
+ char ctx[crypto_shash_descsize(ctx->base_hash)];
+ } desc;
+ unsigned int i;
+ char ipad[ss];
+ char opad[ss];
+
+ desc.shash.tfm = ctx->base_hash;
+ desc.shash.flags = crypto_shash_get_flags(ctx->base_hash) &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ if (keylen > bs) {
+ int err;
+
+ err =
+ crypto_shash_digest(&desc.shash, key, keylen, ipad);
+ if (err)
+ return err;
+
+ keylen = ds;
+ } else
+ memcpy(ipad, key, keylen);
+
+ memset(ipad + keylen, 0, bs - keylen);
+ memcpy(opad, ipad, bs);
+
+ for (i = 0; i < bs; i++) {
+ ipad[i] ^= 0x36;
+ opad[i] ^= 0x5c;
+ }
+
+ rc = crypto_shash_init(&desc.shash) ? :
+ crypto_shash_update(&desc.shash, ipad, bs) ? :
+ crypto_shash_export(&desc.shash, ipad) ? :
+ crypto_shash_init(&desc.shash) ? :
+ crypto_shash_update(&desc.shash, opad, bs) ? :
+ crypto_shash_export(&desc.shash, opad);
+
+ if (rc == 0)
+ mv_hash_init_ivs(ctx, ipad, opad);
+
+ return rc;
+ }
+}
+
+static int mv_cra_hash_init(struct crypto_tfm *tfm, const char *base_hash_name,
+ enum hash_op op, int count_add)
+{
+ const char *fallback_driver_name = tfm->__crt_alg->cra_name;
+ struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_shash *fallback_tfm = NULL;
+ struct crypto_shash *base_hash = NULL;
+ int err = -ENOMEM;
+
+ ctx->op = op;
+ ctx->count_add = count_add;
+
+ /* Allocate a fallback and abort if it failed. */
+ fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback_tfm)) {
+ printk(KERN_WARNING MV_CESA
+ "Fallback driver '%s' could not be loaded!\n",
+ fallback_driver_name);
+ err = PTR_ERR(fallback_tfm);
+ goto out;
+ }
+ ctx->fallback = fallback_tfm;
+
+ if (base_hash_name) {
+ /* Allocate a hash to compute the ipad/opad of hmac. */
+ base_hash = crypto_alloc_shash(base_hash_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(base_hash)) {
+ printk(KERN_WARNING MV_CESA
+ "Base driver '%s' could not be loaded!\n",
+ base_hash_name);
+ err = PTR_ERR(base_hash);
+ goto err_bad_base;
+ }
+ }
+ ctx->base_hash = base_hash;
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct mv_req_hash_ctx) +
+ crypto_shash_descsize(ctx->fallback));
+ return 0;
+err_bad_base:
+ crypto_free_shash(fallback_tfm);
+out:
+ return err;
+}
+
+static void mv_cra_hash_exit(struct crypto_tfm *tfm)
+{
+ struct mv_tfm_hash_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_shash(ctx->fallback);
+ if (ctx->base_hash)
+ crypto_free_shash(ctx->base_hash);
+}
+
+static int mv_cra_hash_sha1_init(struct crypto_tfm *tfm)
+{
+ return mv_cra_hash_init(tfm, NULL, COP_SHA1, 0);
+}
+
+static int mv_cra_hash_hmac_sha1_init(struct crypto_tfm *tfm)
+{
+ return mv_cra_hash_init(tfm, "sha1", COP_HMAC_SHA1, SHA1_BLOCK_SIZE);
+}
+
+irqreturn_t crypto_int(int irq, void *priv)
+{
+ u32 val;
+
+ val = readl(cpg->reg + SEC_ACCEL_INT_STATUS);
+ if (!(val & SEC_INT_ACCEL0_DONE))
+ return IRQ_NONE;
+
+ val &= ~SEC_INT_ACCEL0_DONE;
+ writel(val, cpg->reg + FPGA_INT_STATUS);
+ writel(val, cpg->reg + SEC_ACCEL_INT_STATUS);
+ BUG_ON(cpg->eng_st != ENGINE_BUSY);
+ cpg->eng_st = ENGINE_W_DEQUEUE;
+ wake_up_process(cpg->queue_th);
+ return IRQ_HANDLED;
+}
+
+struct crypto_alg mv_aes_alg_ecb = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "mv-ecb-aes",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 16,
+ .cra_ctxsize = sizeof(struct mv_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = mv_cra_init,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = mv_setkey_aes,
+ .encrypt = mv_enc_aes_ecb,
+ .decrypt = mv_dec_aes_ecb,
+ },
+ },
+};
+
+struct crypto_alg mv_aes_alg_cbc = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "mv-cbc-aes",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct mv_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = mv_cra_init,
+ .cra_u = {
+ .ablkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = mv_setkey_aes,
+ .encrypt = mv_enc_aes_cbc,
+ .decrypt = mv_dec_aes_cbc,
+ },
+ },
+};
+
+struct ahash_alg mv_sha1_alg = {
+ .init = mv_hash_init,
+ .update = mv_hash_update,
+ .final = mv_hash_final,
+ .finup = mv_hash_finup,
+ .digest = mv_hash_digest,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "mv-sha1",
+ .cra_priority = 300,
+ .cra_flags =
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx),
+ .cra_init = mv_cra_hash_sha1_init,
+ .cra_exit = mv_cra_hash_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+};
+
+struct ahash_alg mv_hmac_sha1_alg = {
+ .init = mv_hash_init,
+ .update = mv_hash_update,
+ .final = mv_hash_final,
+ .finup = mv_hash_finup,
+ .digest = mv_hash_digest,
+ .setkey = mv_hash_setkey,
+ .halg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "mv-hmac-sha1",
+ .cra_priority = 300,
+ .cra_flags =
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct mv_tfm_hash_ctx),
+ .cra_init = mv_cra_hash_hmac_sha1_init,
+ .cra_exit = mv_cra_hash_exit,
+ .cra_module = THIS_MODULE,
+ }
+ }
+};
+
+static int mv_probe(struct platform_device *pdev)
+{
+ struct crypto_priv *cp;
+ struct resource *res;
+ int irq;
+ int ret;
+
+ if (cpg) {
+ printk(KERN_ERR MV_CESA "Second crypto dev?\n");
+ return -EEXIST;
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs");
+ if (!res)
+ return -ENXIO;
+
+ cp = kzalloc(sizeof(*cp), GFP_KERNEL);
+ if (!cp)
+ return -ENOMEM;
+
+ spin_lock_init(&cp->lock);
+ crypto_init_queue(&cp->queue, 50);
+ cp->reg = ioremap(res->start, resource_size(res));
+ if (!cp->reg) {
+ ret = -ENOMEM;
+ goto err;
+ }
+
+ res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram");
+ if (!res) {
+ ret = -ENXIO;
+ goto err_unmap_reg;
+ }
+ cp->sram_size = resource_size(res);
+ cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE;
+ cp->sram = ioremap(res->start, cp->sram_size);
+ if (!cp->sram) {
+ ret = -ENOMEM;
+ goto err_unmap_reg;
+ }
+
+ irq = platform_get_irq(pdev, 0);
+ if (irq < 0 || irq == NO_IRQ) {
+ ret = irq;
+ goto err_unmap_sram;
+ }
+ cp->irq = irq;
+
+ platform_set_drvdata(pdev, cp);
+ cpg = cp;
+
+ cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto");
+ if (IS_ERR(cp->queue_th)) {
+ ret = PTR_ERR(cp->queue_th);
+ goto err_unmap_sram;
+ }
+
+ ret = request_irq(irq, crypto_int, IRQF_DISABLED, dev_name(&pdev->dev),
+ cp);
+ if (ret)
+ goto err_thread;
+
+ writel(SEC_INT_ACCEL0_DONE, cpg->reg + SEC_ACCEL_INT_MASK);
+ writel(SEC_CFG_STOP_DIG_ERR, cpg->reg + SEC_ACCEL_CFG);
+ writel(SRAM_CONFIG, cpg->reg + SEC_ACCEL_DESC_P0);
+
+ ret = crypto_register_alg(&mv_aes_alg_ecb);
+ if (ret) {
+ printk(KERN_WARNING MV_CESA
+ "Could not register aes-ecb driver\n");
+ goto err_irq;
+ }
+
+ ret = crypto_register_alg(&mv_aes_alg_cbc);
+ if (ret) {
+ printk(KERN_WARNING MV_CESA
+ "Could not register aes-cbc driver\n");
+ goto err_unreg_ecb;
+ }
+
+ ret = crypto_register_ahash(&mv_sha1_alg);
+ if (ret == 0)
+ cpg->has_sha1 = 1;
+ else
+ printk(KERN_WARNING MV_CESA "Could not register sha1 driver\n");
+
+ ret = crypto_register_ahash(&mv_hmac_sha1_alg);
+ if (ret == 0) {
+ cpg->has_hmac_sha1 = 1;
+ } else {
+ printk(KERN_WARNING MV_CESA
+ "Could not register hmac-sha1 driver\n");
+ }
+
+ return 0;
+err_unreg_ecb:
+ crypto_unregister_alg(&mv_aes_alg_ecb);
+err_irq:
+ free_irq(irq, cp);
+err_thread:
+ kthread_stop(cp->queue_th);
+err_unmap_sram:
+ iounmap(cp->sram);
+err_unmap_reg:
+ iounmap(cp->reg);
+err:
+ kfree(cp);
+ cpg = NULL;
+ platform_set_drvdata(pdev, NULL);
+ return ret;
+}
+
+static int mv_remove(struct platform_device *pdev)
+{
+ struct crypto_priv *cp = platform_get_drvdata(pdev);
+
+ crypto_unregister_alg(&mv_aes_alg_ecb);
+ crypto_unregister_alg(&mv_aes_alg_cbc);
+ if (cp->has_sha1)
+ crypto_unregister_ahash(&mv_sha1_alg);
+ if (cp->has_hmac_sha1)
+ crypto_unregister_ahash(&mv_hmac_sha1_alg);
+ kthread_stop(cp->queue_th);
+ free_irq(cp->irq, cp);
+ memset(cp->sram, 0, cp->sram_size);
+ iounmap(cp->sram);
+ iounmap(cp->reg);
+ kfree(cp);
+ cpg = NULL;
+ return 0;
+}
+
+static struct platform_driver marvell_crypto = {
+ .probe = mv_probe,
+ .remove = mv_remove,
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = "mv_crypto",
+ },
+};
+MODULE_ALIAS("platform:mv_crypto");
+
+module_platform_driver(marvell_crypto);
+
+MODULE_AUTHOR("Sebastian Andrzej Siewior <sebastian@breakpoint.cc>");
+MODULE_DESCRIPTION("Support for Marvell's cryptographic engine");
+MODULE_LICENSE("GPL");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.h b/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.h
new file mode 100644
index 0000000..08fcb11
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/mv_cesa.h
@@ -0,0 +1,149 @@
+#ifndef __MV_CRYPTO_H__
+
+#define DIGEST_INITIAL_VAL_A 0xdd00
+#define DIGEST_INITIAL_VAL_B 0xdd04
+#define DIGEST_INITIAL_VAL_C 0xdd08
+#define DIGEST_INITIAL_VAL_D 0xdd0c
+#define DIGEST_INITIAL_VAL_E 0xdd10
+#define DES_CMD_REG 0xdd58
+
+#define SEC_ACCEL_CMD 0xde00
+#define SEC_CMD_EN_SEC_ACCL0 (1 << 0)
+#define SEC_CMD_EN_SEC_ACCL1 (1 << 1)
+#define SEC_CMD_DISABLE_SEC (1 << 2)
+
+#define SEC_ACCEL_DESC_P0 0xde04
+#define SEC_DESC_P0_PTR(x) (x)
+
+#define SEC_ACCEL_DESC_P1 0xde14
+#define SEC_DESC_P1_PTR(x) (x)
+
+#define SEC_ACCEL_CFG 0xde08
+#define SEC_CFG_STOP_DIG_ERR (1 << 0)
+#define SEC_CFG_CH0_W_IDMA (1 << 7)
+#define SEC_CFG_CH1_W_IDMA (1 << 8)
+#define SEC_CFG_ACT_CH0_IDMA (1 << 9)
+#define SEC_CFG_ACT_CH1_IDMA (1 << 10)
+
+#define SEC_ACCEL_STATUS 0xde0c
+#define SEC_ST_ACT_0 (1 << 0)
+#define SEC_ST_ACT_1 (1 << 1)
+
+/*
+ * FPGA_INT_STATUS looks like a FPGA leftover and is documented only in Errata
+ * 4.12. It looks like that it was part of an IRQ-controller in FPGA and
+ * someone forgot to remove it while switching to the core and moving to
+ * SEC_ACCEL_INT_STATUS.
+ */
+#define FPGA_INT_STATUS 0xdd68
+#define SEC_ACCEL_INT_STATUS 0xde20
+#define SEC_INT_AUTH_DONE (1 << 0)
+#define SEC_INT_DES_E_DONE (1 << 1)
+#define SEC_INT_AES_E_DONE (1 << 2)
+#define SEC_INT_AES_D_DONE (1 << 3)
+#define SEC_INT_ENC_DONE (1 << 4)
+#define SEC_INT_ACCEL0_DONE (1 << 5)
+#define SEC_INT_ACCEL1_DONE (1 << 6)
+#define SEC_INT_ACC0_IDMA_DONE (1 << 7)
+#define SEC_INT_ACC1_IDMA_DONE (1 << 8)
+
+#define SEC_ACCEL_INT_MASK 0xde24
+
+#define AES_KEY_LEN (8 * 4)
+
+struct sec_accel_config {
+
+ u32 config;
+#define CFG_OP_MAC_ONLY 0
+#define CFG_OP_CRYPT_ONLY 1
+#define CFG_OP_MAC_CRYPT 2
+#define CFG_OP_CRYPT_MAC 3
+#define CFG_MACM_MD5 (4 << 4)
+#define CFG_MACM_SHA1 (5 << 4)
+#define CFG_MACM_HMAC_MD5 (6 << 4)
+#define CFG_MACM_HMAC_SHA1 (7 << 4)
+#define CFG_ENCM_DES (1 << 8)
+#define CFG_ENCM_3DES (2 << 8)
+#define CFG_ENCM_AES (3 << 8)
+#define CFG_DIR_ENC (0 << 12)
+#define CFG_DIR_DEC (1 << 12)
+#define CFG_ENC_MODE_ECB (0 << 16)
+#define CFG_ENC_MODE_CBC (1 << 16)
+#define CFG_3DES_EEE (0 << 20)
+#define CFG_3DES_EDE (1 << 20)
+#define CFG_AES_LEN_128 (0 << 24)
+#define CFG_AES_LEN_192 (1 << 24)
+#define CFG_AES_LEN_256 (2 << 24)
+#define CFG_NOT_FRAG (0 << 30)
+#define CFG_FIRST_FRAG (1 << 30)
+#define CFG_LAST_FRAG (2 << 30)
+#define CFG_MID_FRAG (3 << 30)
+
+ u32 enc_p;
+#define ENC_P_SRC(x) (x)
+#define ENC_P_DST(x) ((x) << 16)
+
+ u32 enc_len;
+#define ENC_LEN(x) (x)
+
+ u32 enc_key_p;
+#define ENC_KEY_P(x) (x)
+
+ u32 enc_iv;
+#define ENC_IV_POINT(x) ((x) << 0)
+#define ENC_IV_BUF_POINT(x) ((x) << 16)
+
+ u32 mac_src_p;
+#define MAC_SRC_DATA_P(x) (x)
+#define MAC_SRC_TOTAL_LEN(x) ((x) << 16)
+
+ u32 mac_digest;
+#define MAC_DIGEST_P(x) (x)
+#define MAC_FRAG_LEN(x) ((x) << 16)
+ u32 mac_iv;
+#define MAC_INNER_IV_P(x) (x)
+#define MAC_OUTER_IV_P(x) ((x) << 16)
+}__attribute__ ((packed));
+ /*
+ * /-----------\ 0
+ * | ACCEL CFG | 4 * 8
+ * |-----------| 0x20
+ * | CRYPT KEY | 8 * 4
+ * |-----------| 0x40
+ * | IV IN | 4 * 4
+ * |-----------| 0x40 (inplace)
+ * | IV BUF | 4 * 4
+ * |-----------| 0x80
+ * | DATA IN | 16 * x (max ->max_req_size)
+ * |-----------| 0x80 (inplace operation)
+ * | DATA OUT | 16 * x (max ->max_req_size)
+ * \-----------/ SRAM size
+ */
+
+ /* Hashing memory map:
+ * /-----------\ 0
+ * | ACCEL CFG | 4 * 8
+ * |-----------| 0x20
+ * | Inner IV | 5 * 4
+ * |-----------| 0x34
+ * | Outer IV | 5 * 4
+ * |-----------| 0x48
+ * | Output BUF| 5 * 4
+ * |-----------| 0x80
+ * | DATA IN | 64 * x (max ->max_req_size)
+ * \-----------/ SRAM size
+ */
+#define SRAM_CONFIG 0x00
+#define SRAM_DATA_KEY_P 0x20
+#define SRAM_DATA_IV 0x40
+#define SRAM_DATA_IV_BUF 0x40
+#define SRAM_DATA_IN_START 0x80
+#define SRAM_DATA_OUT_START 0x80
+
+#define SRAM_HMAC_IV_IN 0x20
+#define SRAM_HMAC_IV_OUT 0x34
+#define SRAM_DIGEST_BUF 0x48
+
+#define SRAM_CFG_SPACE 0x80
+
+#endif
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/n2_asm.S b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_asm.S
new file mode 100644
index 0000000..f7c7937
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_asm.S
@@ -0,0 +1,95 @@
+/* n2_asm.S: Hypervisor calls for NCS support.
+ *
+ * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
+ */
+
+#include <linux/linkage.h>
+#include <asm/hypervisor.h>
+#include "n2_core.h"
+
+ /* o0: queue type
+ * o1: RA of queue
+ * o2: num entries in queue
+ * o3: address of queue handle return
+ */
+ENTRY(sun4v_ncs_qconf)
+ mov HV_FAST_NCS_QCONF, %o5
+ ta HV_FAST_TRAP
+ stx %o1, [%o3]
+ retl
+ nop
+ENDPROC(sun4v_ncs_qconf)
+
+ /* %o0: queue handle
+ * %o1: address of queue type return
+ * %o2: address of queue base address return
+ * %o3: address of queue num entries return
+ */
+ENTRY(sun4v_ncs_qinfo)
+ mov %o1, %g1
+ mov %o2, %g2
+ mov %o3, %g3
+ mov HV_FAST_NCS_QINFO, %o5
+ ta HV_FAST_TRAP
+ stx %o1, [%g1]
+ stx %o2, [%g2]
+ stx %o3, [%g3]
+ retl
+ nop
+ENDPROC(sun4v_ncs_qinfo)
+
+ /* %o0: queue handle
+ * %o1: address of head offset return
+ */
+ENTRY(sun4v_ncs_gethead)
+ mov %o1, %o2
+ mov HV_FAST_NCS_GETHEAD, %o5
+ ta HV_FAST_TRAP
+ stx %o1, [%o2]
+ retl
+ nop
+ENDPROC(sun4v_ncs_gethead)
+
+ /* %o0: queue handle
+ * %o1: address of tail offset return
+ */
+ENTRY(sun4v_ncs_gettail)
+ mov %o1, %o2
+ mov HV_FAST_NCS_GETTAIL, %o5
+ ta HV_FAST_TRAP
+ stx %o1, [%o2]
+ retl
+ nop
+ENDPROC(sun4v_ncs_gettail)
+
+ /* %o0: queue handle
+ * %o1: new tail offset
+ */
+ENTRY(sun4v_ncs_settail)
+ mov HV_FAST_NCS_SETTAIL, %o5
+ ta HV_FAST_TRAP
+ retl
+ nop
+ENDPROC(sun4v_ncs_settail)
+
+ /* %o0: queue handle
+ * %o1: address of devino return
+ */
+ENTRY(sun4v_ncs_qhandle_to_devino)
+ mov %o1, %o2
+ mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5
+ ta HV_FAST_TRAP
+ stx %o1, [%o2]
+ retl
+ nop
+ENDPROC(sun4v_ncs_qhandle_to_devino)
+
+ /* %o0: queue handle
+ * %o1: new head offset
+ */
+ENTRY(sun4v_ncs_sethead_marker)
+ mov HV_FAST_NCS_SETHEAD_MARKER, %o5
+ ta HV_FAST_TRAP
+ retl
+ nop
+ENDPROC(sun4v_ncs_sethead_marker)
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.c b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.c
new file mode 100644
index 0000000..67b97c5
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.c
@@ -0,0 +1,2271 @@
+/* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
+ *
+ * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_device.h>
+#include <linux/cpumask.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/crypto.h>
+#include <crypto/md5.h>
+#include <crypto/sha.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <linux/mutex.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/algapi.h>
+
+#include <asm/hypervisor.h>
+#include <asm/mdesc.h>
+
+#include "n2_core.h"
+
+#define DRV_MODULE_NAME "n2_crypto"
+#define DRV_MODULE_VERSION "0.2"
+#define DRV_MODULE_RELDATE "July 28, 2011"
+
+static char version[] __devinitdata =
+ DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
+
+MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
+MODULE_DESCRIPTION("Niagara2 Crypto driver");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(DRV_MODULE_VERSION);
+
+#define N2_CRA_PRIORITY 300
+
+static DEFINE_MUTEX(spu_lock);
+
+struct spu_queue {
+ cpumask_t sharing;
+ unsigned long qhandle;
+
+ spinlock_t lock;
+ u8 q_type;
+ void *q;
+ unsigned long head;
+ unsigned long tail;
+ struct list_head jobs;
+
+ unsigned long devino;
+
+ char irq_name[32];
+ unsigned int irq;
+
+ struct list_head list;
+};
+
+static struct spu_queue **cpu_to_cwq;
+static struct spu_queue **cpu_to_mau;
+
+static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off)
+{
+ if (q->q_type == HV_NCS_QTYPE_MAU) {
+ off += MAU_ENTRY_SIZE;
+ if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES))
+ off = 0;
+ } else {
+ off += CWQ_ENTRY_SIZE;
+ if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES))
+ off = 0;
+ }
+ return off;
+}
+
+struct n2_request_common {
+ struct list_head entry;
+ unsigned int offset;
+};
+#define OFFSET_NOT_RUNNING (~(unsigned int)0)
+
+/* An async job request records the final tail value it used in
+ * n2_request_common->offset, test to see if that offset is in
+ * the range old_head, new_head, inclusive.
+ */
+static inline bool job_finished(struct spu_queue *q, unsigned int offset,
+ unsigned long old_head, unsigned long new_head)
+{
+ if (old_head <= new_head) {
+ if (offset > old_head && offset <= new_head)
+ return true;
+ } else {
+ if (offset > old_head || offset <= new_head)
+ return true;
+ }
+ return false;
+}
+
+/* When the HEAD marker is unequal to the actual HEAD, we get
+ * a virtual device INO interrupt. We should process the
+ * completed CWQ entries and adjust the HEAD marker to clear
+ * the IRQ.
+ */
+static irqreturn_t cwq_intr(int irq, void *dev_id)
+{
+ unsigned long off, new_head, hv_ret;
+ struct spu_queue *q = dev_id;
+
+ pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
+ smp_processor_id(), q->qhandle);
+
+ spin_lock(&q->lock);
+
+ hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head);
+
+ pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
+ smp_processor_id(), new_head, hv_ret);
+
+ for (off = q->head; off != new_head; off = spu_next_offset(q, off)) {
+ /* XXX ... XXX */
+ }
+
+ hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head);
+ if (hv_ret == HV_EOK)
+ q->head = new_head;
+
+ spin_unlock(&q->lock);
+
+ return IRQ_HANDLED;
+}
+
+static irqreturn_t mau_intr(int irq, void *dev_id)
+{
+ struct spu_queue *q = dev_id;
+ unsigned long head, hv_ret;
+
+ spin_lock(&q->lock);
+
+ pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
+ smp_processor_id(), q->qhandle);
+
+ hv_ret = sun4v_ncs_gethead(q->qhandle, &head);
+
+ pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
+ smp_processor_id(), head, hv_ret);
+
+ sun4v_ncs_sethead_marker(q->qhandle, head);
+
+ spin_unlock(&q->lock);
+
+ return IRQ_HANDLED;
+}
+
+static void *spu_queue_next(struct spu_queue *q, void *cur)
+{
+ return q->q + spu_next_offset(q, cur - q->q);
+}
+
+static int spu_queue_num_free(struct spu_queue *q)
+{
+ unsigned long head = q->head;
+ unsigned long tail = q->tail;
+ unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES);
+ unsigned long diff;
+
+ if (head > tail)
+ diff = head - tail;
+ else
+ diff = (end - tail) + head;
+
+ return (diff / CWQ_ENTRY_SIZE) - 1;
+}
+
+static void *spu_queue_alloc(struct spu_queue *q, int num_entries)
+{
+ int avail = spu_queue_num_free(q);
+
+ if (avail >= num_entries)
+ return q->q + q->tail;
+
+ return NULL;
+}
+
+static unsigned long spu_queue_submit(struct spu_queue *q, void *last)
+{
+ unsigned long hv_ret, new_tail;
+
+ new_tail = spu_next_offset(q, last - q->q);
+
+ hv_ret = sun4v_ncs_settail(q->qhandle, new_tail);
+ if (hv_ret == HV_EOK)
+ q->tail = new_tail;
+ return hv_ret;
+}
+
+static u64 control_word_base(unsigned int len, unsigned int hmac_key_len,
+ int enc_type, int auth_type,
+ unsigned int hash_len,
+ bool sfas, bool sob, bool eob, bool encrypt,
+ int opcode)
+{
+ u64 word = (len - 1) & CONTROL_LEN;
+
+ word |= ((u64) opcode << CONTROL_OPCODE_SHIFT);
+ word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT);
+ word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT);
+ if (sfas)
+ word |= CONTROL_STORE_FINAL_AUTH_STATE;
+ if (sob)
+ word |= CONTROL_START_OF_BLOCK;
+ if (eob)
+ word |= CONTROL_END_OF_BLOCK;
+ if (encrypt)
+ word |= CONTROL_ENCRYPT;
+ if (hmac_key_len)
+ word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT;
+ if (hash_len)
+ word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT;
+
+ return word;
+}
+
+#if 0
+static inline bool n2_should_run_async(struct spu_queue *qp, int this_len)
+{
+ if (this_len >= 64 ||
+ qp->head != qp->tail)
+ return true;
+ return false;
+}
+#endif
+
+struct n2_ahash_alg {
+ struct list_head entry;
+ const char *hash_zero;
+ const u32 *hash_init;
+ u8 hw_op_hashsz;
+ u8 digest_size;
+ u8 auth_type;
+ u8 hmac_type;
+ struct ahash_alg alg;
+};
+
+static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct ahash_alg *ahash_alg;
+
+ ahash_alg = container_of(alg, struct ahash_alg, halg.base);
+
+ return container_of(ahash_alg, struct n2_ahash_alg, alg);
+}
+
+struct n2_hmac_alg {
+ const char *child_alg;
+ struct n2_ahash_alg derived;
+};
+
+static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct ahash_alg *ahash_alg;
+
+ ahash_alg = container_of(alg, struct ahash_alg, halg.base);
+
+ return container_of(ahash_alg, struct n2_hmac_alg, derived.alg);
+}
+
+struct n2_hash_ctx {
+ struct crypto_ahash *fallback_tfm;
+};
+
+#define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */
+
+struct n2_hmac_ctx {
+ struct n2_hash_ctx base;
+
+ struct crypto_shash *child_shash;
+
+ int hash_key_len;
+ unsigned char hash_key[N2_HASH_KEY_MAX];
+};
+
+struct n2_hash_req_ctx {
+ union {
+ struct md5_state md5;
+ struct sha1_state sha1;
+ struct sha256_state sha256;
+ } u;
+
+ struct ahash_request fallback_req;
+};
+
+static int n2_hash_async_init(struct ahash_request *req)
+{
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_ahash_init(&rctx->fallback_req);
+}
+
+static int n2_hash_async_update(struct ahash_request *req)
+{
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = req->nbytes;
+ rctx->fallback_req.src = req->src;
+
+ return crypto_ahash_update(&rctx->fallback_req);
+}
+
+static int n2_hash_async_final(struct ahash_request *req)
+{
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.result = req->result;
+
+ return crypto_ahash_final(&rctx->fallback_req);
+}
+
+static int n2_hash_async_finup(struct ahash_request *req)
+{
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = req->nbytes;
+ rctx->fallback_req.src = req->src;
+ rctx->fallback_req.result = req->result;
+
+ return crypto_ahash_finup(&rctx->fallback_req);
+}
+
+static int n2_hash_cra_init(struct crypto_tfm *tfm)
+{
+ const char *fallback_driver_name = tfm->__crt_alg->cra_name;
+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct crypto_ahash *fallback_tfm;
+ int err;
+
+ fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback_tfm)) {
+ pr_warning("Fallback driver '%s' could not be loaded!\n",
+ fallback_driver_name);
+ err = PTR_ERR(fallback_tfm);
+ goto out;
+ }
+
+ crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
+ crypto_ahash_reqsize(fallback_tfm)));
+
+ ctx->fallback_tfm = fallback_tfm;
+ return 0;
+
+out:
+ return err;
+}
+
+static void n2_hash_cra_exit(struct crypto_tfm *tfm)
+{
+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash);
+
+ crypto_free_ahash(ctx->fallback_tfm);
+}
+
+static int n2_hmac_cra_init(struct crypto_tfm *tfm)
+{
+ const char *fallback_driver_name = tfm->__crt_alg->cra_name;
+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+ struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
+ struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm);
+ struct crypto_ahash *fallback_tfm;
+ struct crypto_shash *child_shash;
+ int err;
+
+ fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback_tfm)) {
+ pr_warning("Fallback driver '%s' could not be loaded!\n",
+ fallback_driver_name);
+ err = PTR_ERR(fallback_tfm);
+ goto out;
+ }
+
+ child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0);
+ if (IS_ERR(child_shash)) {
+ pr_warning("Child shash '%s' could not be loaded!\n",
+ n2alg->child_alg);
+ err = PTR_ERR(child_shash);
+ goto out_free_fallback;
+ }
+
+ crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) +
+ crypto_ahash_reqsize(fallback_tfm)));
+
+ ctx->child_shash = child_shash;
+ ctx->base.fallback_tfm = fallback_tfm;
+ return 0;
+
+out_free_fallback:
+ crypto_free_ahash(fallback_tfm);
+
+out:
+ return err;
+}
+
+static void n2_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
+ struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash);
+
+ crypto_free_ahash(ctx->base.fallback_tfm);
+ crypto_free_shash(ctx->child_shash);
+}
+
+static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct crypto_shash *child_shash = ctx->child_shash;
+ struct crypto_ahash *fallback_tfm;
+ struct {
+ struct shash_desc shash;
+ char ctx[crypto_shash_descsize(child_shash)];
+ } desc;
+ int err, bs, ds;
+
+ fallback_tfm = ctx->base.fallback_tfm;
+ err = crypto_ahash_setkey(fallback_tfm, key, keylen);
+ if (err)
+ return err;
+
+ desc.shash.tfm = child_shash;
+ desc.shash.flags = crypto_ahash_get_flags(tfm) &
+ CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ bs = crypto_shash_blocksize(child_shash);
+ ds = crypto_shash_digestsize(child_shash);
+ BUG_ON(ds > N2_HASH_KEY_MAX);
+ if (keylen > bs) {
+ err = crypto_shash_digest(&desc.shash, key, keylen,
+ ctx->hash_key);
+ if (err)
+ return err;
+ keylen = ds;
+ } else if (keylen <= N2_HASH_KEY_MAX)
+ memcpy(ctx->hash_key, key, keylen);
+
+ ctx->hash_key_len = keylen;
+
+ return err;
+}
+
+static unsigned long wait_for_tail(struct spu_queue *qp)
+{
+ unsigned long head, hv_ret;
+
+ do {
+ hv_ret = sun4v_ncs_gethead(qp->qhandle, &head);
+ if (hv_ret != HV_EOK) {
+ pr_err("Hypervisor error on gethead\n");
+ break;
+ }
+ if (head == qp->tail) {
+ qp->head = head;
+ break;
+ }
+ } while (1);
+ return hv_ret;
+}
+
+static unsigned long submit_and_wait_for_tail(struct spu_queue *qp,
+ struct cwq_initial_entry *ent)
+{
+ unsigned long hv_ret = spu_queue_submit(qp, ent);
+
+ if (hv_ret == HV_EOK)
+ hv_ret = wait_for_tail(qp);
+
+ return hv_ret;
+}
+
+static int n2_do_async_digest(struct ahash_request *req,
+ unsigned int auth_type, unsigned int digest_size,
+ unsigned int result_size, void *hash_loc,
+ unsigned long auth_key, unsigned int auth_key_len)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct cwq_initial_entry *ent;
+ struct crypto_hash_walk walk;
+ struct spu_queue *qp;
+ unsigned long flags;
+ int err = -ENODEV;
+ int nbytes, cpu;
+
+ /* The total effective length of the operation may not
+ * exceed 2^16.
+ */
+ if (unlikely(req->nbytes > (1 << 16))) {
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags =
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = req->nbytes;
+ rctx->fallback_req.src = req->src;
+ rctx->fallback_req.result = req->result;
+
+ return crypto_ahash_digest(&rctx->fallback_req);
+ }
+
+ nbytes = crypto_hash_walk_first(req, &walk);
+
+ cpu = get_cpu();
+ qp = cpu_to_cwq[cpu];
+ if (!qp)
+ goto out;
+
+ spin_lock_irqsave(&qp->lock, flags);
+
+ /* XXX can do better, improve this later by doing a by-hand scatterlist
+ * XXX walk, etc.
+ */
+ ent = qp->q + qp->tail;
+
+ ent->control = control_word_base(nbytes, auth_key_len, 0,
+ auth_type, digest_size,
+ false, true, false, false,
+ OPCODE_INPLACE_BIT |
+ OPCODE_AUTH_MAC);
+ ent->src_addr = __pa(walk.data);
+ ent->auth_key_addr = auth_key;
+ ent->auth_iv_addr = __pa(hash_loc);
+ ent->final_auth_state_addr = 0UL;
+ ent->enc_key_addr = 0UL;
+ ent->enc_iv_addr = 0UL;
+ ent->dest_addr = __pa(hash_loc);
+
+ nbytes = crypto_hash_walk_done(&walk, 0);
+ while (nbytes > 0) {
+ ent = spu_queue_next(qp, ent);
+
+ ent->control = (nbytes - 1);
+ ent->src_addr = __pa(walk.data);
+ ent->auth_key_addr = 0UL;
+ ent->auth_iv_addr = 0UL;
+ ent->final_auth_state_addr = 0UL;
+ ent->enc_key_addr = 0UL;
+ ent->enc_iv_addr = 0UL;
+ ent->dest_addr = 0UL;
+
+ nbytes = crypto_hash_walk_done(&walk, 0);
+ }
+ ent->control |= CONTROL_END_OF_BLOCK;
+
+ if (submit_and_wait_for_tail(qp, ent) != HV_EOK)
+ err = -EINVAL;
+ else
+ err = 0;
+
+ spin_unlock_irqrestore(&qp->lock, flags);
+
+ if (!err)
+ memcpy(req->result, hash_loc, result_size);
+out:
+ put_cpu();
+
+ return err;
+}
+
+static int n2_hash_async_digest(struct ahash_request *req)
+{
+ struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm);
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ int ds;
+
+ ds = n2alg->digest_size;
+ if (unlikely(req->nbytes == 0)) {
+ memcpy(req->result, n2alg->hash_zero, ds);
+ return 0;
+ }
+ memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz);
+
+ return n2_do_async_digest(req, n2alg->auth_type,
+ n2alg->hw_op_hashsz, ds,
+ &rctx->u, 0UL, 0);
+}
+
+static int n2_hmac_async_digest(struct ahash_request *req)
+{
+ struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm);
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm);
+ int ds;
+
+ ds = n2alg->derived.digest_size;
+ if (unlikely(req->nbytes == 0) ||
+ unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) {
+ struct n2_hash_req_ctx *rctx = ahash_request_ctx(req);
+ struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
+ rctx->fallback_req.base.flags =
+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ rctx->fallback_req.nbytes = req->nbytes;
+ rctx->fallback_req.src = req->src;
+ rctx->fallback_req.result = req->result;
+
+ return crypto_ahash_digest(&rctx->fallback_req);
+ }
+ memcpy(&rctx->u, n2alg->derived.hash_init,
+ n2alg->derived.hw_op_hashsz);
+
+ return n2_do_async_digest(req, n2alg->derived.hmac_type,
+ n2alg->derived.hw_op_hashsz, ds,
+ &rctx->u,
+ __pa(&ctx->hash_key),
+ ctx->hash_key_len);
+}
+
+struct n2_cipher_context {
+ int key_len;
+ int enc_type;
+ union {
+ u8 aes[AES_MAX_KEY_SIZE];
+ u8 des[DES_KEY_SIZE];
+ u8 des3[3 * DES_KEY_SIZE];
+ u8 arc4[258]; /* S-box, X, Y */
+ } key;
+};
+
+#define N2_CHUNK_ARR_LEN 16
+
+struct n2_crypto_chunk {
+ struct list_head entry;
+ unsigned long iv_paddr : 44;
+ unsigned long arr_len : 20;
+ unsigned long dest_paddr;
+ unsigned long dest_final;
+ struct {
+ unsigned long src_paddr : 44;
+ unsigned long src_len : 20;
+ } arr[N2_CHUNK_ARR_LEN];
+};
+
+struct n2_request_context {
+ struct ablkcipher_walk walk;
+ struct list_head chunk_list;
+ struct n2_crypto_chunk chunk;
+ u8 temp_iv[16];
+};
+
+/* The SPU allows some level of flexibility for partial cipher blocks
+ * being specified in a descriptor.
+ *
+ * It merely requires that every descriptor's length field is at least
+ * as large as the cipher block size. This means that a cipher block
+ * can span at most 2 descriptors. However, this does not allow a
+ * partial block to span into the final descriptor as that would
+ * violate the rule (since every descriptor's length must be at lest
+ * the block size). So, for example, assuming an 8 byte block size:
+ *
+ * 0xe --> 0xa --> 0x8
+ *
+ * is a valid length sequence, whereas:
+ *
+ * 0xe --> 0xb --> 0x7
+ *
+ * is not a valid sequence.
+ */
+
+struct n2_cipher_alg {
+ struct list_head entry;
+ u8 enc_type;
+ struct crypto_alg alg;
+};
+
+static inline struct n2_cipher_alg *n2_cipher_alg(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+
+ return container_of(alg, struct n2_cipher_alg, alg);
+}
+
+struct n2_cipher_request_context {
+ struct ablkcipher_walk walk;
+};
+
+static int n2_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
+ struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
+
+ ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK);
+
+ switch (keylen) {
+ case AES_KEYSIZE_128:
+ ctx->enc_type |= ENC_TYPE_ALG_AES128;
+ break;
+ case AES_KEYSIZE_192:
+ ctx->enc_type |= ENC_TYPE_ALG_AES192;
+ break;
+ case AES_KEYSIZE_256:
+ ctx->enc_type |= ENC_TYPE_ALG_AES256;
+ break;
+ default:
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ ctx->key_len = keylen;
+ memcpy(ctx->key.aes, key, keylen);
+ return 0;
+}
+
+static int n2_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
+ struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
+ u32 tmp[DES_EXPKEY_WORDS];
+ int err;
+
+ ctx->enc_type = n2alg->enc_type;
+
+ if (keylen != DES_KEY_SIZE) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ err = des_ekey(tmp, key);
+ if (err == 0 && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
+ tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ return -EINVAL;
+ }
+
+ ctx->key_len = keylen;
+ memcpy(ctx->key.des, key, keylen);
+ return 0;
+}
+
+static int n2_3des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
+ struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
+
+ ctx->enc_type = n2alg->enc_type;
+
+ if (keylen != (3 * DES_KEY_SIZE)) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+ ctx->key_len = keylen;
+ memcpy(ctx->key.des3, key, keylen);
+ return 0;
+}
+
+static int n2_arc4_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
+ struct n2_cipher_alg *n2alg = n2_cipher_alg(tfm);
+ u8 *s = ctx->key.arc4;
+ u8 *x = s + 256;
+ u8 *y = x + 1;
+ int i, j, k;
+
+ ctx->enc_type = n2alg->enc_type;
+
+ j = k = 0;
+ *x = 0;
+ *y = 0;
+ for (i = 0; i < 256; i++)
+ s[i] = i;
+ for (i = 0; i < 256; i++) {
+ u8 a = s[i];
+ j = (j + key[k] + a) & 0xff;
+ s[i] = s[j];
+ s[j] = a;
+ if (++k >= keylen)
+ k = 0;
+ }
+
+ return 0;
+}
+
+static inline int cipher_descriptor_len(int nbytes, unsigned int block_size)
+{
+ int this_len = nbytes;
+
+ this_len -= (nbytes & (block_size - 1));
+ return this_len > (1 << 16) ? (1 << 16) : this_len;
+}
+
+static int __n2_crypt_chunk(struct crypto_tfm *tfm, struct n2_crypto_chunk *cp,
+ struct spu_queue *qp, bool encrypt)
+{
+ struct n2_cipher_context *ctx = crypto_tfm_ctx(tfm);
+ struct cwq_initial_entry *ent;
+ bool in_place;
+ int i;
+
+ ent = spu_queue_alloc(qp, cp->arr_len);
+ if (!ent) {
+ pr_info("queue_alloc() of %d fails\n",
+ cp->arr_len);
+ return -EBUSY;
+ }
+
+ in_place = (cp->dest_paddr == cp->arr[0].src_paddr);
+
+ ent->control = control_word_base(cp->arr[0].src_len,
+ 0, ctx->enc_type, 0, 0,
+ false, true, false, encrypt,
+ OPCODE_ENCRYPT |
+ (in_place ? OPCODE_INPLACE_BIT : 0));
+ ent->src_addr = cp->arr[0].src_paddr;
+ ent->auth_key_addr = 0UL;
+ ent->auth_iv_addr = 0UL;
+ ent->final_auth_state_addr = 0UL;
+ ent->enc_key_addr = __pa(&ctx->key);
+ ent->enc_iv_addr = cp->iv_paddr;
+ ent->dest_addr = (in_place ? 0UL : cp->dest_paddr);
+
+ for (i = 1; i < cp->arr_len; i++) {
+ ent = spu_queue_next(qp, ent);
+
+ ent->control = cp->arr[i].src_len - 1;
+ ent->src_addr = cp->arr[i].src_paddr;
+ ent->auth_key_addr = 0UL;
+ ent->auth_iv_addr = 0UL;
+ ent->final_auth_state_addr = 0UL;
+ ent->enc_key_addr = 0UL;
+ ent->enc_iv_addr = 0UL;
+ ent->dest_addr = 0UL;
+ }
+ ent->control |= CONTROL_END_OF_BLOCK;
+
+ return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0;
+}
+
+static int n2_compute_chunks(struct ablkcipher_request *req)
+{
+ struct n2_request_context *rctx = ablkcipher_request_ctx(req);
+ struct ablkcipher_walk *walk = &rctx->walk;
+ struct n2_crypto_chunk *chunk;
+ unsigned long dest_prev;
+ unsigned int tot_len;
+ bool prev_in_place;
+ int err, nbytes;
+
+ ablkcipher_walk_init(walk, req->dst, req->src, req->nbytes);
+ err = ablkcipher_walk_phys(req, walk);
+ if (err)
+ return err;
+
+ INIT_LIST_HEAD(&rctx->chunk_list);
+
+ chunk = &rctx->chunk;
+ INIT_LIST_HEAD(&chunk->entry);
+
+ chunk->iv_paddr = 0UL;
+ chunk->arr_len = 0;
+ chunk->dest_paddr = 0UL;
+
+ prev_in_place = false;
+ dest_prev = ~0UL;
+ tot_len = 0;
+
+ while ((nbytes = walk->nbytes) != 0) {
+ unsigned long dest_paddr, src_paddr;
+ bool in_place;
+ int this_len;
+
+ src_paddr = (page_to_phys(walk->src.page) +
+ walk->src.offset);
+ dest_paddr = (page_to_phys(walk->dst.page) +
+ walk->dst.offset);
+ in_place = (src_paddr == dest_paddr);
+ this_len = cipher_descriptor_len(nbytes, walk->blocksize);
+
+ if (chunk->arr_len != 0) {
+ if (in_place != prev_in_place ||
+ (!prev_in_place &&
+ dest_paddr != dest_prev) ||
+ chunk->arr_len == N2_CHUNK_ARR_LEN ||
+ tot_len + this_len > (1 << 16)) {
+ chunk->dest_final = dest_prev;
+ list_add_tail(&chunk->entry,
+ &rctx->chunk_list);
+ chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC);
+ if (!chunk) {
+ err = -ENOMEM;
+ break;
+ }
+ INIT_LIST_HEAD(&chunk->entry);
+ }
+ }
+ if (chunk->arr_len == 0) {
+ chunk->dest_paddr = dest_paddr;
+ tot_len = 0;
+ }
+ chunk->arr[chunk->arr_len].src_paddr = src_paddr;
+ chunk->arr[chunk->arr_len].src_len = this_len;
+ chunk->arr_len++;
+
+ dest_prev = dest_paddr + this_len;
+ prev_in_place = in_place;
+ tot_len += this_len;
+
+ err = ablkcipher_walk_done(req, walk, nbytes - this_len);
+ if (err)
+ break;
+ }
+ if (!err && chunk->arr_len != 0) {
+ chunk->dest_final = dest_prev;
+ list_add_tail(&chunk->entry, &rctx->chunk_list);
+ }
+
+ return err;
+}
+
+static void n2_chunk_complete(struct ablkcipher_request *req, void *final_iv)
+{
+ struct n2_request_context *rctx = ablkcipher_request_ctx(req);
+ struct n2_crypto_chunk *c, *tmp;
+
+ if (final_iv)
+ memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize);
+
+ ablkcipher_walk_complete(&rctx->walk);
+ list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
+ list_del(&c->entry);
+ if (unlikely(c != &rctx->chunk))
+ kfree(c);
+ }
+
+}
+
+static int n2_do_ecb(struct ablkcipher_request *req, bool encrypt)
+{
+ struct n2_request_context *rctx = ablkcipher_request_ctx(req);
+ struct crypto_tfm *tfm = req->base.tfm;
+ int err = n2_compute_chunks(req);
+ struct n2_crypto_chunk *c, *tmp;
+ unsigned long flags, hv_ret;
+ struct spu_queue *qp;
+
+ if (err)
+ return err;
+
+ qp = cpu_to_cwq[get_cpu()];
+ err = -ENODEV;
+ if (!qp)
+ goto out;
+
+ spin_lock_irqsave(&qp->lock, flags);
+
+ list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) {
+ err = __n2_crypt_chunk(tfm, c, qp, encrypt);
+ if (err)
+ break;
+ list_del(&c->entry);
+ if (unlikely(c != &rctx->chunk))
+ kfree(c);
+ }
+ if (!err) {
+ hv_ret = wait_for_tail(qp);
+ if (hv_ret != HV_EOK)
+ err = -EINVAL;
+ }
+
+ spin_unlock_irqrestore(&qp->lock, flags);
+
+out:
+ put_cpu();
+
+ n2_chunk_complete(req, NULL);
+ return err;
+}
+
+static int n2_encrypt_ecb(struct ablkcipher_request *req)
+{
+ return n2_do_ecb(req, true);
+}
+
+static int n2_decrypt_ecb(struct ablkcipher_request *req)
+{
+ return n2_do_ecb(req, false);
+}
+
+static int n2_do_chaining(struct ablkcipher_request *req, bool encrypt)
+{
+ struct n2_request_context *rctx = ablkcipher_request_ctx(req);
+ struct crypto_tfm *tfm = req->base.tfm;
+ unsigned long flags, hv_ret, iv_paddr;
+ int err = n2_compute_chunks(req);
+ struct n2_crypto_chunk *c, *tmp;
+ struct spu_queue *qp;
+ void *final_iv_addr;
+
+ final_iv_addr = NULL;
+
+ if (err)
+ return err;
+
+ qp = cpu_to_cwq[get_cpu()];
+ err = -ENODEV;
+ if (!qp)
+ goto out;
+
+ spin_lock_irqsave(&qp->lock, flags);
+
+ if (encrypt) {
+ iv_paddr = __pa(rctx->walk.iv);
+ list_for_each_entry_safe(c, tmp, &rctx->chunk_list,
+ entry) {
+ c->iv_paddr = iv_paddr;
+ err = __n2_crypt_chunk(tfm, c, qp, true);
+ if (err)
+ break;
+ iv_paddr = c->dest_final - rctx->walk.blocksize;
+ list_del(&c->entry);
+ if (unlikely(c != &rctx->chunk))
+ kfree(c);
+ }
+ final_iv_addr = __va(iv_paddr);
+ } else {
+ list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list,
+ entry) {
+ if (c == &rctx->chunk) {
+ iv_paddr = __pa(rctx->walk.iv);
+ } else {
+ iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr +
+ tmp->arr[tmp->arr_len-1].src_len -
+ rctx->walk.blocksize);
+ }
+ if (!final_iv_addr) {
+ unsigned long pa;
+
+ pa = (c->arr[c->arr_len-1].src_paddr +
+ c->arr[c->arr_len-1].src_len -
+ rctx->walk.blocksize);
+ final_iv_addr = rctx->temp_iv;
+ memcpy(rctx->temp_iv, __va(pa),
+ rctx->walk.blocksize);
+ }
+ c->iv_paddr = iv_paddr;
+ err = __n2_crypt_chunk(tfm, c, qp, false);
+ if (err)
+ break;
+ list_del(&c->entry);
+ if (unlikely(c != &rctx->chunk))
+ kfree(c);
+ }
+ }
+ if (!err) {
+ hv_ret = wait_for_tail(qp);
+ if (hv_ret != HV_EOK)
+ err = -EINVAL;
+ }
+
+ spin_unlock_irqrestore(&qp->lock, flags);
+
+out:
+ put_cpu();
+
+ n2_chunk_complete(req, err ? NULL : final_iv_addr);
+ return err;
+}
+
+static int n2_encrypt_chaining(struct ablkcipher_request *req)
+{
+ return n2_do_chaining(req, true);
+}
+
+static int n2_decrypt_chaining(struct ablkcipher_request *req)
+{
+ return n2_do_chaining(req, false);
+}
+
+struct n2_cipher_tmpl {
+ const char *name;
+ const char *drv_name;
+ u8 block_size;
+ u8 enc_type;
+ struct ablkcipher_alg ablkcipher;
+};
+
+static const struct n2_cipher_tmpl cipher_tmpls[] = {
+ /* ARC4: only ECB is supported (chaining bits ignored) */
+ { .name = "ecb(arc4)",
+ .drv_name = "ecb-arc4",
+ .block_size = 1,
+ .enc_type = (ENC_TYPE_ALG_RC4_STREAM |
+ ENC_TYPE_CHAINING_ECB),
+ .ablkcipher = {
+ .min_keysize = 1,
+ .max_keysize = 256,
+ .setkey = n2_arc4_setkey,
+ .encrypt = n2_encrypt_ecb,
+ .decrypt = n2_decrypt_ecb,
+ },
+ },
+
+ /* DES: ECB CBC and CFB are supported */
+ { .name = "ecb(des)",
+ .drv_name = "ecb-des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_DES |
+ ENC_TYPE_CHAINING_ECB),
+ .ablkcipher = {
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .setkey = n2_des_setkey,
+ .encrypt = n2_encrypt_ecb,
+ .decrypt = n2_decrypt_ecb,
+ },
+ },
+ { .name = "cbc(des)",
+ .drv_name = "cbc-des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_DES |
+ ENC_TYPE_CHAINING_CBC),
+ .ablkcipher = {
+ .ivsize = DES_BLOCK_SIZE,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .setkey = n2_des_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_decrypt_chaining,
+ },
+ },
+ { .name = "cfb(des)",
+ .drv_name = "cfb-des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_DES |
+ ENC_TYPE_CHAINING_CFB),
+ .ablkcipher = {
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .setkey = n2_des_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_decrypt_chaining,
+ },
+ },
+
+ /* 3DES: ECB CBC and CFB are supported */
+ { .name = "ecb(des3_ede)",
+ .drv_name = "ecb-3des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_3DES |
+ ENC_TYPE_CHAINING_ECB),
+ .ablkcipher = {
+ .min_keysize = 3 * DES_KEY_SIZE,
+ .max_keysize = 3 * DES_KEY_SIZE,
+ .setkey = n2_3des_setkey,
+ .encrypt = n2_encrypt_ecb,
+ .decrypt = n2_decrypt_ecb,
+ },
+ },
+ { .name = "cbc(des3_ede)",
+ .drv_name = "cbc-3des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_3DES |
+ ENC_TYPE_CHAINING_CBC),
+ .ablkcipher = {
+ .ivsize = DES_BLOCK_SIZE,
+ .min_keysize = 3 * DES_KEY_SIZE,
+ .max_keysize = 3 * DES_KEY_SIZE,
+ .setkey = n2_3des_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_decrypt_chaining,
+ },
+ },
+ { .name = "cfb(des3_ede)",
+ .drv_name = "cfb-3des",
+ .block_size = DES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_3DES |
+ ENC_TYPE_CHAINING_CFB),
+ .ablkcipher = {
+ .min_keysize = 3 * DES_KEY_SIZE,
+ .max_keysize = 3 * DES_KEY_SIZE,
+ .setkey = n2_3des_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_decrypt_chaining,
+ },
+ },
+ /* AES: ECB CBC and CTR are supported */
+ { .name = "ecb(aes)",
+ .drv_name = "ecb-aes",
+ .block_size = AES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_AES128 |
+ ENC_TYPE_CHAINING_ECB),
+ .ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = n2_aes_setkey,
+ .encrypt = n2_encrypt_ecb,
+ .decrypt = n2_decrypt_ecb,
+ },
+ },
+ { .name = "cbc(aes)",
+ .drv_name = "cbc-aes",
+ .block_size = AES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_AES128 |
+ ENC_TYPE_CHAINING_CBC),
+ .ablkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = n2_aes_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_decrypt_chaining,
+ },
+ },
+ { .name = "ctr(aes)",
+ .drv_name = "ctr-aes",
+ .block_size = AES_BLOCK_SIZE,
+ .enc_type = (ENC_TYPE_ALG_AES128 |
+ ENC_TYPE_CHAINING_COUNTER),
+ .ablkcipher = {
+ .ivsize = AES_BLOCK_SIZE,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = n2_aes_setkey,
+ .encrypt = n2_encrypt_chaining,
+ .decrypt = n2_encrypt_chaining,
+ },
+ },
+
+};
+#define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
+
+static LIST_HEAD(cipher_algs);
+
+struct n2_hash_tmpl {
+ const char *name;
+ const char *hash_zero;
+ const u32 *hash_init;
+ u8 hw_op_hashsz;
+ u8 digest_size;
+ u8 block_size;
+ u8 auth_type;
+ u8 hmac_type;
+};
+
+static const char md5_zero[MD5_DIGEST_SIZE] = {
+ 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
+ 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
+};
+static const u32 md5_init[MD5_HASH_WORDS] = {
+ cpu_to_le32(0x67452301),
+ cpu_to_le32(0xefcdab89),
+ cpu_to_le32(0x98badcfe),
+ cpu_to_le32(0x10325476),
+};
+static const char sha1_zero[SHA1_DIGEST_SIZE] = {
+ 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
+ 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
+ 0x07, 0x09
+};
+static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = {
+ SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4,
+};
+static const char sha256_zero[SHA256_DIGEST_SIZE] = {
+ 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
+ 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
+ 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
+ 0x1b, 0x78, 0x52, 0xb8, 0x55
+};
+static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = {
+ SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3,
+ SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7,
+};
+static const char sha224_zero[SHA224_DIGEST_SIZE] = {
+ 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
+ 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
+ 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
+ 0x2f
+};
+static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = {
+ SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3,
+ SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7,
+};
+
+static const struct n2_hash_tmpl hash_tmpls[] = {
+ { .name = "md5",
+ .hash_zero = md5_zero,
+ .hash_init = md5_init,
+ .auth_type = AUTH_TYPE_MD5,
+ .hmac_type = AUTH_TYPE_HMAC_MD5,
+ .hw_op_hashsz = MD5_DIGEST_SIZE,
+ .digest_size = MD5_DIGEST_SIZE,
+ .block_size = MD5_HMAC_BLOCK_SIZE },
+ { .name = "sha1",
+ .hash_zero = sha1_zero,
+ .hash_init = sha1_init,
+ .auth_type = AUTH_TYPE_SHA1,
+ .hmac_type = AUTH_TYPE_HMAC_SHA1,
+ .hw_op_hashsz = SHA1_DIGEST_SIZE,
+ .digest_size = SHA1_DIGEST_SIZE,
+ .block_size = SHA1_BLOCK_SIZE },
+ { .name = "sha256",
+ .hash_zero = sha256_zero,
+ .hash_init = sha256_init,
+ .auth_type = AUTH_TYPE_SHA256,
+ .hmac_type = AUTH_TYPE_HMAC_SHA256,
+ .hw_op_hashsz = SHA256_DIGEST_SIZE,
+ .digest_size = SHA256_DIGEST_SIZE,
+ .block_size = SHA256_BLOCK_SIZE },
+ { .name = "sha224",
+ .hash_zero = sha224_zero,
+ .hash_init = sha224_init,
+ .auth_type = AUTH_TYPE_SHA256,
+ .hmac_type = AUTH_TYPE_RESERVED,
+ .hw_op_hashsz = SHA256_DIGEST_SIZE,
+ .digest_size = SHA224_DIGEST_SIZE,
+ .block_size = SHA224_BLOCK_SIZE },
+};
+#define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
+
+static LIST_HEAD(ahash_algs);
+static LIST_HEAD(hmac_algs);
+
+static int algs_registered;
+
+static void __n2_unregister_algs(void)
+{
+ struct n2_cipher_alg *cipher, *cipher_tmp;
+ struct n2_ahash_alg *alg, *alg_tmp;
+ struct n2_hmac_alg *hmac, *hmac_tmp;
+
+ list_for_each_entry_safe(cipher, cipher_tmp, &cipher_algs, entry) {
+ crypto_unregister_alg(&cipher->alg);
+ list_del(&cipher->entry);
+ kfree(cipher);
+ }
+ list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) {
+ crypto_unregister_ahash(&hmac->derived.alg);
+ list_del(&hmac->derived.entry);
+ kfree(hmac);
+ }
+ list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) {
+ crypto_unregister_ahash(&alg->alg);
+ list_del(&alg->entry);
+ kfree(alg);
+ }
+}
+
+static int n2_cipher_cra_init(struct crypto_tfm *tfm)
+{
+ tfm->crt_ablkcipher.reqsize = sizeof(struct n2_request_context);
+ return 0;
+}
+
+static int __devinit __n2_register_one_cipher(const struct n2_cipher_tmpl *tmpl)
+{
+ struct n2_cipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
+ struct crypto_alg *alg;
+ int err;
+
+ if (!p)
+ return -ENOMEM;
+
+ alg = &p->alg;
+
+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name);
+ alg->cra_priority = N2_CRA_PRIORITY;
+ alg->cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
+ alg->cra_blocksize = tmpl->block_size;
+ p->enc_type = tmpl->enc_type;
+ alg->cra_ctxsize = sizeof(struct n2_cipher_context);
+ alg->cra_type = &crypto_ablkcipher_type;
+ alg->cra_u.ablkcipher = tmpl->ablkcipher;
+ alg->cra_init = n2_cipher_cra_init;
+ alg->cra_module = THIS_MODULE;
+
+ list_add(&p->entry, &cipher_algs);
+ err = crypto_register_alg(alg);
+ if (err) {
+ pr_err("%s alg registration failed\n", alg->cra_name);
+ list_del(&p->entry);
+ kfree(p);
+ } else {
+ pr_info("%s alg registered\n", alg->cra_name);
+ }
+ return err;
+}
+
+static int __devinit __n2_register_one_hmac(struct n2_ahash_alg *n2ahash)
+{
+ struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
+ struct ahash_alg *ahash;
+ struct crypto_alg *base;
+ int err;
+
+ if (!p)
+ return -ENOMEM;
+
+ p->child_alg = n2ahash->alg.halg.base.cra_name;
+ memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg));
+ INIT_LIST_HEAD(&p->derived.entry);
+
+ ahash = &p->derived.alg;
+ ahash->digest = n2_hmac_async_digest;
+ ahash->setkey = n2_hmac_async_setkey;
+
+ base = &ahash->halg.base;
+ snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", p->child_alg);
+ snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", p->child_alg);
+
+ base->cra_ctxsize = sizeof(struct n2_hmac_ctx);
+ base->cra_init = n2_hmac_cra_init;
+ base->cra_exit = n2_hmac_cra_exit;
+
+ list_add(&p->derived.entry, &hmac_algs);
+ err = crypto_register_ahash(ahash);
+ if (err) {
+ pr_err("%s alg registration failed\n", base->cra_name);
+ list_del(&p->derived.entry);
+ kfree(p);
+ } else {
+ pr_info("%s alg registered\n", base->cra_name);
+ }
+ return err;
+}
+
+static int __devinit __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl)
+{
+ struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL);
+ struct hash_alg_common *halg;
+ struct crypto_alg *base;
+ struct ahash_alg *ahash;
+ int err;
+
+ if (!p)
+ return -ENOMEM;
+
+ p->hash_zero = tmpl->hash_zero;
+ p->hash_init = tmpl->hash_init;
+ p->auth_type = tmpl->auth_type;
+ p->hmac_type = tmpl->hmac_type;
+ p->hw_op_hashsz = tmpl->hw_op_hashsz;
+ p->digest_size = tmpl->digest_size;
+
+ ahash = &p->alg;
+ ahash->init = n2_hash_async_init;
+ ahash->update = n2_hash_async_update;
+ ahash->final = n2_hash_async_final;
+ ahash->finup = n2_hash_async_finup;
+ ahash->digest = n2_hash_async_digest;
+
+ halg = &ahash->halg;
+ halg->digestsize = tmpl->digest_size;
+
+ base = &halg->base;
+ snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
+ snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name);
+ base->cra_priority = N2_CRA_PRIORITY;
+ base->cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK;
+ base->cra_blocksize = tmpl->block_size;
+ base->cra_ctxsize = sizeof(struct n2_hash_ctx);
+ base->cra_module = THIS_MODULE;
+ base->cra_init = n2_hash_cra_init;
+ base->cra_exit = n2_hash_cra_exit;
+
+ list_add(&p->entry, &ahash_algs);
+ err = crypto_register_ahash(ahash);
+ if (err) {
+ pr_err("%s alg registration failed\n", base->cra_name);
+ list_del(&p->entry);
+ kfree(p);
+ } else {
+ pr_info("%s alg registered\n", base->cra_name);
+ }
+ if (!err && p->hmac_type != AUTH_TYPE_RESERVED)
+ err = __n2_register_one_hmac(p);
+ return err;
+}
+
+static int __devinit n2_register_algs(void)
+{
+ int i, err = 0;
+
+ mutex_lock(&spu_lock);
+ if (algs_registered++)
+ goto out;
+
+ for (i = 0; i < NUM_HASH_TMPLS; i++) {
+ err = __n2_register_one_ahash(&hash_tmpls[i]);
+ if (err) {
+ __n2_unregister_algs();
+ goto out;
+ }
+ }
+ for (i = 0; i < NUM_CIPHER_TMPLS; i++) {
+ err = __n2_register_one_cipher(&cipher_tmpls[i]);
+ if (err) {
+ __n2_unregister_algs();
+ goto out;
+ }
+ }
+
+out:
+ mutex_unlock(&spu_lock);
+ return err;
+}
+
+static void __devexit n2_unregister_algs(void)
+{
+ mutex_lock(&spu_lock);
+ if (!--algs_registered)
+ __n2_unregister_algs();
+ mutex_unlock(&spu_lock);
+}
+
+/* To map CWQ queues to interrupt sources, the hypervisor API provides
+ * a devino. This isn't very useful to us because all of the
+ * interrupts listed in the device_node have been translated to
+ * Linux virtual IRQ cookie numbers.
+ *
+ * So we have to back-translate, going through the 'intr' and 'ino'
+ * property tables of the n2cp MDESC node, matching it with the OF
+ * 'interrupts' property entries, in order to to figure out which
+ * devino goes to which already-translated IRQ.
+ */
+static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip,
+ unsigned long dev_ino)
+{
+ const unsigned int *dev_intrs;
+ unsigned int intr;
+ int i;
+
+ for (i = 0; i < ip->num_intrs; i++) {
+ if (ip->ino_table[i].ino == dev_ino)
+ break;
+ }
+ if (i == ip->num_intrs)
+ return -ENODEV;
+
+ intr = ip->ino_table[i].intr;
+
+ dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL);
+ if (!dev_intrs)
+ return -ENODEV;
+
+ for (i = 0; i < dev->archdata.num_irqs; i++) {
+ if (dev_intrs[i] == intr)
+ return i;
+ }
+
+ return -ENODEV;
+}
+
+static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip,
+ const char *irq_name, struct spu_queue *p,
+ irq_handler_t handler)
+{
+ unsigned long herr;
+ int index;
+
+ herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino);
+ if (herr)
+ return -EINVAL;
+
+ index = find_devino_index(dev, ip, p->devino);
+ if (index < 0)
+ return index;
+
+ p->irq = dev->archdata.irqs[index];
+
+ sprintf(p->irq_name, "%s-%d", irq_name, index);
+
+ return request_irq(p->irq, handler, IRQF_SAMPLE_RANDOM,
+ p->irq_name, p);
+}
+
+static struct kmem_cache *queue_cache[2];
+
+static void *new_queue(unsigned long q_type)
+{
+ return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL);
+}
+
+static void free_queue(void *p, unsigned long q_type)
+{
+ return kmem_cache_free(queue_cache[q_type - 1], p);
+}
+
+static int queue_cache_init(void)
+{
+ if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
+ queue_cache[HV_NCS_QTYPE_MAU - 1] =
+ kmem_cache_create("mau_queue",
+ (MAU_NUM_ENTRIES *
+ MAU_ENTRY_SIZE),
+ MAU_ENTRY_SIZE, 0, NULL);
+ if (!queue_cache[HV_NCS_QTYPE_MAU - 1])
+ return -ENOMEM;
+
+ if (!queue_cache[HV_NCS_QTYPE_CWQ - 1])
+ queue_cache[HV_NCS_QTYPE_CWQ - 1] =
+ kmem_cache_create("cwq_queue",
+ (CWQ_NUM_ENTRIES *
+ CWQ_ENTRY_SIZE),
+ CWQ_ENTRY_SIZE, 0, NULL);
+ if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) {
+ kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
+ return -ENOMEM;
+ }
+ return 0;
+}
+
+static void queue_cache_destroy(void)
+{
+ kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]);
+ kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]);
+}
+
+static int spu_queue_register(struct spu_queue *p, unsigned long q_type)
+{
+ cpumask_var_t old_allowed;
+ unsigned long hv_ret;
+
+ if (cpumask_empty(&p->sharing))
+ return -EINVAL;
+
+ if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
+ return -ENOMEM;
+
+ cpumask_copy(old_allowed, ¤t->cpus_allowed);
+
+ set_cpus_allowed_ptr(current, &p->sharing);
+
+ hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q),
+ CWQ_NUM_ENTRIES, &p->qhandle);
+ if (!hv_ret)
+ sun4v_ncs_sethead_marker(p->qhandle, 0);
+
+ set_cpus_allowed_ptr(current, old_allowed);
+
+ free_cpumask_var(old_allowed);
+
+ return (hv_ret ? -EINVAL : 0);
+}
+
+static int spu_queue_setup(struct spu_queue *p)
+{
+ int err;
+
+ p->q = new_queue(p->q_type);
+ if (!p->q)
+ return -ENOMEM;
+
+ err = spu_queue_register(p, p->q_type);
+ if (err) {
+ free_queue(p->q, p->q_type);
+ p->q = NULL;
+ }
+
+ return err;
+}
+
+static void spu_queue_destroy(struct spu_queue *p)
+{
+ unsigned long hv_ret;
+
+ if (!p->q)
+ return;
+
+ hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle);
+
+ if (!hv_ret)
+ free_queue(p->q, p->q_type);
+}
+
+static void spu_list_destroy(struct list_head *list)
+{
+ struct spu_queue *p, *n;
+
+ list_for_each_entry_safe(p, n, list, list) {
+ int i;
+
+ for (i = 0; i < NR_CPUS; i++) {
+ if (cpu_to_cwq[i] == p)
+ cpu_to_cwq[i] = NULL;
+ }
+
+ if (p->irq) {
+ free_irq(p->irq, p);
+ p->irq = 0;
+ }
+ spu_queue_destroy(p);
+ list_del(&p->list);
+ kfree(p);
+ }
+}
+
+/* Walk the backward arcs of a CWQ 'exec-unit' node,
+ * gathering cpu membership information.
+ */
+static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc,
+ struct platform_device *dev,
+ u64 node, struct spu_queue *p,
+ struct spu_queue **table)
+{
+ u64 arc;
+
+ mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) {
+ u64 tgt = mdesc_arc_target(mdesc, arc);
+ const char *name = mdesc_node_name(mdesc, tgt);
+ const u64 *id;
+
+ if (strcmp(name, "cpu"))
+ continue;
+ id = mdesc_get_property(mdesc, tgt, "id", NULL);
+ if (table[*id] != NULL) {
+ dev_err(&dev->dev, "%s: SPU cpu slot already set.\n",
+ dev->dev.of_node->full_name);
+ return -EINVAL;
+ }
+ cpu_set(*id, p->sharing);
+ table[*id] = p;
+ }
+ return 0;
+}
+
+/* Process an 'exec-unit' MDESC node of type 'cwq'. */
+static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list,
+ struct platform_device *dev, struct mdesc_handle *mdesc,
+ u64 node, const char *iname, unsigned long q_type,
+ irq_handler_t handler, struct spu_queue **table)
+{
+ struct spu_queue *p;
+ int err;
+
+ p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL);
+ if (!p) {
+ dev_err(&dev->dev, "%s: Could not allocate SPU queue.\n",
+ dev->dev.of_node->full_name);
+ return -ENOMEM;
+ }
+
+ cpus_clear(p->sharing);
+ spin_lock_init(&p->lock);
+ p->q_type = q_type;
+ INIT_LIST_HEAD(&p->jobs);
+ list_add(&p->list, list);
+
+ err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table);
+ if (err)
+ return err;
+
+ err = spu_queue_setup(p);
+ if (err)
+ return err;
+
+ return spu_map_ino(dev, ip, iname, p, handler);
+}
+
+static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev,
+ struct spu_mdesc_info *ip, struct list_head *list,
+ const char *exec_name, unsigned long q_type,
+ irq_handler_t handler, struct spu_queue **table)
+{
+ int err = 0;
+ u64 node;
+
+ mdesc_for_each_node_by_name(mdesc, node, "exec-unit") {
+ const char *type;
+
+ type = mdesc_get_property(mdesc, node, "type", NULL);
+ if (!type || strcmp(type, exec_name))
+ continue;
+
+ err = handle_exec_unit(ip, list, dev, mdesc, node,
+ exec_name, q_type, handler, table);
+ if (err) {
+ spu_list_destroy(list);
+ break;
+ }
+ }
+
+ return err;
+}
+
+static int __devinit get_irq_props(struct mdesc_handle *mdesc, u64 node,
+ struct spu_mdesc_info *ip)
+{
+ const u64 *ino;
+ int ino_len;
+ int i;
+
+ ino = mdesc_get_property(mdesc, node, "ino", &ino_len);
+ if (!ino) {
+ printk("NO 'ino'\n");
+ return -ENODEV;
+ }
+
+ ip->num_intrs = ino_len / sizeof(u64);
+ ip->ino_table = kzalloc((sizeof(struct ino_blob) *
+ ip->num_intrs),
+ GFP_KERNEL);
+ if (!ip->ino_table)
+ return -ENOMEM;
+
+ for (i = 0; i < ip->num_intrs; i++) {
+ struct ino_blob *b = &ip->ino_table[i];
+ b->intr = i + 1;
+ b->ino = ino[i];
+ }
+
+ return 0;
+}
+
+static int __devinit grab_mdesc_irq_props(struct mdesc_handle *mdesc,
+ struct platform_device *dev,
+ struct spu_mdesc_info *ip,
+ const char *node_name)
+{
+ const unsigned int *reg;
+ u64 node;
+
+ reg = of_get_property(dev->dev.of_node, "reg", NULL);
+ if (!reg)
+ return -ENODEV;
+
+ mdesc_for_each_node_by_name(mdesc, node, "virtual-device") {
+ const char *name;
+ const u64 *chdl;
+
+ name = mdesc_get_property(mdesc, node, "name", NULL);
+ if (!name || strcmp(name, node_name))
+ continue;
+ chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL);
+ if (!chdl || (*chdl != *reg))
+ continue;
+ ip->cfg_handle = *chdl;
+ return get_irq_props(mdesc, node, ip);
+ }
+
+ return -ENODEV;
+}
+
+static unsigned long n2_spu_hvapi_major;
+static unsigned long n2_spu_hvapi_minor;
+
+static int __devinit n2_spu_hvapi_register(void)
+{
+ int err;
+
+ n2_spu_hvapi_major = 2;
+ n2_spu_hvapi_minor = 0;
+
+ err = sun4v_hvapi_register(HV_GRP_NCS,
+ n2_spu_hvapi_major,
+ &n2_spu_hvapi_minor);
+
+ if (!err)
+ pr_info("Registered NCS HVAPI version %lu.%lu\n",
+ n2_spu_hvapi_major,
+ n2_spu_hvapi_minor);
+
+ return err;
+}
+
+static void n2_spu_hvapi_unregister(void)
+{
+ sun4v_hvapi_unregister(HV_GRP_NCS);
+}
+
+static int global_ref;
+
+static int __devinit grab_global_resources(void)
+{
+ int err = 0;
+
+ mutex_lock(&spu_lock);
+
+ if (global_ref++)
+ goto out;
+
+ err = n2_spu_hvapi_register();
+ if (err)
+ goto out;
+
+ err = queue_cache_init();
+ if (err)
+ goto out_hvapi_release;
+
+ err = -ENOMEM;
+ cpu_to_cwq = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
+ GFP_KERNEL);
+ if (!cpu_to_cwq)
+ goto out_queue_cache_destroy;
+
+ cpu_to_mau = kzalloc(sizeof(struct spu_queue *) * NR_CPUS,
+ GFP_KERNEL);
+ if (!cpu_to_mau)
+ goto out_free_cwq_table;
+
+ err = 0;
+
+out:
+ if (err)
+ global_ref--;
+ mutex_unlock(&spu_lock);
+ return err;
+
+out_free_cwq_table:
+ kfree(cpu_to_cwq);
+ cpu_to_cwq = NULL;
+
+out_queue_cache_destroy:
+ queue_cache_destroy();
+
+out_hvapi_release:
+ n2_spu_hvapi_unregister();
+ goto out;
+}
+
+static void release_global_resources(void)
+{
+ mutex_lock(&spu_lock);
+ if (!--global_ref) {
+ kfree(cpu_to_cwq);
+ cpu_to_cwq = NULL;
+
+ kfree(cpu_to_mau);
+ cpu_to_mau = NULL;
+
+ queue_cache_destroy();
+ n2_spu_hvapi_unregister();
+ }
+ mutex_unlock(&spu_lock);
+}
+
+static struct n2_crypto * __devinit alloc_n2cp(void)
+{
+ struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL);
+
+ if (np)
+ INIT_LIST_HEAD(&np->cwq_list);
+
+ return np;
+}
+
+static void free_n2cp(struct n2_crypto *np)
+{
+ if (np->cwq_info.ino_table) {
+ kfree(np->cwq_info.ino_table);
+ np->cwq_info.ino_table = NULL;
+ }
+
+ kfree(np);
+}
+
+static void __devinit n2_spu_driver_version(void)
+{
+ static int n2_spu_version_printed;
+
+ if (n2_spu_version_printed++ == 0)
+ pr_info("%s", version);
+}
+
+static int __devinit n2_crypto_probe(struct platform_device *dev)
+{
+ struct mdesc_handle *mdesc;
+ const char *full_name;
+ struct n2_crypto *np;
+ int err;
+
+ n2_spu_driver_version();
+
+ full_name = dev->dev.of_node->full_name;
+ pr_info("Found N2CP at %s\n", full_name);
+
+ np = alloc_n2cp();
+ if (!np) {
+ dev_err(&dev->dev, "%s: Unable to allocate n2cp.\n",
+ full_name);
+ return -ENOMEM;
+ }
+
+ err = grab_global_resources();
+ if (err) {
+ dev_err(&dev->dev, "%s: Unable to grab "
+ "global resources.\n", full_name);
+ goto out_free_n2cp;
+ }
+
+ mdesc = mdesc_grab();
+
+ if (!mdesc) {
+ dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
+ full_name);
+ err = -ENODEV;
+ goto out_free_global;
+ }
+ err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp");
+ if (err) {
+ dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
+ full_name);
+ mdesc_release(mdesc);
+ goto out_free_global;
+ }
+
+ err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list,
+ "cwq", HV_NCS_QTYPE_CWQ, cwq_intr,
+ cpu_to_cwq);
+ mdesc_release(mdesc);
+
+ if (err) {
+ dev_err(&dev->dev, "%s: CWQ MDESC scan failed.\n",
+ full_name);
+ goto out_free_global;
+ }
+
+ err = n2_register_algs();
+ if (err) {
+ dev_err(&dev->dev, "%s: Unable to register algorithms.\n",
+ full_name);
+ goto out_free_spu_list;
+ }
+
+ dev_set_drvdata(&dev->dev, np);
+
+ return 0;
+
+out_free_spu_list:
+ spu_list_destroy(&np->cwq_list);
+
+out_free_global:
+ release_global_resources();
+
+out_free_n2cp:
+ free_n2cp(np);
+
+ return err;
+}
+
+static int __devexit n2_crypto_remove(struct platform_device *dev)
+{
+ struct n2_crypto *np = dev_get_drvdata(&dev->dev);
+
+ n2_unregister_algs();
+
+ spu_list_destroy(&np->cwq_list);
+
+ release_global_resources();
+
+ free_n2cp(np);
+
+ return 0;
+}
+
+static struct n2_mau * __devinit alloc_ncp(void)
+{
+ struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL);
+
+ if (mp)
+ INIT_LIST_HEAD(&mp->mau_list);
+
+ return mp;
+}
+
+static void free_ncp(struct n2_mau *mp)
+{
+ if (mp->mau_info.ino_table) {
+ kfree(mp->mau_info.ino_table);
+ mp->mau_info.ino_table = NULL;
+ }
+
+ kfree(mp);
+}
+
+static int __devinit n2_mau_probe(struct platform_device *dev)
+{
+ struct mdesc_handle *mdesc;
+ const char *full_name;
+ struct n2_mau *mp;
+ int err;
+
+ n2_spu_driver_version();
+
+ full_name = dev->dev.of_node->full_name;
+ pr_info("Found NCP at %s\n", full_name);
+
+ mp = alloc_ncp();
+ if (!mp) {
+ dev_err(&dev->dev, "%s: Unable to allocate ncp.\n",
+ full_name);
+ return -ENOMEM;
+ }
+
+ err = grab_global_resources();
+ if (err) {
+ dev_err(&dev->dev, "%s: Unable to grab "
+ "global resources.\n", full_name);
+ goto out_free_ncp;
+ }
+
+ mdesc = mdesc_grab();
+
+ if (!mdesc) {
+ dev_err(&dev->dev, "%s: Unable to grab MDESC.\n",
+ full_name);
+ err = -ENODEV;
+ goto out_free_global;
+ }
+
+ err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp");
+ if (err) {
+ dev_err(&dev->dev, "%s: Unable to grab IRQ props.\n",
+ full_name);
+ mdesc_release(mdesc);
+ goto out_free_global;
+ }
+
+ err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list,
+ "mau", HV_NCS_QTYPE_MAU, mau_intr,
+ cpu_to_mau);
+ mdesc_release(mdesc);
+
+ if (err) {
+ dev_err(&dev->dev, "%s: MAU MDESC scan failed.\n",
+ full_name);
+ goto out_free_global;
+ }
+
+ dev_set_drvdata(&dev->dev, mp);
+
+ return 0;
+
+out_free_global:
+ release_global_resources();
+
+out_free_ncp:
+ free_ncp(mp);
+
+ return err;
+}
+
+static int __devexit n2_mau_remove(struct platform_device *dev)
+{
+ struct n2_mau *mp = dev_get_drvdata(&dev->dev);
+
+ spu_list_destroy(&mp->mau_list);
+
+ release_global_resources();
+
+ free_ncp(mp);
+
+ return 0;
+}
+
+static struct of_device_id n2_crypto_match[] = {
+ {
+ .name = "n2cp",
+ .compatible = "SUNW,n2-cwq",
+ },
+ {
+ .name = "n2cp",
+ .compatible = "SUNW,vf-cwq",
+ },
+ {
+ .name = "n2cp",
+ .compatible = "SUNW,kt-cwq",
+ },
+ {},
+};
+
+MODULE_DEVICE_TABLE(of, n2_crypto_match);
+
+static struct platform_driver n2_crypto_driver = {
+ .driver = {
+ .name = "n2cp",
+ .owner = THIS_MODULE,
+ .of_match_table = n2_crypto_match,
+ },
+ .probe = n2_crypto_probe,
+ .remove = __devexit_p(n2_crypto_remove),
+};
+
+static struct of_device_id n2_mau_match[] = {
+ {
+ .name = "ncp",
+ .compatible = "SUNW,n2-mau",
+ },
+ {
+ .name = "ncp",
+ .compatible = "SUNW,vf-mau",
+ },
+ {
+ .name = "ncp",
+ .compatible = "SUNW,kt-mau",
+ },
+ {},
+};
+
+MODULE_DEVICE_TABLE(of, n2_mau_match);
+
+static struct platform_driver n2_mau_driver = {
+ .driver = {
+ .name = "ncp",
+ .owner = THIS_MODULE,
+ .of_match_table = n2_mau_match,
+ },
+ .probe = n2_mau_probe,
+ .remove = __devexit_p(n2_mau_remove),
+};
+
+static int __init n2_init(void)
+{
+ int err = platform_driver_register(&n2_crypto_driver);
+
+ if (!err) {
+ err = platform_driver_register(&n2_mau_driver);
+ if (err)
+ platform_driver_unregister(&n2_crypto_driver);
+ }
+ return err;
+}
+
+static void __exit n2_exit(void)
+{
+ platform_driver_unregister(&n2_mau_driver);
+ platform_driver_unregister(&n2_crypto_driver);
+}
+
+module_init(n2_init);
+module_exit(n2_exit);
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.h b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.h
new file mode 100644
index 0000000..4bcbbea
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/n2_core.h
@@ -0,0 +1,231 @@
+#ifndef _N2_CORE_H
+#define _N2_CORE_H
+
+#ifndef __ASSEMBLY__
+
+struct ino_blob {
+ u64 intr;
+ u64 ino;
+};
+
+struct spu_mdesc_info {
+ u64 cfg_handle;
+ struct ino_blob *ino_table;
+ int num_intrs;
+};
+
+struct n2_crypto {
+ struct spu_mdesc_info cwq_info;
+ struct list_head cwq_list;
+};
+
+struct n2_mau {
+ struct spu_mdesc_info mau_info;
+ struct list_head mau_list;
+};
+
+#define CWQ_ENTRY_SIZE 64
+#define CWQ_NUM_ENTRIES 64
+
+#define MAU_ENTRY_SIZE 64
+#define MAU_NUM_ENTRIES 64
+
+struct cwq_initial_entry {
+ u64 control;
+ u64 src_addr;
+ u64 auth_key_addr;
+ u64 auth_iv_addr;
+ u64 final_auth_state_addr;
+ u64 enc_key_addr;
+ u64 enc_iv_addr;
+ u64 dest_addr;
+};
+
+struct cwq_ext_entry {
+ u64 len;
+ u64 src_addr;
+ u64 resv1;
+ u64 resv2;
+ u64 resv3;
+ u64 resv4;
+ u64 resv5;
+ u64 resv6;
+};
+
+struct cwq_final_entry {
+ u64 control;
+ u64 src_addr;
+ u64 resv1;
+ u64 resv2;
+ u64 resv3;
+ u64 resv4;
+ u64 resv5;
+ u64 resv6;
+};
+
+#define CONTROL_LEN 0x000000000000ffffULL
+#define CONTROL_LEN_SHIFT 0
+#define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL
+#define CONTROL_HMAC_KEY_LEN_SHIFT 16
+#define CONTROL_ENC_TYPE 0x00000000ff000000ULL
+#define CONTROL_ENC_TYPE_SHIFT 24
+#define ENC_TYPE_ALG_RC4_STREAM 0x00ULL
+#define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL
+#define ENC_TYPE_ALG_DES 0x08ULL
+#define ENC_TYPE_ALG_3DES 0x0cULL
+#define ENC_TYPE_ALG_AES128 0x10ULL
+#define ENC_TYPE_ALG_AES192 0x14ULL
+#define ENC_TYPE_ALG_AES256 0x18ULL
+#define ENC_TYPE_ALG_RESERVED 0x1cULL
+#define ENC_TYPE_ALG_MASK 0x1cULL
+#define ENC_TYPE_CHAINING_ECB 0x00ULL
+#define ENC_TYPE_CHAINING_CBC 0x01ULL
+#define ENC_TYPE_CHAINING_CFB 0x02ULL
+#define ENC_TYPE_CHAINING_COUNTER 0x03ULL
+#define ENC_TYPE_CHAINING_MASK 0x03ULL
+#define CONTROL_AUTH_TYPE 0x0000001f00000000ULL
+#define CONTROL_AUTH_TYPE_SHIFT 32
+#define AUTH_TYPE_RESERVED 0x00ULL
+#define AUTH_TYPE_MD5 0x01ULL
+#define AUTH_TYPE_SHA1 0x02ULL
+#define AUTH_TYPE_SHA256 0x03ULL
+#define AUTH_TYPE_CRC32 0x04ULL
+#define AUTH_TYPE_HMAC_MD5 0x05ULL
+#define AUTH_TYPE_HMAC_SHA1 0x06ULL
+#define AUTH_TYPE_HMAC_SHA256 0x07ULL
+#define AUTH_TYPE_TCP_CHECKSUM 0x08ULL
+#define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL
+#define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL
+#define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL
+#define CONTROL_STRAND 0x000000e000000000ULL
+#define CONTROL_STRAND_SHIFT 37
+#define CONTROL_HASH_LEN 0x0000ff0000000000ULL
+#define CONTROL_HASH_LEN_SHIFT 40
+#define CONTROL_INTERRUPT 0x0001000000000000ULL
+#define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL
+#define CONTROL_RESERVED 0x001c000000000000ULL
+#define CONTROL_HV_DONE 0x0004000000000000ULL
+#define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL
+#define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL
+#define CONTROL_END_OF_BLOCK 0x0020000000000000ULL
+#define CONTROL_START_OF_BLOCK 0x0040000000000000ULL
+#define CONTROL_ENCRYPT 0x0080000000000000ULL
+#define CONTROL_OPCODE 0xff00000000000000ULL
+#define CONTROL_OPCODE_SHIFT 56
+#define OPCODE_INPLACE_BIT 0x80ULL
+#define OPCODE_SSL_KEYBLOCK 0x10ULL
+#define OPCODE_COPY 0x20ULL
+#define OPCODE_ENCRYPT 0x40ULL
+#define OPCODE_AUTH_MAC 0x41ULL
+
+#endif /* !(__ASSEMBLY__) */
+
+/* NCS v2.0 hypervisor interfaces */
+#define HV_NCS_QTYPE_MAU 0x01
+#define HV_NCS_QTYPE_CWQ 0x02
+
+/* ncs_qconf()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_QCONF
+ * ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
+ * ARG1: Real address of queue, or handle for unconfigure
+ * ARG2: Number of entries in queue, zero for unconfigure
+ * RET0: status
+ * RET1: queue handle
+ *
+ * Configure a queue in the stream processing unit.
+ *
+ * The real address given as the base must be 64-byte
+ * aligned.
+ *
+ * The queue size can range from a minimum of 2 to a maximum
+ * of 64. The queue size must be a power of two.
+ *
+ * To unconfigure a queue, specify a length of zero and place
+ * the queue handle into ARG1.
+ *
+ * On configure success the hypervisor will set the FIRST, HEAD,
+ * and TAIL registers to the address of the first entry in the
+ * queue. The LAST register will be set to point to the last
+ * entry in the queue.
+ */
+#define HV_FAST_NCS_QCONF 0x111
+
+/* ncs_qinfo()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_QINFO
+ * ARG0: Queue handle
+ * RET0: status
+ * RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ})
+ * RET2: Queue base address
+ * RET3: Number of entries
+ */
+#define HV_FAST_NCS_QINFO 0x112
+
+/* ncs_gethead()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_GETHEAD
+ * ARG0: Queue handle
+ * RET0: status
+ * RET1: queue head offset
+ */
+#define HV_FAST_NCS_GETHEAD 0x113
+
+/* ncs_gettail()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_GETTAIL
+ * ARG0: Queue handle
+ * RET0: status
+ * RET1: queue tail offset
+ */
+#define HV_FAST_NCS_GETTAIL 0x114
+
+/* ncs_settail()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_SETTAIL
+ * ARG0: Queue handle
+ * ARG1: New tail offset
+ * RET0: status
+ */
+#define HV_FAST_NCS_SETTAIL 0x115
+
+/* ncs_qhandle_to_devino()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO
+ * ARG0: Queue handle
+ * RET0: status
+ * RET1: devino
+ */
+#define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116
+
+/* ncs_sethead_marker()
+ * TRAP: HV_FAST_TRAP
+ * FUNCTION: HV_FAST_NCS_SETHEAD_MARKER
+ * ARG0: Queue handle
+ * ARG1: New head offset
+ * RET0: status
+ */
+#define HV_FAST_NCS_SETHEAD_MARKER 0x117
+
+#ifndef __ASSEMBLY__
+extern unsigned long sun4v_ncs_qconf(unsigned long queue_type,
+ unsigned long queue_ra,
+ unsigned long num_entries,
+ unsigned long *qhandle);
+extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle,
+ unsigned long *queue_type,
+ unsigned long *queue_ra,
+ unsigned long *num_entries);
+extern unsigned long sun4v_ncs_gethead(unsigned long qhandle,
+ unsigned long *head);
+extern unsigned long sun4v_ncs_gettail(unsigned long qhandle,
+ unsigned long *tail);
+extern unsigned long sun4v_ncs_settail(unsigned long qhandle,
+ unsigned long tail);
+extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle,
+ unsigned long *devino);
+extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle,
+ unsigned long head);
+#endif /* !(__ASSEMBLY__) */
+
+#endif /* _N2_CORE_H */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/omap-aes.c b/ap/os/linux/linux-3.4.x/drivers/crypto/omap-aes.c
new file mode 100644
index 0000000..63e57b5
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/omap-aes.c
@@ -0,0 +1,964 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for OMAP AES HW acceleration.
+ *
+ * Copyright (c) 2010 Nokia Corporation
+ * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/crypto.h>
+#include <linux/interrupt.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/aes.h>
+
+#include <plat/cpu.h>
+#include <plat/dma.h>
+
+/* OMAP TRM gives bitfields as start:end, where start is the higher bit
+ number. For example 7:0 */
+#define FLD_MASK(start, end) (((1 << ((start) - (end) + 1)) - 1) << (end))
+#define FLD_VAL(val, start, end) (((val) << (end)) & FLD_MASK(start, end))
+
+#define AES_REG_KEY(x) (0x1C - ((x ^ 0x01) * 0x04))
+#define AES_REG_IV(x) (0x20 + ((x) * 0x04))
+
+#define AES_REG_CTRL 0x30
+#define AES_REG_CTRL_CTR_WIDTH (1 << 7)
+#define AES_REG_CTRL_CTR (1 << 6)
+#define AES_REG_CTRL_CBC (1 << 5)
+#define AES_REG_CTRL_KEY_SIZE (3 << 3)
+#define AES_REG_CTRL_DIRECTION (1 << 2)
+#define AES_REG_CTRL_INPUT_READY (1 << 1)
+#define AES_REG_CTRL_OUTPUT_READY (1 << 0)
+
+#define AES_REG_DATA 0x34
+#define AES_REG_DATA_N(x) (0x34 + ((x) * 0x04))
+
+#define AES_REG_REV 0x44
+#define AES_REG_REV_MAJOR 0xF0
+#define AES_REG_REV_MINOR 0x0F
+
+#define AES_REG_MASK 0x48
+#define AES_REG_MASK_SIDLE (1 << 6)
+#define AES_REG_MASK_START (1 << 5)
+#define AES_REG_MASK_DMA_OUT_EN (1 << 3)
+#define AES_REG_MASK_DMA_IN_EN (1 << 2)
+#define AES_REG_MASK_SOFTRESET (1 << 1)
+#define AES_REG_AUTOIDLE (1 << 0)
+
+#define AES_REG_SYSSTATUS 0x4C
+#define AES_REG_SYSSTATUS_RESETDONE (1 << 0)
+
+#define DEFAULT_TIMEOUT (5*HZ)
+
+#define FLAGS_MODE_MASK 0x000f
+#define FLAGS_ENCRYPT BIT(0)
+#define FLAGS_CBC BIT(1)
+#define FLAGS_GIV BIT(2)
+
+#define FLAGS_INIT BIT(4)
+#define FLAGS_FAST BIT(5)
+#define FLAGS_BUSY BIT(6)
+
+struct omap_aes_ctx {
+ struct omap_aes_dev *dd;
+
+ int keylen;
+ u32 key[AES_KEYSIZE_256 / sizeof(u32)];
+ unsigned long flags;
+};
+
+struct omap_aes_reqctx {
+ unsigned long mode;
+};
+
+#define OMAP_AES_QUEUE_LENGTH 1
+#define OMAP_AES_CACHE_SIZE 0
+
+struct omap_aes_dev {
+ struct list_head list;
+ unsigned long phys_base;
+ void __iomem *io_base;
+ struct clk *iclk;
+ struct omap_aes_ctx *ctx;
+ struct device *dev;
+ unsigned long flags;
+ int err;
+
+ spinlock_t lock;
+ struct crypto_queue queue;
+
+ struct tasklet_struct done_task;
+ struct tasklet_struct queue_task;
+
+ struct ablkcipher_request *req;
+ size_t total;
+ struct scatterlist *in_sg;
+ size_t in_offset;
+ struct scatterlist *out_sg;
+ size_t out_offset;
+
+ size_t buflen;
+ void *buf_in;
+ size_t dma_size;
+ int dma_in;
+ int dma_lch_in;
+ dma_addr_t dma_addr_in;
+ void *buf_out;
+ int dma_out;
+ int dma_lch_out;
+ dma_addr_t dma_addr_out;
+};
+
+/* keep registered devices data here */
+static LIST_HEAD(dev_list);
+static DEFINE_SPINLOCK(list_lock);
+
+static inline u32 omap_aes_read(struct omap_aes_dev *dd, u32 offset)
+{
+ return __raw_readl(dd->io_base + offset);
+}
+
+static inline void omap_aes_write(struct omap_aes_dev *dd, u32 offset,
+ u32 value)
+{
+ __raw_writel(value, dd->io_base + offset);
+}
+
+static inline void omap_aes_write_mask(struct omap_aes_dev *dd, u32 offset,
+ u32 value, u32 mask)
+{
+ u32 val;
+
+ val = omap_aes_read(dd, offset);
+ val &= ~mask;
+ val |= value;
+ omap_aes_write(dd, offset, val);
+}
+
+static void omap_aes_write_n(struct omap_aes_dev *dd, u32 offset,
+ u32 *value, int count)
+{
+ for (; count--; value++, offset += 4)
+ omap_aes_write(dd, offset, *value);
+}
+
+static int omap_aes_wait(struct omap_aes_dev *dd, u32 offset, u32 bit)
+{
+ unsigned long timeout = jiffies + DEFAULT_TIMEOUT;
+
+ while (!(omap_aes_read(dd, offset) & bit)) {
+ if (time_is_before_jiffies(timeout)) {
+ dev_err(dd->dev, "omap-aes timeout\n");
+ return -ETIMEDOUT;
+ }
+ }
+ return 0;
+}
+
+static int omap_aes_hw_init(struct omap_aes_dev *dd)
+{
+ /*
+ * clocks are enabled when request starts and disabled when finished.
+ * It may be long delays between requests.
+ * Device might go to off mode to save power.
+ */
+ clk_enable(dd->iclk);
+
+ if (!(dd->flags & FLAGS_INIT)) {
+ /* is it necessary to reset before every operation? */
+ omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_SOFTRESET,
+ AES_REG_MASK_SOFTRESET);
+ /*
+ * prevent OCP bus error (SRESP) in case an access to the module
+ * is performed while the module is coming out of soft reset
+ */
+ __asm__ __volatile__("nop");
+ __asm__ __volatile__("nop");
+
+ if (omap_aes_wait(dd, AES_REG_SYSSTATUS,
+ AES_REG_SYSSTATUS_RESETDONE))
+ return -ETIMEDOUT;
+
+ dd->flags |= FLAGS_INIT;
+ dd->err = 0;
+ }
+
+ return 0;
+}
+
+static int omap_aes_write_ctrl(struct omap_aes_dev *dd)
+{
+ unsigned int key32;
+ int i, err;
+ u32 val, mask;
+
+ err = omap_aes_hw_init(dd);
+ if (err)
+ return err;
+
+ val = 0;
+ if (dd->dma_lch_out >= 0)
+ val |= AES_REG_MASK_DMA_OUT_EN;
+ if (dd->dma_lch_in >= 0)
+ val |= AES_REG_MASK_DMA_IN_EN;
+
+ mask = AES_REG_MASK_DMA_IN_EN | AES_REG_MASK_DMA_OUT_EN;
+
+ omap_aes_write_mask(dd, AES_REG_MASK, val, mask);
+
+ key32 = dd->ctx->keylen / sizeof(u32);
+
+ /* it seems a key should always be set even if it has not changed */
+ for (i = 0; i < key32; i++) {
+ omap_aes_write(dd, AES_REG_KEY(i),
+ __le32_to_cpu(dd->ctx->key[i]));
+ }
+
+ if ((dd->flags & FLAGS_CBC) && dd->req->info)
+ omap_aes_write_n(dd, AES_REG_IV(0), dd->req->info, 4);
+
+ val = FLD_VAL(((dd->ctx->keylen >> 3) - 1), 4, 3);
+ if (dd->flags & FLAGS_CBC)
+ val |= AES_REG_CTRL_CBC;
+ if (dd->flags & FLAGS_ENCRYPT)
+ val |= AES_REG_CTRL_DIRECTION;
+
+ mask = AES_REG_CTRL_CBC | AES_REG_CTRL_DIRECTION |
+ AES_REG_CTRL_KEY_SIZE;
+
+ omap_aes_write_mask(dd, AES_REG_CTRL, val, mask);
+
+ /* IN */
+ omap_set_dma_dest_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_CONSTANT,
+ dd->phys_base + AES_REG_DATA, 0, 4);
+
+ omap_set_dma_dest_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
+ omap_set_dma_src_burst_mode(dd->dma_lch_in, OMAP_DMA_DATA_BURST_4);
+
+ /* OUT */
+ omap_set_dma_src_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_CONSTANT,
+ dd->phys_base + AES_REG_DATA, 0, 4);
+
+ omap_set_dma_src_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
+ omap_set_dma_dest_burst_mode(dd->dma_lch_out, OMAP_DMA_DATA_BURST_4);
+
+ return 0;
+}
+
+static struct omap_aes_dev *omap_aes_find_dev(struct omap_aes_ctx *ctx)
+{
+ struct omap_aes_dev *dd = NULL, *tmp;
+
+ spin_lock_bh(&list_lock);
+ if (!ctx->dd) {
+ list_for_each_entry(tmp, &dev_list, list) {
+ /* FIXME: take fist available aes core */
+ dd = tmp;
+ break;
+ }
+ ctx->dd = dd;
+ } else {
+ /* already found before */
+ dd = ctx->dd;
+ }
+ spin_unlock_bh(&list_lock);
+
+ return dd;
+}
+
+static void omap_aes_dma_callback(int lch, u16 ch_status, void *data)
+{
+ struct omap_aes_dev *dd = data;
+
+ if (ch_status != OMAP_DMA_BLOCK_IRQ) {
+ pr_err("omap-aes DMA error status: 0x%hx\n", ch_status);
+ dd->err = -EIO;
+ dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
+ } else if (lch == dd->dma_lch_in) {
+ return;
+ }
+
+ /* dma_lch_out - completed */
+ tasklet_schedule(&dd->done_task);
+}
+
+static int omap_aes_dma_init(struct omap_aes_dev *dd)
+{
+ int err = -ENOMEM;
+
+ dd->dma_lch_out = -1;
+ dd->dma_lch_in = -1;
+
+ dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
+ dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, OMAP_AES_CACHE_SIZE);
+ dd->buflen = PAGE_SIZE << OMAP_AES_CACHE_SIZE;
+ dd->buflen &= ~(AES_BLOCK_SIZE - 1);
+
+ if (!dd->buf_in || !dd->buf_out) {
+ dev_err(dd->dev, "unable to alloc pages.\n");
+ goto err_alloc;
+ }
+
+ /* MAP here */
+ dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, dd->buflen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dd->dev, dd->dma_addr_in)) {
+ dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
+ err = -EINVAL;
+ goto err_map_in;
+ }
+
+ dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, dd->buflen,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(dd->dev, dd->dma_addr_out)) {
+ dev_err(dd->dev, "dma %d bytes error\n", dd->buflen);
+ err = -EINVAL;
+ goto err_map_out;
+ }
+
+ err = omap_request_dma(dd->dma_in, "omap-aes-rx",
+ omap_aes_dma_callback, dd, &dd->dma_lch_in);
+ if (err) {
+ dev_err(dd->dev, "Unable to request DMA channel\n");
+ goto err_dma_in;
+ }
+ err = omap_request_dma(dd->dma_out, "omap-aes-tx",
+ omap_aes_dma_callback, dd, &dd->dma_lch_out);
+ if (err) {
+ dev_err(dd->dev, "Unable to request DMA channel\n");
+ goto err_dma_out;
+ }
+
+ return 0;
+
+err_dma_out:
+ omap_free_dma(dd->dma_lch_in);
+err_dma_in:
+ dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
+ DMA_FROM_DEVICE);
+err_map_out:
+ dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
+err_map_in:
+ free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
+ free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
+err_alloc:
+ if (err)
+ pr_err("error: %d\n", err);
+ return err;
+}
+
+static void omap_aes_dma_cleanup(struct omap_aes_dev *dd)
+{
+ omap_free_dma(dd->dma_lch_out);
+ omap_free_dma(dd->dma_lch_in);
+ dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,
+ DMA_FROM_DEVICE);
+ dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE);
+ free_pages((unsigned long)dd->buf_out, OMAP_AES_CACHE_SIZE);
+ free_pages((unsigned long)dd->buf_in, OMAP_AES_CACHE_SIZE);
+}
+
+static void sg_copy_buf(void *buf, struct scatterlist *sg,
+ unsigned int start, unsigned int nbytes, int out)
+{
+ struct scatter_walk walk;
+
+ if (!nbytes)
+ return;
+
+ scatterwalk_start(&walk, sg);
+ scatterwalk_advance(&walk, start);
+ scatterwalk_copychunks(buf, &walk, nbytes, out);
+ scatterwalk_done(&walk, out, 0);
+}
+
+static int sg_copy(struct scatterlist **sg, size_t *offset, void *buf,
+ size_t buflen, size_t total, int out)
+{
+ unsigned int count, off = 0;
+
+ while (buflen && total) {
+ count = min((*sg)->length - *offset, total);
+ count = min(count, buflen);
+
+ if (!count)
+ return off;
+
+ /*
+ * buflen and total are AES_BLOCK_SIZE size aligned,
+ * so count should be also aligned
+ */
+
+ sg_copy_buf(buf + off, *sg, *offset, count, out);
+
+ off += count;
+ buflen -= count;
+ *offset += count;
+ total -= count;
+
+ if (*offset == (*sg)->length) {
+ *sg = sg_next(*sg);
+ if (*sg)
+ *offset = 0;
+ else
+ total = 0;
+ }
+ }
+
+ return off;
+}
+
+static int omap_aes_crypt_dma(struct crypto_tfm *tfm, dma_addr_t dma_addr_in,
+ dma_addr_t dma_addr_out, int length)
+{
+ struct omap_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct omap_aes_dev *dd = ctx->dd;
+ int len32;
+
+ pr_debug("len: %d\n", length);
+
+ dd->dma_size = length;
+
+ if (!(dd->flags & FLAGS_FAST))
+ dma_sync_single_for_device(dd->dev, dma_addr_in, length,
+ DMA_TO_DEVICE);
+
+ len32 = DIV_ROUND_UP(length, sizeof(u32));
+
+ /* IN */
+ omap_set_dma_transfer_params(dd->dma_lch_in, OMAP_DMA_DATA_TYPE_S32,
+ len32, 1, OMAP_DMA_SYNC_PACKET, dd->dma_in,
+ OMAP_DMA_DST_SYNC);
+
+ omap_set_dma_src_params(dd->dma_lch_in, 0, OMAP_DMA_AMODE_POST_INC,
+ dma_addr_in, 0, 0);
+
+ /* OUT */
+ omap_set_dma_transfer_params(dd->dma_lch_out, OMAP_DMA_DATA_TYPE_S32,
+ len32, 1, OMAP_DMA_SYNC_PACKET,
+ dd->dma_out, OMAP_DMA_SRC_SYNC);
+
+ omap_set_dma_dest_params(dd->dma_lch_out, 0, OMAP_DMA_AMODE_POST_INC,
+ dma_addr_out, 0, 0);
+
+ omap_start_dma(dd->dma_lch_in);
+ omap_start_dma(dd->dma_lch_out);
+
+ /* start DMA or disable idle mode */
+ omap_aes_write_mask(dd, AES_REG_MASK, AES_REG_MASK_START,
+ AES_REG_MASK_START);
+
+ return 0;
+}
+
+static int omap_aes_crypt_dma_start(struct omap_aes_dev *dd)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(
+ crypto_ablkcipher_reqtfm(dd->req));
+ int err, fast = 0, in, out;
+ size_t count;
+ dma_addr_t addr_in, addr_out;
+
+ pr_debug("total: %d\n", dd->total);
+
+ if (sg_is_last(dd->in_sg) && sg_is_last(dd->out_sg)) {
+ /* check for alignment */
+ in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32));
+ out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32));
+
+ fast = in && out;
+ }
+
+ if (fast) {
+ count = min(dd->total, sg_dma_len(dd->in_sg));
+ count = min(count, sg_dma_len(dd->out_sg));
+
+ if (count != dd->total) {
+ pr_err("request length != buffer length\n");
+ return -EINVAL;
+ }
+
+ pr_debug("fast\n");
+
+ err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ if (!err) {
+ dev_err(dd->dev, "dma_map_sg() error\n");
+ return -EINVAL;
+ }
+
+ err = dma_map_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
+ if (!err) {
+ dev_err(dd->dev, "dma_map_sg() error\n");
+ dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ return -EINVAL;
+ }
+
+ addr_in = sg_dma_address(dd->in_sg);
+ addr_out = sg_dma_address(dd->out_sg);
+
+ dd->flags |= FLAGS_FAST;
+
+ } else {
+ /* use cache buffers */
+ count = sg_copy(&dd->in_sg, &dd->in_offset, dd->buf_in,
+ dd->buflen, dd->total, 0);
+
+ addr_in = dd->dma_addr_in;
+ addr_out = dd->dma_addr_out;
+
+ dd->flags &= ~FLAGS_FAST;
+
+ }
+
+ dd->total -= count;
+
+ err = omap_aes_crypt_dma(tfm, addr_in, addr_out, count);
+ if (err) {
+ dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);
+ }
+
+ return err;
+}
+
+static void omap_aes_finish_req(struct omap_aes_dev *dd, int err)
+{
+ struct ablkcipher_request *req = dd->req;
+
+ pr_debug("err: %d\n", err);
+
+ clk_disable(dd->iclk);
+ dd->flags &= ~FLAGS_BUSY;
+
+ req->base.complete(&req->base, err);
+}
+
+static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd)
+{
+ int err = 0;
+ size_t count;
+
+ pr_debug("total: %d\n", dd->total);
+
+ omap_aes_write_mask(dd, AES_REG_MASK, 0, AES_REG_MASK_START);
+
+ omap_stop_dma(dd->dma_lch_in);
+ omap_stop_dma(dd->dma_lch_out);
+
+ if (dd->flags & FLAGS_FAST) {
+ dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);
+ dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);
+ } else {
+ dma_sync_single_for_device(dd->dev, dd->dma_addr_out,
+ dd->dma_size, DMA_FROM_DEVICE);
+
+ /* copy data */
+ count = sg_copy(&dd->out_sg, &dd->out_offset, dd->buf_out,
+ dd->buflen, dd->dma_size, 1);
+ if (count != dd->dma_size) {
+ err = -EINVAL;
+ pr_err("not all data converted: %u\n", count);
+ }
+ }
+
+ return err;
+}
+
+static int omap_aes_handle_queue(struct omap_aes_dev *dd,
+ struct ablkcipher_request *req)
+{
+ struct crypto_async_request *async_req, *backlog;
+ struct omap_aes_ctx *ctx;
+ struct omap_aes_reqctx *rctx;
+ unsigned long flags;
+ int err, ret = 0;
+
+ spin_lock_irqsave(&dd->lock, flags);
+ if (req)
+ ret = ablkcipher_enqueue_request(&dd->queue, req);
+ if (dd->flags & FLAGS_BUSY) {
+ spin_unlock_irqrestore(&dd->lock, flags);
+ return ret;
+ }
+ backlog = crypto_get_backlog(&dd->queue);
+ async_req = crypto_dequeue_request(&dd->queue);
+ if (async_req)
+ dd->flags |= FLAGS_BUSY;
+ spin_unlock_irqrestore(&dd->lock, flags);
+
+ if (!async_req)
+ return ret;
+
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
+ req = ablkcipher_request_cast(async_req);
+
+ /* assign new request to device */
+ dd->req = req;
+ dd->total = req->nbytes;
+ dd->in_offset = 0;
+ dd->in_sg = req->src;
+ dd->out_offset = 0;
+ dd->out_sg = req->dst;
+
+ rctx = ablkcipher_request_ctx(req);
+ ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
+ rctx->mode &= FLAGS_MODE_MASK;
+ dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
+
+ dd->ctx = ctx;
+ ctx->dd = dd;
+
+ err = omap_aes_write_ctrl(dd);
+ if (!err)
+ err = omap_aes_crypt_dma_start(dd);
+ if (err) {
+ /* aes_task will not finish it, so do it here */
+ omap_aes_finish_req(dd, err);
+ tasklet_schedule(&dd->queue_task);
+ }
+
+ return ret; /* return ret, which is enqueue return value */
+}
+
+static void omap_aes_done_task(unsigned long data)
+{
+ struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
+ int err;
+
+ pr_debug("enter\n");
+
+ err = omap_aes_crypt_dma_stop(dd);
+
+ err = dd->err ? : err;
+
+ if (dd->total && !err) {
+ err = omap_aes_crypt_dma_start(dd);
+ if (!err)
+ return; /* DMA started. Not fininishing. */
+ }
+
+ omap_aes_finish_req(dd, err);
+ omap_aes_handle_queue(dd, NULL);
+
+ pr_debug("exit\n");
+}
+
+static void omap_aes_queue_task(unsigned long data)
+{
+ struct omap_aes_dev *dd = (struct omap_aes_dev *)data;
+
+ omap_aes_handle_queue(dd, NULL);
+}
+
+static int omap_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
+{
+ struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(
+ crypto_ablkcipher_reqtfm(req));
+ struct omap_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+ struct omap_aes_dev *dd;
+
+ pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->nbytes,
+ !!(mode & FLAGS_ENCRYPT),
+ !!(mode & FLAGS_CBC));
+
+ if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
+ pr_err("request size is not exact amount of AES blocks\n");
+ return -EINVAL;
+ }
+
+ dd = omap_aes_find_dev(ctx);
+ if (!dd)
+ return -ENODEV;
+
+ rctx->mode = mode;
+
+ return omap_aes_handle_queue(dd, req);
+}
+
+/* ********************** ALG API ************************************ */
+
+static int omap_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct omap_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+ if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+
+ pr_debug("enter, keylen: %d\n", keylen);
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static int omap_aes_ecb_encrypt(struct ablkcipher_request *req)
+{
+ return omap_aes_crypt(req, FLAGS_ENCRYPT);
+}
+
+static int omap_aes_ecb_decrypt(struct ablkcipher_request *req)
+{
+ return omap_aes_crypt(req, 0);
+}
+
+static int omap_aes_cbc_encrypt(struct ablkcipher_request *req)
+{
+ return omap_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
+}
+
+static int omap_aes_cbc_decrypt(struct ablkcipher_request *req)
+{
+ return omap_aes_crypt(req, FLAGS_CBC);
+}
+
+static int omap_aes_cra_init(struct crypto_tfm *tfm)
+{
+ pr_debug("enter\n");
+
+ tfm->crt_ablkcipher.reqsize = sizeof(struct omap_aes_reqctx);
+
+ return 0;
+}
+
+static void omap_aes_cra_exit(struct crypto_tfm *tfm)
+{
+ pr_debug("enter\n");
+}
+
+/* ********************** ALGS ************************************ */
+
+static struct crypto_alg algs[] = {
+{
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-omap",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_aes_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_aes_cra_init,
+ .cra_exit = omap_aes_cra_exit,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = omap_aes_setkey,
+ .encrypt = omap_aes_ecb_encrypt,
+ .decrypt = omap_aes_ecb_decrypt,
+ }
+},
+{
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-omap",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_aes_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_aes_cra_init,
+ .cra_exit = omap_aes_cra_exit,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = omap_aes_setkey,
+ .encrypt = omap_aes_cbc_encrypt,
+ .decrypt = omap_aes_cbc_decrypt,
+ }
+}
+};
+
+static int omap_aes_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct omap_aes_dev *dd;
+ struct resource *res;
+ int err = -ENOMEM, i, j;
+ u32 reg;
+
+ dd = kzalloc(sizeof(struct omap_aes_dev), GFP_KERNEL);
+ if (dd == NULL) {
+ dev_err(dev, "unable to alloc data struct.\n");
+ goto err_data;
+ }
+ dd->dev = dev;
+ platform_set_drvdata(pdev, dd);
+
+ spin_lock_init(&dd->lock);
+ crypto_init_queue(&dd->queue, OMAP_AES_QUEUE_LENGTH);
+
+ /* Get the base address */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ dev_err(dev, "invalid resource type\n");
+ err = -ENODEV;
+ goto err_res;
+ }
+ dd->phys_base = res->start;
+
+ /* Get the DMA */
+ res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
+ if (!res)
+ dev_info(dev, "no DMA info\n");
+ else
+ dd->dma_out = res->start;
+
+ /* Get the DMA */
+ res = platform_get_resource(pdev, IORESOURCE_DMA, 1);
+ if (!res)
+ dev_info(dev, "no DMA info\n");
+ else
+ dd->dma_in = res->start;
+
+ /* Initializing the clock */
+ dd->iclk = clk_get(dev, "ick");
+ if (IS_ERR(dd->iclk)) {
+ dev_err(dev, "clock intialization failed.\n");
+ err = PTR_ERR(dd->iclk);
+ goto err_res;
+ }
+
+ dd->io_base = ioremap(dd->phys_base, SZ_4K);
+ if (!dd->io_base) {
+ dev_err(dev, "can't ioremap\n");
+ err = -ENOMEM;
+ goto err_io;
+ }
+
+ clk_enable(dd->iclk);
+ reg = omap_aes_read(dd, AES_REG_REV);
+ dev_info(dev, "OMAP AES hw accel rev: %u.%u\n",
+ (reg & AES_REG_REV_MAJOR) >> 4, reg & AES_REG_REV_MINOR);
+ clk_disable(dd->iclk);
+
+ tasklet_init(&dd->done_task, omap_aes_done_task, (unsigned long)dd);
+ tasklet_init(&dd->queue_task, omap_aes_queue_task, (unsigned long)dd);
+
+ err = omap_aes_dma_init(dd);
+ if (err)
+ goto err_dma;
+
+ INIT_LIST_HEAD(&dd->list);
+ spin_lock(&list_lock);
+ list_add_tail(&dd->list, &dev_list);
+ spin_unlock(&list_lock);
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++) {
+ pr_debug("i: %d\n", i);
+ INIT_LIST_HEAD(&algs[i].cra_list);
+ err = crypto_register_alg(&algs[i]);
+ if (err)
+ goto err_algs;
+ }
+
+ pr_info("probe() done\n");
+
+ return 0;
+err_algs:
+ for (j = 0; j < i; j++)
+ crypto_unregister_alg(&algs[j]);
+ omap_aes_dma_cleanup(dd);
+err_dma:
+ tasklet_kill(&dd->done_task);
+ tasklet_kill(&dd->queue_task);
+ iounmap(dd->io_base);
+err_io:
+ clk_put(dd->iclk);
+err_res:
+ kfree(dd);
+ dd = NULL;
+err_data:
+ dev_err(dev, "initialization failed.\n");
+ return err;
+}
+
+static int omap_aes_remove(struct platform_device *pdev)
+{
+ struct omap_aes_dev *dd = platform_get_drvdata(pdev);
+ int i;
+
+ if (!dd)
+ return -ENODEV;
+
+ spin_lock(&list_lock);
+ list_del(&dd->list);
+ spin_unlock(&list_lock);
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++)
+ crypto_unregister_alg(&algs[i]);
+
+ tasklet_kill(&dd->done_task);
+ tasklet_kill(&dd->queue_task);
+ omap_aes_dma_cleanup(dd);
+ iounmap(dd->io_base);
+ clk_put(dd->iclk);
+ kfree(dd);
+ dd = NULL;
+
+ return 0;
+}
+
+static struct platform_driver omap_aes_driver = {
+ .probe = omap_aes_probe,
+ .remove = omap_aes_remove,
+ .driver = {
+ .name = "omap-aes",
+ .owner = THIS_MODULE,
+ },
+};
+
+static int __init omap_aes_mod_init(void)
+{
+ pr_info("loading %s driver\n", "omap-aes");
+
+ if (!cpu_class_is_omap2() || omap_type() != OMAP2_DEVICE_TYPE_SEC) {
+ pr_err("Unsupported cpu\n");
+ return -ENODEV;
+ }
+
+ return platform_driver_register(&omap_aes_driver);
+}
+
+static void __exit omap_aes_mod_exit(void)
+{
+ platform_driver_unregister(&omap_aes_driver);
+}
+
+module_init(omap_aes_mod_init);
+module_exit(omap_aes_mod_exit);
+
+MODULE_DESCRIPTION("OMAP AES hw acceleration support.");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Dmitry Kasatkin");
+
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/omap-sham.c b/ap/os/linux/linux-3.4.x/drivers/crypto/omap-sham.c
new file mode 100644
index 0000000..a3fd6fc
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/omap-sham.c
@@ -0,0 +1,1312 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for OMAP SHA1/MD5 HW acceleration.
+ *
+ * Copyright (c) 2010 Nokia Corporation
+ * Author: Dmitry Kasatkin <dmitry.kasatkin@nokia.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ * Some ideas are from old omap-sha1-md5.c driver.
+ */
+
+#define pr_fmt(fmt) "%s: " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/irq.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/delay.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/algapi.h>
+#include <crypto/sha.h>
+#include <crypto/hash.h>
+#include <crypto/internal/hash.h>
+
+#include <plat/cpu.h>
+#include <plat/dma.h>
+#include <mach/irqs.h>
+
+#define SHA_REG_DIGEST(x) (0x00 + ((x) * 0x04))
+#define SHA_REG_DIN(x) (0x1C + ((x) * 0x04))
+
+#define SHA1_MD5_BLOCK_SIZE SHA1_BLOCK_SIZE
+#define MD5_DIGEST_SIZE 16
+
+#define SHA_REG_DIGCNT 0x14
+
+#define SHA_REG_CTRL 0x18
+#define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
+#define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
+#define SHA_REG_CTRL_ALGO_CONST (1 << 3)
+#define SHA_REG_CTRL_ALGO (1 << 2)
+#define SHA_REG_CTRL_INPUT_READY (1 << 1)
+#define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
+
+#define SHA_REG_REV 0x5C
+#define SHA_REG_REV_MAJOR 0xF0
+#define SHA_REG_REV_MINOR 0x0F
+
+#define SHA_REG_MASK 0x60
+#define SHA_REG_MASK_DMA_EN (1 << 3)
+#define SHA_REG_MASK_IT_EN (1 << 2)
+#define SHA_REG_MASK_SOFTRESET (1 << 1)
+#define SHA_REG_AUTOIDLE (1 << 0)
+
+#define SHA_REG_SYSSTATUS 0x64
+#define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
+
+#define DEFAULT_TIMEOUT_INTERVAL HZ
+
+/* mostly device flags */
+#define FLAGS_BUSY 0
+#define FLAGS_FINAL 1
+#define FLAGS_DMA_ACTIVE 2
+#define FLAGS_OUTPUT_READY 3
+#define FLAGS_INIT 4
+#define FLAGS_CPU 5
+#define FLAGS_DMA_READY 6
+/* context flags */
+#define FLAGS_FINUP 16
+#define FLAGS_SG 17
+#define FLAGS_SHA1 18
+#define FLAGS_HMAC 19
+#define FLAGS_ERROR 20
+
+#define OP_UPDATE 1
+#define OP_FINAL 2
+
+#define OMAP_ALIGN_MASK (sizeof(u32)-1)
+#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
+
+#define BUFLEN PAGE_SIZE
+
+struct omap_sham_dev;
+
+struct omap_sham_reqctx {
+ struct omap_sham_dev *dd;
+ unsigned long flags;
+ unsigned long op;
+
+ u8 digest[SHA1_DIGEST_SIZE] OMAP_ALIGNED;
+ size_t digcnt;
+ size_t bufcnt;
+ size_t buflen;
+ dma_addr_t dma_addr;
+
+ /* walk state */
+ struct scatterlist *sg;
+ unsigned int offset; /* offset in current sg */
+ unsigned int total; /* total request */
+
+ u8 buffer[0] OMAP_ALIGNED;
+};
+
+struct omap_sham_hmac_ctx {
+ struct crypto_shash *shash;
+ u8 ipad[SHA1_MD5_BLOCK_SIZE];
+ u8 opad[SHA1_MD5_BLOCK_SIZE];
+};
+
+struct omap_sham_ctx {
+ struct omap_sham_dev *dd;
+
+ unsigned long flags;
+
+ /* fallback stuff */
+ struct crypto_shash *fallback;
+
+ struct omap_sham_hmac_ctx base[0];
+};
+
+#define OMAP_SHAM_QUEUE_LENGTH 1
+
+struct omap_sham_dev {
+ struct list_head list;
+ unsigned long phys_base;
+ struct device *dev;
+ void __iomem *io_base;
+ int irq;
+ struct clk *iclk;
+ spinlock_t lock;
+ int err;
+ int dma;
+ int dma_lch;
+ struct tasklet_struct done_task;
+
+ unsigned long flags;
+ struct crypto_queue queue;
+ struct ahash_request *req;
+};
+
+struct omap_sham_drv {
+ struct list_head dev_list;
+ spinlock_t lock;
+ unsigned long flags;
+};
+
+static struct omap_sham_drv sham = {
+ .dev_list = LIST_HEAD_INIT(sham.dev_list),
+ .lock = __SPIN_LOCK_UNLOCKED(sham.lock),
+};
+
+static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
+{
+ return __raw_readl(dd->io_base + offset);
+}
+
+static inline void omap_sham_write(struct omap_sham_dev *dd,
+ u32 offset, u32 value)
+{
+ __raw_writel(value, dd->io_base + offset);
+}
+
+static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
+ u32 value, u32 mask)
+{
+ u32 val;
+
+ val = omap_sham_read(dd, address);
+ val &= ~mask;
+ val |= value;
+ omap_sham_write(dd, address, val);
+}
+
+static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
+{
+ unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
+
+ while (!(omap_sham_read(dd, offset) & bit)) {
+ if (time_is_before_jiffies(timeout))
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
+static void omap_sham_copy_hash(struct ahash_request *req, int out)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ u32 *hash = (u32 *)ctx->digest;
+ int i;
+
+ /* MD5 is almost unused. So copy sha1 size to reduce code */
+ for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
+ if (out)
+ hash[i] = omap_sham_read(ctx->dd,
+ SHA_REG_DIGEST(i));
+ else
+ omap_sham_write(ctx->dd,
+ SHA_REG_DIGEST(i), hash[i]);
+ }
+}
+
+static void omap_sham_copy_ready_hash(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ u32 *in = (u32 *)ctx->digest;
+ u32 *hash = (u32 *)req->result;
+ int i;
+
+ if (!hash)
+ return;
+
+ if (likely(ctx->flags & BIT(FLAGS_SHA1))) {
+ /* SHA1 results are in big endian */
+ for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
+ hash[i] = be32_to_cpu(in[i]);
+ } else {
+ /* MD5 results are in little endian */
+ for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
+ hash[i] = le32_to_cpu(in[i]);
+ }
+}
+
+static int omap_sham_hw_init(struct omap_sham_dev *dd)
+{
+ clk_enable(dd->iclk);
+
+ if (!test_bit(FLAGS_INIT, &dd->flags)) {
+ omap_sham_write_mask(dd, SHA_REG_MASK,
+ SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
+
+ if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
+ SHA_REG_SYSSTATUS_RESETDONE))
+ return -ETIMEDOUT;
+
+ set_bit(FLAGS_INIT, &dd->flags);
+ dd->err = 0;
+ }
+
+ return 0;
+}
+
+static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
+ int final, int dma)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ u32 val = length << 5, mask;
+
+ if (likely(ctx->digcnt))
+ omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
+
+ omap_sham_write_mask(dd, SHA_REG_MASK,
+ SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
+ SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
+ /*
+ * Setting ALGO_CONST only for the first iteration
+ * and CLOSE_HASH only for the last one.
+ */
+ if (ctx->flags & BIT(FLAGS_SHA1))
+ val |= SHA_REG_CTRL_ALGO;
+ if (!ctx->digcnt)
+ val |= SHA_REG_CTRL_ALGO_CONST;
+ if (final)
+ val |= SHA_REG_CTRL_CLOSE_HASH;
+
+ mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
+ SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
+
+ omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
+}
+
+static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
+ size_t length, int final)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ int count, len32;
+ const u32 *buffer = (const u32 *)buf;
+
+ dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
+ ctx->digcnt, length, final);
+
+ omap_sham_write_ctrl(dd, length, final, 0);
+
+ /* should be non-zero before next lines to disable clocks later */
+ ctx->digcnt += length;
+
+ if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY))
+ return -ETIMEDOUT;
+
+ if (final)
+ set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
+
+ set_bit(FLAGS_CPU, &dd->flags);
+
+ len32 = DIV_ROUND_UP(length, sizeof(u32));
+
+ for (count = 0; count < len32; count++)
+ omap_sham_write(dd, SHA_REG_DIN(count), buffer[count]);
+
+ return -EINPROGRESS;
+}
+
+static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
+ size_t length, int final)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ int len32;
+
+ dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
+ ctx->digcnt, length, final);
+
+ len32 = DIV_ROUND_UP(length, sizeof(u32));
+
+ omap_set_dma_transfer_params(dd->dma_lch, OMAP_DMA_DATA_TYPE_S32, len32,
+ 1, OMAP_DMA_SYNC_PACKET, dd->dma,
+ OMAP_DMA_DST_SYNC_PREFETCH);
+
+ omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
+ dma_addr, 0, 0);
+
+ omap_sham_write_ctrl(dd, length, final, 1);
+
+ ctx->digcnt += length;
+
+ if (final)
+ set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
+
+ set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
+
+ omap_start_dma(dd->dma_lch);
+
+ return -EINPROGRESS;
+}
+
+static size_t omap_sham_append_buffer(struct omap_sham_reqctx *ctx,
+ const u8 *data, size_t length)
+{
+ size_t count = min(length, ctx->buflen - ctx->bufcnt);
+
+ count = min(count, ctx->total);
+ if (count <= 0)
+ return 0;
+ memcpy(ctx->buffer + ctx->bufcnt, data, count);
+ ctx->bufcnt += count;
+
+ return count;
+}
+
+static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
+{
+ size_t count;
+
+ while (ctx->sg) {
+ count = omap_sham_append_buffer(ctx,
+ sg_virt(ctx->sg) + ctx->offset,
+ ctx->sg->length - ctx->offset);
+ if (!count)
+ break;
+ ctx->offset += count;
+ ctx->total -= count;
+ if (ctx->offset == ctx->sg->length) {
+ ctx->sg = sg_next(ctx->sg);
+ if (ctx->sg)
+ ctx->offset = 0;
+ else
+ ctx->total = 0;
+ }
+ }
+
+ return 0;
+}
+
+static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
+ struct omap_sham_reqctx *ctx,
+ size_t length, int final)
+{
+ ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
+ dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
+ return -EINVAL;
+ }
+
+ ctx->flags &= ~BIT(FLAGS_SG);
+
+ /* next call does not fail... so no unmap in the case of error */
+ return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
+}
+
+static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ unsigned int final;
+ size_t count;
+
+ omap_sham_append_sg(ctx);
+
+ final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+
+ dev_dbg(dd->dev, "slow: bufcnt: %u, digcnt: %d, final: %d\n",
+ ctx->bufcnt, ctx->digcnt, final);
+
+ if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
+ count = ctx->bufcnt;
+ ctx->bufcnt = 0;
+ return omap_sham_xmit_dma_map(dd, ctx, count, final);
+ }
+
+ return 0;
+}
+
+/* Start address alignment */
+#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
+/* SHA1 block size alignment */
+#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
+
+static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ unsigned int length, final, tail;
+ struct scatterlist *sg;
+
+ if (!ctx->total)
+ return 0;
+
+ if (ctx->bufcnt || ctx->offset)
+ return omap_sham_update_dma_slow(dd);
+
+ dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
+ ctx->digcnt, ctx->bufcnt, ctx->total);
+
+ sg = ctx->sg;
+
+ if (!SG_AA(sg))
+ return omap_sham_update_dma_slow(dd);
+
+ if (!sg_is_last(sg) && !SG_SA(sg))
+ /* size is not SHA1_BLOCK_SIZE aligned */
+ return omap_sham_update_dma_slow(dd);
+
+ length = min(ctx->total, sg->length);
+
+ if (sg_is_last(sg)) {
+ if (!(ctx->flags & BIT(FLAGS_FINUP))) {
+ /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
+ tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
+ /* without finup() we need one block to close hash */
+ if (!tail)
+ tail = SHA1_MD5_BLOCK_SIZE;
+ length -= tail;
+ }
+ }
+
+ if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
+ dev_err(dd->dev, "dma_map_sg error\n");
+ return -EINVAL;
+ }
+
+ ctx->flags |= BIT(FLAGS_SG);
+
+ ctx->total -= length;
+ ctx->offset = length; /* offset where to start slow */
+
+ final = (ctx->flags & BIT(FLAGS_FINUP)) && !ctx->total;
+
+ /* next call does not fail... so no unmap in the case of error */
+ return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
+}
+
+static int omap_sham_update_cpu(struct omap_sham_dev *dd)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ int bufcnt;
+
+ omap_sham_append_sg(ctx);
+ bufcnt = ctx->bufcnt;
+ ctx->bufcnt = 0;
+
+ return omap_sham_xmit_cpu(dd, ctx->buffer, bufcnt, 1);
+}
+
+static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+
+ omap_stop_dma(dd->dma_lch);
+ if (ctx->flags & BIT(FLAGS_SG)) {
+ dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
+ if (ctx->sg->length == ctx->offset) {
+ ctx->sg = sg_next(ctx->sg);
+ if (ctx->sg)
+ ctx->offset = 0;
+ }
+ } else {
+ dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
+ DMA_TO_DEVICE);
+ }
+
+ return 0;
+}
+
+static int omap_sham_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = NULL, *tmp;
+
+ spin_lock_bh(&sham.lock);
+ if (!tctx->dd) {
+ list_for_each_entry(tmp, &sham.dev_list, list) {
+ dd = tmp;
+ break;
+ }
+ tctx->dd = dd;
+ } else {
+ dd = tctx->dd;
+ }
+ spin_unlock_bh(&sham.lock);
+
+ ctx->dd = dd;
+
+ ctx->flags = 0;
+
+ dev_dbg(dd->dev, "init: digest size: %d\n",
+ crypto_ahash_digestsize(tfm));
+
+ if (crypto_ahash_digestsize(tfm) == SHA1_DIGEST_SIZE)
+ ctx->flags |= BIT(FLAGS_SHA1);
+
+ ctx->bufcnt = 0;
+ ctx->digcnt = 0;
+ ctx->buflen = BUFLEN;
+
+ if (tctx->flags & BIT(FLAGS_HMAC)) {
+ struct omap_sham_hmac_ctx *bctx = tctx->base;
+
+ memcpy(ctx->buffer, bctx->ipad, SHA1_MD5_BLOCK_SIZE);
+ ctx->bufcnt = SHA1_MD5_BLOCK_SIZE;
+ ctx->flags |= BIT(FLAGS_HMAC);
+ }
+
+ return 0;
+
+}
+
+static int omap_sham_update_req(struct omap_sham_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ int err;
+
+ dev_dbg(dd->dev, "update_req: total: %u, digcnt: %d, finup: %d\n",
+ ctx->total, ctx->digcnt, (ctx->flags & BIT(FLAGS_FINUP)) != 0);
+
+ if (ctx->flags & BIT(FLAGS_CPU))
+ err = omap_sham_update_cpu(dd);
+ else
+ err = omap_sham_update_dma_start(dd);
+
+ /* wait for dma completion before can take more data */
+ dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
+
+ return err;
+}
+
+static int omap_sham_final_req(struct omap_sham_dev *dd)
+{
+ struct ahash_request *req = dd->req;
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ int err = 0, use_dma = 1;
+
+ if (ctx->bufcnt <= 64)
+ /* faster to handle last block with cpu */
+ use_dma = 0;
+
+ if (use_dma)
+ err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
+ else
+ err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
+
+ ctx->bufcnt = 0;
+
+ dev_dbg(dd->dev, "final_req: err: %d\n", err);
+
+ return err;
+}
+
+static int omap_sham_finish_hmac(struct ahash_request *req)
+{
+ struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
+ struct omap_sham_hmac_ctx *bctx = tctx->base;
+ int bs = crypto_shash_blocksize(bctx->shash);
+ int ds = crypto_shash_digestsize(bctx->shash);
+ struct {
+ struct shash_desc shash;
+ char ctx[crypto_shash_descsize(bctx->shash)];
+ } desc;
+
+ desc.shash.tfm = bctx->shash;
+ desc.shash.flags = 0; /* not CRYPTO_TFM_REQ_MAY_SLEEP */
+
+ return crypto_shash_init(&desc.shash) ?:
+ crypto_shash_update(&desc.shash, bctx->opad, bs) ?:
+ crypto_shash_finup(&desc.shash, req->result, ds, req->result);
+}
+
+static int omap_sham_finish(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
+ int err = 0;
+
+ if (ctx->digcnt) {
+ omap_sham_copy_ready_hash(req);
+ if (ctx->flags & BIT(FLAGS_HMAC))
+ err = omap_sham_finish_hmac(req);
+ }
+
+ dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
+
+ return err;
+}
+
+static void omap_sham_finish_req(struct ahash_request *req, int err)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
+
+ if (!err) {
+ omap_sham_copy_hash(req, 1);
+ if (test_bit(FLAGS_FINAL, &dd->flags))
+ err = omap_sham_finish(req);
+ } else {
+ ctx->flags |= BIT(FLAGS_ERROR);
+ }
+
+ /* atomic operation is not needed here */
+ dd->flags &= ~(BIT(FLAGS_BUSY) | BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
+ BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
+ clk_disable(dd->iclk);
+
+ if (req->base.complete)
+ req->base.complete(&req->base, err);
+
+ /* handle new request */
+ tasklet_schedule(&dd->done_task);
+}
+
+static int omap_sham_handle_queue(struct omap_sham_dev *dd,
+ struct ahash_request *req)
+{
+ struct crypto_async_request *async_req, *backlog;
+ struct omap_sham_reqctx *ctx;
+ unsigned long flags;
+ int err = 0, ret = 0;
+
+ spin_lock_irqsave(&dd->lock, flags);
+ if (req)
+ ret = ahash_enqueue_request(&dd->queue, req);
+ if (test_bit(FLAGS_BUSY, &dd->flags)) {
+ spin_unlock_irqrestore(&dd->lock, flags);
+ return ret;
+ }
+ backlog = crypto_get_backlog(&dd->queue);
+ async_req = crypto_dequeue_request(&dd->queue);
+ if (async_req)
+ set_bit(FLAGS_BUSY, &dd->flags);
+ spin_unlock_irqrestore(&dd->lock, flags);
+
+ if (!async_req)
+ return ret;
+
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
+ req = ahash_request_cast(async_req);
+ dd->req = req;
+ ctx = ahash_request_ctx(req);
+
+ dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
+ ctx->op, req->nbytes);
+
+ err = omap_sham_hw_init(dd);
+ if (err)
+ goto err1;
+
+ omap_set_dma_dest_params(dd->dma_lch, 0,
+ OMAP_DMA_AMODE_CONSTANT,
+ dd->phys_base + SHA_REG_DIN(0), 0, 16);
+
+ omap_set_dma_dest_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_16);
+
+ omap_set_dma_src_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_4);
+
+ if (ctx->digcnt)
+ /* request has changed - restore hash */
+ omap_sham_copy_hash(req, 0);
+
+ if (ctx->op == OP_UPDATE) {
+ err = omap_sham_update_req(dd);
+ if (err != -EINPROGRESS && (ctx->flags & BIT(FLAGS_FINUP)))
+ /* no final() after finup() */
+ err = omap_sham_final_req(dd);
+ } else if (ctx->op == OP_FINAL) {
+ err = omap_sham_final_req(dd);
+ }
+err1:
+ if (err != -EINPROGRESS)
+ /* done_task will not finish it, so do it here */
+ omap_sham_finish_req(req, err);
+
+ dev_dbg(dd->dev, "exit, err: %d\n", err);
+
+ return ret;
+}
+
+static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
+ struct omap_sham_dev *dd = tctx->dd;
+
+ ctx->op = op;
+
+ return omap_sham_handle_queue(dd, req);
+}
+
+static int omap_sham_update(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+
+ if (!req->nbytes)
+ return 0;
+
+ ctx->total = req->nbytes;
+ ctx->sg = req->src;
+ ctx->offset = 0;
+
+ if (ctx->flags & BIT(FLAGS_FINUP)) {
+ if ((ctx->digcnt + ctx->bufcnt + ctx->total) < 9) {
+ /*
+ * OMAP HW accel works only with buffers >= 9
+ * will switch to bypass in final()
+ * final has the same request and data
+ */
+ omap_sham_append_sg(ctx);
+ return 0;
+ } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
+ /*
+ * faster to use CPU for short transfers
+ */
+ ctx->flags |= BIT(FLAGS_CPU);
+ }
+ } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
+ omap_sham_append_sg(ctx);
+ return 0;
+ }
+
+ return omap_sham_enqueue(req, OP_UPDATE);
+}
+
+static int omap_sham_shash_digest(struct crypto_shash *shash, u32 flags,
+ const u8 *data, unsigned int len, u8 *out)
+{
+ struct {
+ struct shash_desc shash;
+ char ctx[crypto_shash_descsize(shash)];
+ } desc;
+
+ desc.shash.tfm = shash;
+ desc.shash.flags = flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_shash_digest(&desc.shash, data, len, out);
+}
+
+static int omap_sham_final_shash(struct ahash_request *req)
+{
+ struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+
+ return omap_sham_shash_digest(tctx->fallback, req->base.flags,
+ ctx->buffer, ctx->bufcnt, req->result);
+}
+
+static int omap_sham_final(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+
+ ctx->flags |= BIT(FLAGS_FINUP);
+
+ if (ctx->flags & BIT(FLAGS_ERROR))
+ return 0; /* uncompleted hash is not needed */
+
+ /* OMAP HW accel works only with buffers >= 9 */
+ /* HMAC is always >= 9 because ipad == block size */
+ if ((ctx->digcnt + ctx->bufcnt) < 9)
+ return omap_sham_final_shash(req);
+ else if (ctx->bufcnt)
+ return omap_sham_enqueue(req, OP_FINAL);
+
+ /* copy ready hash (+ finalize hmac) */
+ return omap_sham_finish(req);
+}
+
+static int omap_sham_finup(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ int err1, err2;
+
+ ctx->flags |= BIT(FLAGS_FINUP);
+
+ err1 = omap_sham_update(req);
+ if (err1 == -EINPROGRESS || err1 == -EBUSY)
+ return err1;
+ /*
+ * final() has to be always called to cleanup resources
+ * even if udpate() failed, except EINPROGRESS
+ */
+ err2 = omap_sham_final(req);
+
+ return err1 ?: err2;
+}
+
+static int omap_sham_digest(struct ahash_request *req)
+{
+ return omap_sham_init(req) ?: omap_sham_finup(req);
+}
+
+static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
+ struct omap_sham_hmac_ctx *bctx = tctx->base;
+ int bs = crypto_shash_blocksize(bctx->shash);
+ int ds = crypto_shash_digestsize(bctx->shash);
+ int err, i;
+ err = crypto_shash_setkey(tctx->fallback, key, keylen);
+ if (err)
+ return err;
+
+ if (keylen > bs) {
+ err = omap_sham_shash_digest(bctx->shash,
+ crypto_shash_get_flags(bctx->shash),
+ key, keylen, bctx->ipad);
+ if (err)
+ return err;
+ keylen = ds;
+ } else {
+ memcpy(bctx->ipad, key, keylen);
+ }
+
+ memset(bctx->ipad + keylen, 0, bs - keylen);
+ memcpy(bctx->opad, bctx->ipad, bs);
+
+ for (i = 0; i < bs; i++) {
+ bctx->ipad[i] ^= 0x36;
+ bctx->opad[i] ^= 0x5c;
+ }
+
+ return err;
+}
+
+static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
+{
+ struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
+ const char *alg_name = crypto_tfm_alg_name(tfm);
+
+ /* Allocate a fallback and abort if it failed. */
+ tctx->fallback = crypto_alloc_shash(alg_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(tctx->fallback)) {
+ pr_err("omap-sham: fallback driver '%s' "
+ "could not be loaded.\n", alg_name);
+ return PTR_ERR(tctx->fallback);
+ }
+
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct omap_sham_reqctx) + BUFLEN);
+
+ if (alg_base) {
+ struct omap_sham_hmac_ctx *bctx = tctx->base;
+ tctx->flags |= BIT(FLAGS_HMAC);
+ bctx->shash = crypto_alloc_shash(alg_base, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(bctx->shash)) {
+ pr_err("omap-sham: base driver '%s' "
+ "could not be loaded.\n", alg_base);
+ crypto_free_shash(tctx->fallback);
+ return PTR_ERR(bctx->shash);
+ }
+
+ }
+
+ return 0;
+}
+
+static int omap_sham_cra_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, NULL);
+}
+
+static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "sha1");
+}
+
+static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
+{
+ return omap_sham_cra_init_alg(tfm, "md5");
+}
+
+static void omap_sham_cra_exit(struct crypto_tfm *tfm)
+{
+ struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_shash(tctx->fallback);
+ tctx->fallback = NULL;
+
+ if (tctx->flags & BIT(FLAGS_HMAC)) {
+ struct omap_sham_hmac_ctx *bctx = tctx->base;
+ crypto_free_shash(bctx->shash);
+ }
+}
+
+static struct ahash_alg algs[] = {
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "omap-sha1",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .halg.digestsize = MD5_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "md5",
+ .cra_driver_name = "omap-md5",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "omap-hmac-sha1",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_sha1_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+},
+{
+ .init = omap_sham_init,
+ .update = omap_sham_update,
+ .final = omap_sham_final,
+ .finup = omap_sham_finup,
+ .digest = omap_sham_digest,
+ .setkey = omap_sham_setkey,
+ .halg.digestsize = MD5_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(md5)",
+ .cra_driver_name = "omap-hmac-md5",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct omap_sham_ctx) +
+ sizeof(struct omap_sham_hmac_ctx),
+ .cra_alignmask = OMAP_ALIGN_MASK,
+ .cra_module = THIS_MODULE,
+ .cra_init = omap_sham_cra_md5_init,
+ .cra_exit = omap_sham_cra_exit,
+ }
+}
+};
+
+static void omap_sham_done_task(unsigned long data)
+{
+ struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
+ int err = 0;
+
+ if (!test_bit(FLAGS_BUSY, &dd->flags)) {
+ omap_sham_handle_queue(dd, NULL);
+ return;
+ }
+
+ if (test_bit(FLAGS_CPU, &dd->flags)) {
+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
+ goto finish;
+ } else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
+ if (test_and_clear_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
+ omap_sham_update_dma_stop(dd);
+ if (dd->err) {
+ err = dd->err;
+ goto finish;
+ }
+ }
+ if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
+ /* hash or semi-hash ready */
+ clear_bit(FLAGS_DMA_READY, &dd->flags);
+ err = omap_sham_update_dma_start(dd);
+ if (err != -EINPROGRESS)
+ goto finish;
+ }
+ }
+
+ return;
+
+finish:
+ dev_dbg(dd->dev, "update done: err: %d\n", err);
+ /* finish curent request */
+ omap_sham_finish_req(dd->req, err);
+}
+
+static irqreturn_t omap_sham_irq(int irq, void *dev_id)
+{
+ struct omap_sham_dev *dd = dev_id;
+
+ if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
+ /* final -> allow device to go to power-saving mode */
+ omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
+
+ omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
+ SHA_REG_CTRL_OUTPUT_READY);
+ omap_sham_read(dd, SHA_REG_CTRL);
+
+ if (!test_bit(FLAGS_BUSY, &dd->flags)) {
+ dev_warn(dd->dev, "Interrupt when no active requests.\n");
+ return IRQ_HANDLED;
+ }
+
+ set_bit(FLAGS_OUTPUT_READY, &dd->flags);
+ tasklet_schedule(&dd->done_task);
+
+ return IRQ_HANDLED;
+}
+
+static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
+{
+ struct omap_sham_dev *dd = data;
+
+ if (ch_status != OMAP_DMA_BLOCK_IRQ) {
+ pr_err("omap-sham DMA error status: 0x%hx\n", ch_status);
+ dd->err = -EIO;
+ clear_bit(FLAGS_INIT, &dd->flags);/* request to re-initialize */
+ }
+
+ set_bit(FLAGS_DMA_READY, &dd->flags);
+ tasklet_schedule(&dd->done_task);
+}
+
+static int omap_sham_dma_init(struct omap_sham_dev *dd)
+{
+ int err;
+
+ dd->dma_lch = -1;
+
+ err = omap_request_dma(dd->dma, dev_name(dd->dev),
+ omap_sham_dma_callback, dd, &dd->dma_lch);
+ if (err) {
+ dev_err(dd->dev, "Unable to request DMA channel\n");
+ return err;
+ }
+
+ return 0;
+}
+
+static void omap_sham_dma_cleanup(struct omap_sham_dev *dd)
+{
+ if (dd->dma_lch >= 0) {
+ omap_free_dma(dd->dma_lch);
+ dd->dma_lch = -1;
+ }
+}
+
+static int __devinit omap_sham_probe(struct platform_device *pdev)
+{
+ struct omap_sham_dev *dd;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+ int err, i, j;
+
+ dd = kzalloc(sizeof(struct omap_sham_dev), GFP_KERNEL);
+ if (dd == NULL) {
+ dev_err(dev, "unable to alloc data struct.\n");
+ err = -ENOMEM;
+ goto data_err;
+ }
+ dd->dev = dev;
+ platform_set_drvdata(pdev, dd);
+
+ INIT_LIST_HEAD(&dd->list);
+ spin_lock_init(&dd->lock);
+ tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
+ crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
+
+ dd->irq = -1;
+
+ /* Get the base address */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ dev_err(dev, "no MEM resource info\n");
+ err = -ENODEV;
+ goto res_err;
+ }
+ dd->phys_base = res->start;
+
+ /* Get the DMA */
+ res = platform_get_resource(pdev, IORESOURCE_DMA, 0);
+ if (!res) {
+ dev_err(dev, "no DMA resource info\n");
+ err = -ENODEV;
+ goto res_err;
+ }
+ dd->dma = res->start;
+
+ /* Get the IRQ */
+ dd->irq = platform_get_irq(pdev, 0);
+ if (dd->irq < 0) {
+ dev_err(dev, "no IRQ resource info\n");
+ err = dd->irq;
+ goto res_err;
+ }
+
+ err = request_irq(dd->irq, omap_sham_irq,
+ IRQF_TRIGGER_LOW, dev_name(dev), dd);
+ if (err) {
+ dev_err(dev, "unable to request irq.\n");
+ goto res_err;
+ }
+
+ err = omap_sham_dma_init(dd);
+ if (err)
+ goto dma_err;
+
+ /* Initializing the clock */
+ dd->iclk = clk_get(dev, "ick");
+ if (IS_ERR(dd->iclk)) {
+ dev_err(dev, "clock intialization failed.\n");
+ err = PTR_ERR(dd->iclk);
+ goto clk_err;
+ }
+
+ dd->io_base = ioremap(dd->phys_base, SZ_4K);
+ if (!dd->io_base) {
+ dev_err(dev, "can't ioremap\n");
+ err = -ENOMEM;
+ goto io_err;
+ }
+
+ clk_enable(dd->iclk);
+ dev_info(dev, "hw accel on OMAP rev %u.%u\n",
+ (omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MAJOR) >> 4,
+ omap_sham_read(dd, SHA_REG_REV) & SHA_REG_REV_MINOR);
+ clk_disable(dd->iclk);
+
+ spin_lock(&sham.lock);
+ list_add_tail(&dd->list, &sham.dev_list);
+ spin_unlock(&sham.lock);
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++) {
+ err = crypto_register_ahash(&algs[i]);
+ if (err)
+ goto err_algs;
+ }
+
+ return 0;
+
+err_algs:
+ for (j = 0; j < i; j++)
+ crypto_unregister_ahash(&algs[j]);
+ iounmap(dd->io_base);
+io_err:
+ clk_put(dd->iclk);
+clk_err:
+ omap_sham_dma_cleanup(dd);
+dma_err:
+ if (dd->irq >= 0)
+ free_irq(dd->irq, dd);
+res_err:
+ kfree(dd);
+ dd = NULL;
+data_err:
+ dev_err(dev, "initialization failed.\n");
+
+ return err;
+}
+
+static int __devexit omap_sham_remove(struct platform_device *pdev)
+{
+ static struct omap_sham_dev *dd;
+ int i;
+
+ dd = platform_get_drvdata(pdev);
+ if (!dd)
+ return -ENODEV;
+ spin_lock(&sham.lock);
+ list_del(&dd->list);
+ spin_unlock(&sham.lock);
+ for (i = 0; i < ARRAY_SIZE(algs); i++)
+ crypto_unregister_ahash(&algs[i]);
+ tasklet_kill(&dd->done_task);
+ iounmap(dd->io_base);
+ clk_put(dd->iclk);
+ omap_sham_dma_cleanup(dd);
+ if (dd->irq >= 0)
+ free_irq(dd->irq, dd);
+ kfree(dd);
+ dd = NULL;
+
+ return 0;
+}
+
+static struct platform_driver omap_sham_driver = {
+ .probe = omap_sham_probe,
+ .remove = omap_sham_remove,
+ .driver = {
+ .name = "omap-sham",
+ .owner = THIS_MODULE,
+ },
+};
+
+static int __init omap_sham_mod_init(void)
+{
+ pr_info("loading %s driver\n", "omap-sham");
+
+ if (!cpu_class_is_omap2() ||
+ (omap_type() != OMAP2_DEVICE_TYPE_SEC &&
+ omap_type() != OMAP2_DEVICE_TYPE_EMU)) {
+ pr_err("Unsupported cpu\n");
+ return -ENODEV;
+ }
+
+ return platform_driver_register(&omap_sham_driver);
+}
+
+static void __exit omap_sham_mod_exit(void)
+{
+ platform_driver_unregister(&omap_sham_driver);
+}
+
+module_init(omap_sham_mod_init);
+module_exit(omap_sham_mod_exit);
+
+MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Dmitry Kasatkin");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-aes.c b/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-aes.c
new file mode 100644
index 0000000..37b2e94
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-aes.c
@@ -0,0 +1,569 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for VIA PadLock hardware crypto engine.
+ *
+ * Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
+ *
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/padlock.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/percpu.h>
+#include <linux/smp.h>
+#include <linux/slab.h>
+#include <asm/cpu_device_id.h>
+#include <asm/byteorder.h>
+#include <asm/processor.h>
+#include <asm/i387.h>
+
+/*
+ * Number of data blocks actually fetched for each xcrypt insn.
+ * Processors with prefetch errata will fetch extra blocks.
+ */
+static unsigned int ecb_fetch_blocks = 2;
+#define MAX_ECB_FETCH_BLOCKS (8)
+#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
+
+static unsigned int cbc_fetch_blocks = 1;
+#define MAX_CBC_FETCH_BLOCKS (4)
+#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
+
+/* Control word. */
+struct cword {
+ unsigned int __attribute__ ((__packed__))
+ rounds:4,
+ algo:3,
+ keygen:1,
+ interm:1,
+ encdec:1,
+ ksize:2;
+} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+
+/* Whenever making any changes to the following
+ * structure *make sure* you keep E, d_data
+ * and cword aligned on 16 Bytes boundaries and
+ * the Hardware can access 16 * 16 bytes of E and d_data
+ * (only the first 15 * 16 bytes matter but the HW reads
+ * more).
+ */
+struct aes_ctx {
+ u32 E[AES_MAX_KEYLENGTH_U32]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+ u32 d_data[AES_MAX_KEYLENGTH_U32]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+ struct {
+ struct cword encrypt;
+ struct cword decrypt;
+ } cword;
+ u32 *D;
+};
+
+static DEFINE_PER_CPU(struct cword *, paes_last_cword);
+
+/* Tells whether the ACE is capable to generate
+ the extended key for a given key_len. */
+static inline int
+aes_hw_extkey_available(uint8_t key_len)
+{
+ /* TODO: We should check the actual CPU model/stepping
+ as it's possible that the capability will be
+ added in the next CPU revisions. */
+ if (key_len == 16)
+ return 1;
+ return 0;
+}
+
+static inline struct aes_ctx *aes_ctx_common(void *ctx)
+{
+ unsigned long addr = (unsigned long)ctx;
+ unsigned long align = PADLOCK_ALIGNMENT;
+
+ if (align <= crypto_tfm_ctx_alignment())
+ align = 1;
+ return (struct aes_ctx *)ALIGN(addr, align);
+}
+
+static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
+{
+ return aes_ctx_common(crypto_tfm_ctx(tfm));
+}
+
+static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
+{
+ return aes_ctx_common(crypto_blkcipher_ctx(tfm));
+}
+
+static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ const __le32 *key = (const __le32 *)in_key;
+ u32 *flags = &tfm->crt_flags;
+ struct crypto_aes_ctx gen_aes;
+ int cpu;
+
+ if (key_len % 8) {
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+
+ /*
+ * If the hardware is capable of generating the extended key
+ * itself we must supply the plain key for both encryption
+ * and decryption.
+ */
+ ctx->D = ctx->E;
+
+ ctx->E[0] = le32_to_cpu(key[0]);
+ ctx->E[1] = le32_to_cpu(key[1]);
+ ctx->E[2] = le32_to_cpu(key[2]);
+ ctx->E[3] = le32_to_cpu(key[3]);
+
+ /* Prepare control words. */
+ memset(&ctx->cword, 0, sizeof(ctx->cword));
+
+ ctx->cword.decrypt.encdec = 1;
+ ctx->cword.encrypt.rounds = 10 + (key_len - 16) / 4;
+ ctx->cword.decrypt.rounds = ctx->cword.encrypt.rounds;
+ ctx->cword.encrypt.ksize = (key_len - 16) / 8;
+ ctx->cword.decrypt.ksize = ctx->cword.encrypt.ksize;
+
+ /* Don't generate extended keys if the hardware can do it. */
+ if (aes_hw_extkey_available(key_len))
+ goto ok;
+
+ ctx->D = ctx->d_data;
+ ctx->cword.encrypt.keygen = 1;
+ ctx->cword.decrypt.keygen = 1;
+
+ if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+
+ memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
+ memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
+
+ok:
+ for_each_online_cpu(cpu)
+ if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
+ &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
+ per_cpu(paes_last_cword, cpu) = NULL;
+
+ return 0;
+}
+
+/* ====== Encryption/decryption routines ====== */
+
+/* These are the real call to PadLock. */
+static inline void padlock_reset_key(struct cword *cword)
+{
+ int cpu = raw_smp_processor_id();
+
+ if (cword != per_cpu(paes_last_cword, cpu))
+#ifndef CONFIG_X86_64
+ asm volatile ("pushfl; popfl");
+#else
+ asm volatile ("pushfq; popfq");
+#endif
+}
+
+static inline void padlock_store_cword(struct cword *cword)
+{
+ per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
+}
+
+/*
+ * While the padlock instructions don't use FP/SSE registers, they
+ * generate a spurious DNA fault when cr0.ts is '1'. These instructions
+ * should be used only inside the irq_ts_save/restore() context
+ */
+
+static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
+ struct cword *control_word, int count)
+{
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(count));
+}
+
+static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, struct cword *control_word, int count)
+{
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count));
+ return iv;
+}
+
+static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ struct cword *cword, int count)
+{
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
+ u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ rep_xcrypt_ecb(tmp, out, key, cword, count);
+}
+
+static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
+{
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
+ u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
+}
+
+static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
+ struct cword *cword, int count)
+{
+ /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
+ * We could avoid some copying here but it's probably not worth it.
+ */
+ if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) {
+ ecb_crypt_copy(in, out, key, cword, count);
+ return;
+ }
+
+ rep_xcrypt_ecb(in, out, key, cword, count);
+}
+
+static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
+{
+ /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
+ if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE))
+ return cbc_crypt_copy(in, out, key, iv, cword, count);
+
+ return rep_xcrypt_cbc(in, out, key, iv, cword, count);
+}
+
+static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
+ void *control_word, u32 count)
+{
+ u32 initial = count & (ecb_fetch_blocks - 1);
+
+ if (count < ecb_fetch_blocks) {
+ ecb_crypt(input, output, key, control_word, count);
+ return;
+ }
+
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(initial));
+
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(count - initial));
+}
+
+static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, void *control_word, u32 count)
+{
+ u32 initial = count & (cbc_fetch_blocks - 1);
+
+ if (count < cbc_fetch_blocks)
+ return cbc_crypt(input, output, key, iv, control_word, count);
+
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (initial));
+
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count-initial));
+ return iv;
+}
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+ ts_state = irq_ts_save();
+ ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
+ irq_ts_restore(ts_state);
+ padlock_store_cword(&ctx->cword.encrypt);
+}
+
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+ ts_state = irq_ts_save();
+ ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
+ irq_ts_restore(ts_state);
+ padlock_store_cword(&ctx->cword.encrypt);
+}
+
+static struct crypto_alg aes_alg = {
+ .cra_name = "aes",
+ .cra_driver_name = "aes-padlock",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
+ .cra_u = {
+ .cipher = {
+ .cia_min_keysize = AES_MIN_KEY_SIZE,
+ .cia_max_keysize = AES_MAX_KEY_SIZE,
+ .cia_setkey = aes_set_key,
+ .cia_encrypt = aes_encrypt,
+ .cia_decrypt = aes_decrypt,
+ }
+ }
+};
+
+static int ecb_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->E, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static int ecb_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.decrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static struct crypto_alg ecb_aes_alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = ecb_aes_encrypt,
+ .decrypt = ecb_aes_decrypt,
+ }
+ }
+};
+
+static int cbc_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
+ walk.dst.virt.addr, ctx->E,
+ walk.iv, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ memcpy(walk.iv, iv, AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.decrypt);
+
+ return err;
+}
+
+static int cbc_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, walk.iv, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static struct crypto_alg cbc_aes_alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = cbc_aes_encrypt,
+ .decrypt = cbc_aes_decrypt,
+ }
+ }
+};
+
+static struct x86_cpu_id padlock_cpu_id[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_XCRYPT),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, padlock_cpu_id);
+
+static int __init padlock_init(void)
+{
+ int ret;
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ if (!x86_match_cpu(padlock_cpu_id))
+ return -ENODEV;
+
+ if (!cpu_has_xcrypt_enabled) {
+ printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
+ return -ENODEV;
+ }
+
+ if ((ret = crypto_register_alg(&aes_alg)))
+ goto aes_err;
+
+ if ((ret = crypto_register_alg(&ecb_aes_alg)))
+ goto ecb_aes_err;
+
+ if ((ret = crypto_register_alg(&cbc_aes_alg)))
+ goto cbc_aes_err;
+
+ printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+
+ if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
+ ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
+ cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
+ printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
+ }
+
+out:
+ return ret;
+
+cbc_aes_err:
+ crypto_unregister_alg(&ecb_aes_alg);
+ecb_aes_err:
+ crypto_unregister_alg(&aes_alg);
+aes_err:
+ printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
+ goto out;
+}
+
+static void __exit padlock_fini(void)
+{
+ crypto_unregister_alg(&cbc_aes_alg);
+ crypto_unregister_alg(&ecb_aes_alg);
+ crypto_unregister_alg(&aes_alg);
+}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("aes");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-sha.c b/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-sha.c
new file mode 100644
index 0000000..9266c0e
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/padlock-sha.c
@@ -0,0 +1,599 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for VIA PadLock hardware crypto engine.
+ *
+ * Copyright (c) 2006 Michal Ludvig <michal@logix.cz>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <crypto/internal/hash.h>
+#include <crypto/padlock.h>
+#include <crypto/sha.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include <asm/cpu_device_id.h>
+#include <asm/i387.h>
+
+struct padlock_sha_desc {
+ struct shash_desc fallback;
+};
+
+struct padlock_sha_ctx {
+ struct crypto_shash *fallback;
+};
+
+static int padlock_sha_init(struct shash_desc *desc)
+{
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+ struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
+
+ dctx->fallback.tfm = ctx->fallback;
+ dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ return crypto_shash_init(&dctx->fallback);
+}
+
+static int padlock_sha_update(struct shash_desc *desc,
+ const u8 *data, unsigned int length)
+{
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+
+ dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ return crypto_shash_update(&dctx->fallback, data, length);
+}
+
+static int padlock_sha_export(struct shash_desc *desc, void *out)
+{
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+
+ return crypto_shash_export(&dctx->fallback, out);
+}
+
+static int padlock_sha_import(struct shash_desc *desc, const void *in)
+{
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+ struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
+
+ dctx->fallback.tfm = ctx->fallback;
+ dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ return crypto_shash_import(&dctx->fallback, in);
+}
+
+static inline void padlock_output_block(uint32_t *src,
+ uint32_t *dst, size_t count)
+{
+ while (count--)
+ *dst++ = swab32(*src++);
+}
+
+static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in,
+ unsigned int count, u8 *out)
+{
+ /* We can't store directly to *out as it may be unaligned. */
+ /* BTW Don't reduce the buffer size below 128 Bytes!
+ * PadLock microcode needs it that big. */
+ char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
+ ((aligned(STACK_ALIGN)));
+ char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+ struct sha1_state state;
+ unsigned int space;
+ unsigned int leftover;
+ int ts_state;
+ int err;
+
+ dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ err = crypto_shash_export(&dctx->fallback, &state);
+ if (err)
+ goto out;
+
+ if (state.count + count > ULONG_MAX)
+ return crypto_shash_finup(&dctx->fallback, in, count, out);
+
+ leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1;
+ space = SHA1_BLOCK_SIZE - leftover;
+ if (space) {
+ if (count > space) {
+ err = crypto_shash_update(&dctx->fallback, in, space) ?:
+ crypto_shash_export(&dctx->fallback, &state);
+ if (err)
+ goto out;
+ count -= space;
+ in += space;
+ } else {
+ memcpy(state.buffer + leftover, in, count);
+ in = state.buffer;
+ count += leftover;
+ state.count &= ~(SHA1_BLOCK_SIZE - 1);
+ }
+ }
+
+ memcpy(result, &state.state, SHA1_DIGEST_SIZE);
+
+ /* prevent taking the spurious DNA fault with padlock. */
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
+ : \
+ : "c"((unsigned long)state.count + count), \
+ "a"((unsigned long)state.count), \
+ "S"(in), "D"(result));
+ irq_ts_restore(ts_state);
+
+ padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);
+
+out:
+ return err;
+}
+
+static int padlock_sha1_final(struct shash_desc *desc, u8 *out)
+{
+ u8 buf[4];
+
+ return padlock_sha1_finup(desc, buf, 0, out);
+}
+
+static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in,
+ unsigned int count, u8 *out)
+{
+ /* We can't store directly to *out as it may be unaligned. */
+ /* BTW Don't reduce the buffer size below 128 Bytes!
+ * PadLock microcode needs it that big. */
+ char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
+ ((aligned(STACK_ALIGN)));
+ char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+ struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
+ struct sha256_state state;
+ unsigned int space;
+ unsigned int leftover;
+ int ts_state;
+ int err;
+
+ dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP;
+ err = crypto_shash_export(&dctx->fallback, &state);
+ if (err)
+ goto out;
+
+ if (state.count + count > ULONG_MAX)
+ return crypto_shash_finup(&dctx->fallback, in, count, out);
+
+ leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1;
+ space = SHA256_BLOCK_SIZE - leftover;
+ if (space) {
+ if (count > space) {
+ err = crypto_shash_update(&dctx->fallback, in, space) ?:
+ crypto_shash_export(&dctx->fallback, &state);
+ if (err)
+ goto out;
+ count -= space;
+ in += space;
+ } else {
+ memcpy(state.buf + leftover, in, count);
+ in = state.buf;
+ count += leftover;
+ state.count &= ~(SHA1_BLOCK_SIZE - 1);
+ }
+ }
+
+ memcpy(result, &state.state, SHA256_DIGEST_SIZE);
+
+ /* prevent taking the spurious DNA fault with padlock. */
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
+ : \
+ : "c"((unsigned long)state.count + count), \
+ "a"((unsigned long)state.count), \
+ "S"(in), "D"(result));
+ irq_ts_restore(ts_state);
+
+ padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);
+
+out:
+ return err;
+}
+
+static int padlock_sha256_final(struct shash_desc *desc, u8 *out)
+{
+ u8 buf[4];
+
+ return padlock_sha256_finup(desc, buf, 0, out);
+}
+
+static int padlock_cra_init(struct crypto_tfm *tfm)
+{
+ struct crypto_shash *hash = __crypto_shash_cast(tfm);
+ const char *fallback_driver_name = tfm->__crt_alg->cra_name;
+ struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_shash *fallback_tfm;
+ int err = -ENOMEM;
+
+ /* Allocate a fallback and abort if it failed. */
+ fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(fallback_tfm)) {
+ printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
+ fallback_driver_name);
+ err = PTR_ERR(fallback_tfm);
+ goto out;
+ }
+
+ ctx->fallback = fallback_tfm;
+ hash->descsize += crypto_shash_descsize(fallback_tfm);
+ return 0;
+
+out:
+ return err;
+}
+
+static void padlock_cra_exit(struct crypto_tfm *tfm)
+{
+ struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ crypto_free_shash(ctx->fallback);
+}
+
+static struct shash_alg sha1_alg = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .init = padlock_sha_init,
+ .update = padlock_sha_update,
+ .finup = padlock_sha1_finup,
+ .final = padlock_sha1_final,
+ .export = padlock_sha_export,
+ .import = padlock_sha_import,
+ .descsize = sizeof(struct padlock_sha_desc),
+ .statesize = sizeof(struct sha1_state),
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-padlock",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct padlock_sha_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = padlock_cra_init,
+ .cra_exit = padlock_cra_exit,
+ }
+};
+
+static struct shash_alg sha256_alg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .init = padlock_sha_init,
+ .update = padlock_sha_update,
+ .finup = padlock_sha256_finup,
+ .final = padlock_sha256_final,
+ .export = padlock_sha_export,
+ .import = padlock_sha_import,
+ .descsize = sizeof(struct padlock_sha_desc),
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-padlock",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct padlock_sha_ctx),
+ .cra_module = THIS_MODULE,
+ .cra_init = padlock_cra_init,
+ .cra_exit = padlock_cra_exit,
+ }
+};
+
+/* Add two shash_alg instance for hardware-implemented *
+* multiple-parts hash supported by VIA Nano Processor.*/
+static int padlock_sha1_init_nano(struct shash_desc *desc)
+{
+ struct sha1_state *sctx = shash_desc_ctx(desc);
+
+ *sctx = (struct sha1_state){
+ .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
+ };
+
+ return 0;
+}
+
+static int padlock_sha1_update_nano(struct shash_desc *desc,
+ const u8 *data, unsigned int len)
+{
+ struct sha1_state *sctx = shash_desc_ctx(desc);
+ unsigned int partial, done;
+ const u8 *src;
+ /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
+ u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
+ ((aligned(STACK_ALIGN)));
+ u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+ int ts_state;
+
+ partial = sctx->count & 0x3f;
+ sctx->count += len;
+ done = 0;
+ src = data;
+ memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE);
+
+ if ((partial + len) >= SHA1_BLOCK_SIZE) {
+
+ /* Append the bytes in state's buffer to a block to handle */
+ if (partial) {
+ done = -partial;
+ memcpy(sctx->buffer + partial, data,
+ done + SHA1_BLOCK_SIZE);
+ src = sctx->buffer;
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
+ : "+S"(src), "+D"(dst) \
+ : "a"((long)-1), "c"((unsigned long)1));
+ irq_ts_restore(ts_state);
+ done += SHA1_BLOCK_SIZE;
+ src = data + done;
+ }
+
+ /* Process the left bytes from the input data */
+ if (len - done >= SHA1_BLOCK_SIZE) {
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
+ : "+S"(src), "+D"(dst)
+ : "a"((long)-1),
+ "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE)));
+ irq_ts_restore(ts_state);
+ done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE);
+ src = data + done;
+ }
+ partial = 0;
+ }
+ memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE);
+ memcpy(sctx->buffer + partial, src, len - done);
+
+ return 0;
+}
+
+static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out)
+{
+ struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc);
+ unsigned int partial, padlen;
+ __be64 bits;
+ static const u8 padding[64] = { 0x80, };
+
+ bits = cpu_to_be64(state->count << 3);
+
+ /* Pad out to 56 mod 64 */
+ partial = state->count & 0x3f;
+ padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
+ padlock_sha1_update_nano(desc, padding, padlen);
+
+ /* Append length field bytes */
+ padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits));
+
+ /* Swap to output */
+ padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5);
+
+ return 0;
+}
+
+static int padlock_sha256_init_nano(struct shash_desc *desc)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+
+ *sctx = (struct sha256_state){
+ .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \
+ SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7},
+ };
+
+ return 0;
+}
+
+static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data,
+ unsigned int len)
+{
+ struct sha256_state *sctx = shash_desc_ctx(desc);
+ unsigned int partial, done;
+ const u8 *src;
+ /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
+ u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
+ ((aligned(STACK_ALIGN)));
+ u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+ int ts_state;
+
+ partial = sctx->count & 0x3f;
+ sctx->count += len;
+ done = 0;
+ src = data;
+ memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE);
+
+ if ((partial + len) >= SHA256_BLOCK_SIZE) {
+
+ /* Append the bytes in state's buffer to a block to handle */
+ if (partial) {
+ done = -partial;
+ memcpy(sctx->buf + partial, data,
+ done + SHA256_BLOCK_SIZE);
+ src = sctx->buf;
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
+ : "+S"(src), "+D"(dst)
+ : "a"((long)-1), "c"((unsigned long)1));
+ irq_ts_restore(ts_state);
+ done += SHA256_BLOCK_SIZE;
+ src = data + done;
+ }
+
+ /* Process the left bytes from input data*/
+ if (len - done >= SHA256_BLOCK_SIZE) {
+ ts_state = irq_ts_save();
+ asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
+ : "+S"(src), "+D"(dst)
+ : "a"((long)-1),
+ "c"((unsigned long)((len - done) / 64)));
+ irq_ts_restore(ts_state);
+ done += ((len - done) - (len - done) % 64);
+ src = data + done;
+ }
+ partial = 0;
+ }
+ memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE);
+ memcpy(sctx->buf + partial, src, len - done);
+
+ return 0;
+}
+
+static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out)
+{
+ struct sha256_state *state =
+ (struct sha256_state *)shash_desc_ctx(desc);
+ unsigned int partial, padlen;
+ __be64 bits;
+ static const u8 padding[64] = { 0x80, };
+
+ bits = cpu_to_be64(state->count << 3);
+
+ /* Pad out to 56 mod 64 */
+ partial = state->count & 0x3f;
+ padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
+ padlock_sha256_update_nano(desc, padding, padlen);
+
+ /* Append length field bytes */
+ padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits));
+
+ /* Swap to output */
+ padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8);
+
+ return 0;
+}
+
+static int padlock_sha_export_nano(struct shash_desc *desc,
+ void *out)
+{
+ int statesize = crypto_shash_statesize(desc->tfm);
+ void *sctx = shash_desc_ctx(desc);
+
+ memcpy(out, sctx, statesize);
+ return 0;
+}
+
+static int padlock_sha_import_nano(struct shash_desc *desc,
+ const void *in)
+{
+ int statesize = crypto_shash_statesize(desc->tfm);
+ void *sctx = shash_desc_ctx(desc);
+
+ memcpy(sctx, in, statesize);
+ return 0;
+}
+
+static struct shash_alg sha1_alg_nano = {
+ .digestsize = SHA1_DIGEST_SIZE,
+ .init = padlock_sha1_init_nano,
+ .update = padlock_sha1_update_nano,
+ .final = padlock_sha1_final_nano,
+ .export = padlock_sha_export_nano,
+ .import = padlock_sha_import_nano,
+ .descsize = sizeof(struct sha1_state),
+ .statesize = sizeof(struct sha1_state),
+ .base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-padlock-nano",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static struct shash_alg sha256_alg_nano = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .init = padlock_sha256_init_nano,
+ .update = padlock_sha256_update_nano,
+ .final = padlock_sha256_final_nano,
+ .export = padlock_sha_export_nano,
+ .import = padlock_sha_import_nano,
+ .descsize = sizeof(struct sha256_state),
+ .statesize = sizeof(struct sha256_state),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-padlock-nano",
+ .cra_priority = PADLOCK_CRA_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_SHASH,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ }
+};
+
+static struct x86_cpu_id padlock_sha_ids[] = {
+ X86_FEATURE_MATCH(X86_FEATURE_PHE),
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids);
+
+static int __init padlock_init(void)
+{
+ int rc = -ENODEV;
+ struct cpuinfo_x86 *c = &cpu_data(0);
+ struct shash_alg *sha1;
+ struct shash_alg *sha256;
+
+ if (!x86_match_cpu(padlock_sha_ids) || !cpu_has_phe_enabled)
+ return -ENODEV;
+
+ /* Register the newly added algorithm module if on *
+ * VIA Nano processor, or else just do as before */
+ if (c->x86_model < 0x0f) {
+ sha1 = &sha1_alg;
+ sha256 = &sha256_alg;
+ } else {
+ sha1 = &sha1_alg_nano;
+ sha256 = &sha256_alg_nano;
+ }
+
+ rc = crypto_register_shash(sha1);
+ if (rc)
+ goto out;
+
+ rc = crypto_register_shash(sha256);
+ if (rc)
+ goto out_unreg1;
+
+ printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
+
+ return 0;
+
+out_unreg1:
+ crypto_unregister_shash(sha1);
+
+out:
+ printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
+ return rc;
+}
+
+static void __exit padlock_fini(void)
+{
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ if (c->x86_model >= 0x0f) {
+ crypto_unregister_shash(&sha1_alg_nano);
+ crypto_unregister_shash(&sha256_alg_nano);
+ } else {
+ crypto_unregister_shash(&sha1_alg);
+ crypto_unregister_shash(&sha256_alg);
+ }
+}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("sha1-all");
+MODULE_ALIAS("sha256-all");
+MODULE_ALIAS("sha1-padlock");
+MODULE_ALIAS("sha256-padlock");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto.c b/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto.c
new file mode 100644
index 0000000..410a03c
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto.c
@@ -0,0 +1,1884 @@
+/*
+ * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#include <crypto/aead.h>
+#include <crypto/aes.h>
+#include <crypto/algapi.h>
+#include <crypto/authenc.h>
+#include <crypto/des.h>
+#include <crypto/md5.h>
+#include <crypto/sha.h>
+#include <crypto/internal/skcipher.h>
+#include <linux/clk.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmapool.h>
+#include <linux/err.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/rtnetlink.h>
+#include <linux/scatterlist.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
+#include <linux/timer.h>
+
+#include "picoxcell_crypto_regs.h"
+
+/*
+ * The threshold for the number of entries in the CMD FIFO available before
+ * the CMD0_CNT interrupt is raised. Increasing this value will reduce the
+ * number of interrupts raised to the CPU.
+ */
+#define CMD0_IRQ_THRESHOLD 1
+
+/*
+ * The timeout period (in jiffies) for a PDU. When the the number of PDUs in
+ * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled.
+ * When there are packets in flight but lower than the threshold, we enable
+ * the timer and at expiry, attempt to remove any processed packets from the
+ * queue and if there are still packets left, schedule the timer again.
+ */
+#define PACKET_TIMEOUT 1
+
+/* The priority to register each algorithm with. */
+#define SPACC_CRYPTO_ALG_PRIORITY 10000
+
+#define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16
+#define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64
+#define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64
+#define SPACC_CRYPTO_IPSEC_MAX_CTXS 32
+#define SPACC_CRYPTO_IPSEC_FIFO_SZ 32
+#define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64
+#define SPACC_CRYPTO_L2_HASH_PG_SZ 64
+#define SPACC_CRYPTO_L2_MAX_CTXS 128
+#define SPACC_CRYPTO_L2_FIFO_SZ 128
+
+#define MAX_DDT_LEN 16
+
+/* DDT format. This must match the hardware DDT format exactly. */
+struct spacc_ddt {
+ dma_addr_t p;
+ u32 len;
+};
+
+/*
+ * Asynchronous crypto request structure.
+ *
+ * This structure defines a request that is either queued for processing or
+ * being processed.
+ */
+struct spacc_req {
+ struct list_head list;
+ struct spacc_engine *engine;
+ struct crypto_async_request *req;
+ int result;
+ bool is_encrypt;
+ unsigned ctx_id;
+ dma_addr_t src_addr, dst_addr;
+ struct spacc_ddt *src_ddt, *dst_ddt;
+ void (*complete)(struct spacc_req *req);
+
+ /* AEAD specific bits. */
+ u8 *giv;
+ size_t giv_len;
+ dma_addr_t giv_pa;
+};
+
+struct spacc_engine {
+ void __iomem *regs;
+ struct list_head pending;
+ int next_ctx;
+ spinlock_t hw_lock;
+ int in_flight;
+ struct list_head completed;
+ struct list_head in_progress;
+ struct tasklet_struct complete;
+ unsigned long fifo_sz;
+ void __iomem *cipher_ctx_base;
+ void __iomem *hash_key_base;
+ struct spacc_alg *algs;
+ unsigned num_algs;
+ struct list_head registered_algs;
+ size_t cipher_pg_sz;
+ size_t hash_pg_sz;
+ const char *name;
+ struct clk *clk;
+ struct device *dev;
+ unsigned max_ctxs;
+ struct timer_list packet_timeout;
+ unsigned stat_irq_thresh;
+ struct dma_pool *req_pool;
+};
+
+/* Algorithm type mask. */
+#define SPACC_CRYPTO_ALG_MASK 0x7
+
+/* SPACC definition of a crypto algorithm. */
+struct spacc_alg {
+ unsigned long ctrl_default;
+ unsigned long type;
+ struct crypto_alg alg;
+ struct spacc_engine *engine;
+ struct list_head entry;
+ int key_offs;
+ int iv_offs;
+};
+
+/* Generic context structure for any algorithm type. */
+struct spacc_generic_ctx {
+ struct spacc_engine *engine;
+ int flags;
+ int key_offs;
+ int iv_offs;
+};
+
+/* Block cipher context. */
+struct spacc_ablk_ctx {
+ struct spacc_generic_ctx generic;
+ u8 key[AES_MAX_KEY_SIZE];
+ u8 key_len;
+ /*
+ * The fallback cipher. If the operation can't be done in hardware,
+ * fallback to a software version.
+ */
+ struct crypto_ablkcipher *sw_cipher;
+};
+
+/* AEAD cipher context. */
+struct spacc_aead_ctx {
+ struct spacc_generic_ctx generic;
+ u8 cipher_key[AES_MAX_KEY_SIZE];
+ u8 hash_ctx[SPACC_CRYPTO_IPSEC_HASH_PG_SZ];
+ u8 cipher_key_len;
+ u8 hash_key_len;
+ struct crypto_aead *sw_cipher;
+ size_t auth_size;
+ u8 salt[AES_BLOCK_SIZE];
+};
+
+static int spacc_ablk_submit(struct spacc_req *req);
+
+static inline struct spacc_alg *to_spacc_alg(struct crypto_alg *alg)
+{
+ return alg ? container_of(alg, struct spacc_alg, alg) : NULL;
+}
+
+static inline int spacc_fifo_cmd_full(struct spacc_engine *engine)
+{
+ u32 fifo_stat = readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET);
+
+ return fifo_stat & SPA_FIFO_CMD_FULL;
+}
+
+/*
+ * Given a cipher context, and a context number, get the base address of the
+ * context page.
+ *
+ * Returns the address of the context page where the key/context may
+ * be written.
+ */
+static inline void __iomem *spacc_ctx_page_addr(struct spacc_generic_ctx *ctx,
+ unsigned indx,
+ bool is_cipher_ctx)
+{
+ return is_cipher_ctx ? ctx->engine->cipher_ctx_base +
+ (indx * ctx->engine->cipher_pg_sz) :
+ ctx->engine->hash_key_base + (indx * ctx->engine->hash_pg_sz);
+}
+
+/* The context pages can only be written with 32-bit accesses. */
+static inline void memcpy_toio32(u32 __iomem *dst, const void *src,
+ unsigned count)
+{
+ const u32 *src32 = (const u32 *) src;
+
+ while (count--)
+ writel(*src32++, dst++);
+}
+
+static void spacc_cipher_write_ctx(struct spacc_generic_ctx *ctx,
+ void __iomem *page_addr, const u8 *key,
+ size_t key_len, const u8 *iv, size_t iv_len)
+{
+ void __iomem *key_ptr = page_addr + ctx->key_offs;
+ void __iomem *iv_ptr = page_addr + ctx->iv_offs;
+
+ memcpy_toio32(key_ptr, key, key_len / 4);
+ memcpy_toio32(iv_ptr, iv, iv_len / 4);
+}
+
+/*
+ * Load a context into the engines context memory.
+ *
+ * Returns the index of the context page where the context was loaded.
+ */
+static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx,
+ const u8 *ciph_key, size_t ciph_len,
+ const u8 *iv, size_t ivlen, const u8 *hash_key,
+ size_t hash_len)
+{
+ unsigned indx = ctx->engine->next_ctx++;
+ void __iomem *ciph_page_addr, *hash_page_addr;
+
+ ciph_page_addr = spacc_ctx_page_addr(ctx, indx, 1);
+ hash_page_addr = spacc_ctx_page_addr(ctx, indx, 0);
+
+ ctx->engine->next_ctx &= ctx->engine->fifo_sz - 1;
+ spacc_cipher_write_ctx(ctx, ciph_page_addr, ciph_key, ciph_len, iv,
+ ivlen);
+ writel(ciph_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET) |
+ (1 << SPA_KEY_SZ_CIPHER_OFFSET),
+ ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
+
+ if (hash_key) {
+ memcpy_toio32(hash_page_addr, hash_key, hash_len / 4);
+ writel(hash_len | (indx << SPA_KEY_SZ_CTX_INDEX_OFFSET),
+ ctx->engine->regs + SPA_KEY_SZ_REG_OFFSET);
+ }
+
+ return indx;
+}
+
+/* Count the number of scatterlist entries in a scatterlist. */
+static int sg_count(struct scatterlist *sg_list, int nbytes)
+{
+ struct scatterlist *sg = sg_list;
+ int sg_nents = 0;
+
+ while (nbytes > 0) {
+ ++sg_nents;
+ nbytes -= sg->length;
+ sg = sg_next(sg);
+ }
+
+ return sg_nents;
+}
+
+static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len)
+{
+ ddt->p = phys;
+ ddt->len = len;
+}
+
+/*
+ * Take a crypto request and scatterlists for the data and turn them into DDTs
+ * for passing to the crypto engines. This also DMA maps the data so that the
+ * crypto engines can DMA to/from them.
+ */
+static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine,
+ struct scatterlist *payload,
+ unsigned nbytes,
+ enum dma_data_direction dir,
+ dma_addr_t *ddt_phys)
+{
+ unsigned nents, mapped_ents;
+ struct scatterlist *cur;
+ struct spacc_ddt *ddt;
+ int i;
+
+ nents = sg_count(payload, nbytes);
+ mapped_ents = dma_map_sg(engine->dev, payload, nents, dir);
+
+ if (mapped_ents + 1 > MAX_DDT_LEN)
+ goto out;
+
+ ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, ddt_phys);
+ if (!ddt)
+ goto out;
+
+ for_each_sg(payload, cur, mapped_ents, i)
+ ddt_set(&ddt[i], sg_dma_address(cur), sg_dma_len(cur));
+ ddt_set(&ddt[mapped_ents], 0, 0);
+
+ return ddt;
+
+out:
+ dma_unmap_sg(engine->dev, payload, nents, dir);
+ return NULL;
+}
+
+static int spacc_aead_make_ddts(struct spacc_req *req, u8 *giv)
+{
+ struct aead_request *areq = container_of(req->req, struct aead_request,
+ base);
+ struct spacc_engine *engine = req->engine;
+ struct spacc_ddt *src_ddt, *dst_ddt;
+ unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(areq));
+ unsigned nents = sg_count(areq->src, areq->cryptlen);
+ dma_addr_t iv_addr;
+ struct scatterlist *cur;
+ int i, dst_ents, src_ents, assoc_ents;
+ u8 *iv = giv ? giv : areq->iv;
+
+ src_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->src_addr);
+ if (!src_ddt)
+ return -ENOMEM;
+
+ dst_ddt = dma_pool_alloc(engine->req_pool, GFP_ATOMIC, &req->dst_addr);
+ if (!dst_ddt) {
+ dma_pool_free(engine->req_pool, src_ddt, req->src_addr);
+ return -ENOMEM;
+ }
+
+ req->src_ddt = src_ddt;
+ req->dst_ddt = dst_ddt;
+
+ assoc_ents = dma_map_sg(engine->dev, areq->assoc,
+ sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE);
+ if (areq->src != areq->dst) {
+ src_ents = dma_map_sg(engine->dev, areq->src, nents,
+ DMA_TO_DEVICE);
+ dst_ents = dma_map_sg(engine->dev, areq->dst, nents,
+ DMA_FROM_DEVICE);
+ } else {
+ src_ents = dma_map_sg(engine->dev, areq->src, nents,
+ DMA_BIDIRECTIONAL);
+ dst_ents = 0;
+ }
+
+ /*
+ * Map the IV/GIV. For the GIV it needs to be bidirectional as it is
+ * formed by the crypto block and sent as the ESP IV for IPSEC.
+ */
+ iv_addr = dma_map_single(engine->dev, iv, ivsize,
+ giv ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
+ req->giv_pa = iv_addr;
+
+ /*
+ * Map the associated data. For decryption we don't copy the
+ * associated data.
+ */
+ for_each_sg(areq->assoc, cur, assoc_ents, i) {
+ ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
+ if (req->is_encrypt)
+ ddt_set(dst_ddt++, sg_dma_address(cur),
+ sg_dma_len(cur));
+ }
+ ddt_set(src_ddt++, iv_addr, ivsize);
+
+ if (giv || req->is_encrypt)
+ ddt_set(dst_ddt++, iv_addr, ivsize);
+
+ /*
+ * Now map in the payload for the source and destination and terminate
+ * with the NULL pointers.
+ */
+ for_each_sg(areq->src, cur, src_ents, i) {
+ ddt_set(src_ddt++, sg_dma_address(cur), sg_dma_len(cur));
+ if (areq->src == areq->dst)
+ ddt_set(dst_ddt++, sg_dma_address(cur),
+ sg_dma_len(cur));
+ }
+
+ for_each_sg(areq->dst, cur, dst_ents, i)
+ ddt_set(dst_ddt++, sg_dma_address(cur),
+ sg_dma_len(cur));
+
+ ddt_set(src_ddt, 0, 0);
+ ddt_set(dst_ddt, 0, 0);
+
+ return 0;
+}
+
+static void spacc_aead_free_ddts(struct spacc_req *req)
+{
+ struct aead_request *areq = container_of(req->req, struct aead_request,
+ base);
+ struct spacc_alg *alg = to_spacc_alg(req->req->tfm->__crt_alg);
+ struct spacc_ablk_ctx *aead_ctx = crypto_tfm_ctx(req->req->tfm);
+ struct spacc_engine *engine = aead_ctx->generic.engine;
+ unsigned ivsize = alg->alg.cra_aead.ivsize;
+ unsigned nents = sg_count(areq->src, areq->cryptlen);
+
+ if (areq->src != areq->dst) {
+ dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE);
+ dma_unmap_sg(engine->dev, areq->dst,
+ sg_count(areq->dst, areq->cryptlen),
+ DMA_FROM_DEVICE);
+ } else
+ dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL);
+
+ dma_unmap_sg(engine->dev, areq->assoc,
+ sg_count(areq->assoc, areq->assoclen), DMA_TO_DEVICE);
+
+ dma_unmap_single(engine->dev, req->giv_pa, ivsize, DMA_BIDIRECTIONAL);
+
+ dma_pool_free(engine->req_pool, req->src_ddt, req->src_addr);
+ dma_pool_free(engine->req_pool, req->dst_ddt, req->dst_addr);
+}
+
+static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt,
+ dma_addr_t ddt_addr, struct scatterlist *payload,
+ unsigned nbytes, enum dma_data_direction dir)
+{
+ unsigned nents = sg_count(payload, nbytes);
+
+ dma_unmap_sg(req->engine->dev, payload, nents, dir);
+ dma_pool_free(req->engine->req_pool, ddt, ddt_addr);
+}
+
+/*
+ * Set key for a DES operation in an AEAD cipher. This also performs weak key
+ * checking if required.
+ */
+static int spacc_aead_des_setkey(struct crypto_aead *aead, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+ u32 tmp[DES_EXPKEY_WORDS];
+
+ if (unlikely(!des_ekey(tmp, key)) &&
+ (crypto_aead_get_flags(aead)) & CRYPTO_TFM_REQ_WEAK_KEY) {
+ tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ return -EINVAL;
+ }
+
+ memcpy(ctx->cipher_key, key, len);
+ ctx->cipher_key_len = len;
+
+ return 0;
+}
+
+/* Set the key for the AES block cipher component of the AEAD transform. */
+static int spacc_aead_aes_setkey(struct crypto_aead *aead, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ /*
+ * IPSec engine only supports 128 and 256 bit AES keys. If we get a
+ * request for any other size (192 bits) then we need to do a software
+ * fallback.
+ */
+ if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256) {
+ /*
+ * Set the fallback transform to use the same request flags as
+ * the hardware transform.
+ */
+ ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+ ctx->sw_cipher->base.crt_flags |=
+ tfm->crt_flags & CRYPTO_TFM_REQ_MASK;
+ return crypto_aead_setkey(ctx->sw_cipher, key, len);
+ }
+
+ memcpy(ctx->cipher_key, key, len);
+ ctx->cipher_key_len = len;
+
+ return 0;
+}
+
+static int spacc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg);
+ struct rtattr *rta = (void *)key;
+ struct crypto_authenc_key_param *param;
+ unsigned int authkeylen, enckeylen;
+ int err = -EINVAL;
+
+ if (!RTA_OK(rta, keylen))
+ goto badkey;
+
+ if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
+ goto badkey;
+
+ if (RTA_PAYLOAD(rta) < sizeof(*param))
+ goto badkey;
+
+ param = RTA_DATA(rta);
+ enckeylen = be32_to_cpu(param->enckeylen);
+
+ key += RTA_ALIGN(rta->rta_len);
+ keylen -= RTA_ALIGN(rta->rta_len);
+
+ if (keylen < enckeylen)
+ goto badkey;
+
+ authkeylen = keylen - enckeylen;
+
+ if (enckeylen > AES_MAX_KEY_SIZE)
+ goto badkey;
+
+ if ((alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
+ SPA_CTRL_CIPH_ALG_AES)
+ err = spacc_aead_aes_setkey(tfm, key + authkeylen, enckeylen);
+ else
+ err = spacc_aead_des_setkey(tfm, key + authkeylen, enckeylen);
+
+ if (err)
+ goto badkey;
+
+ memcpy(ctx->hash_ctx, key, authkeylen);
+ ctx->hash_key_len = authkeylen;
+
+ return 0;
+
+badkey:
+ crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+}
+
+static int spacc_aead_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
+
+ ctx->auth_size = authsize;
+
+ return 0;
+}
+
+/*
+ * Check if an AEAD request requires a fallback operation. Some requests can't
+ * be completed in hardware because the hardware may not support certain key
+ * sizes. In these cases we need to complete the request in software.
+ */
+static int spacc_aead_need_fallback(struct spacc_req *req)
+{
+ struct aead_request *aead_req;
+ struct crypto_tfm *tfm = req->req->tfm;
+ struct crypto_alg *alg = req->req->tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ aead_req = container_of(req->req, struct aead_request, base);
+ /*
+ * If we have a non-supported key-length, then we need to do a
+ * software fallback.
+ */
+ if ((spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
+ SPA_CTRL_CIPH_ALG_AES &&
+ ctx->cipher_key_len != AES_KEYSIZE_128 &&
+ ctx->cipher_key_len != AES_KEYSIZE_256)
+ return 1;
+
+ return 0;
+}
+
+static int spacc_aead_do_fallback(struct aead_request *req, unsigned alg_type,
+ bool is_encrypt)
+{
+ struct crypto_tfm *old_tfm = crypto_aead_tfm(crypto_aead_reqtfm(req));
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(old_tfm);
+ int err;
+
+ if (ctx->sw_cipher) {
+ /*
+ * Change the request to use the software fallback transform,
+ * and once the ciphering has completed, put the old transform
+ * back into the request.
+ */
+ aead_request_set_tfm(req, ctx->sw_cipher);
+ err = is_encrypt ? crypto_aead_encrypt(req) :
+ crypto_aead_decrypt(req);
+ aead_request_set_tfm(req, __crypto_aead_cast(old_tfm));
+ } else
+ err = -EINVAL;
+
+ return err;
+}
+
+static void spacc_aead_complete(struct spacc_req *req)
+{
+ spacc_aead_free_ddts(req);
+ req->req->complete(req->req, req->result);
+}
+
+static int spacc_aead_submit(struct spacc_req *req)
+{
+ struct crypto_tfm *tfm = req->req->tfm;
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_alg *alg = req->req->tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+ struct spacc_engine *engine = ctx->generic.engine;
+ u32 ctrl, proc_len, assoc_len;
+ struct aead_request *aead_req =
+ container_of(req->req, struct aead_request, base);
+
+ req->result = -EINPROGRESS;
+ req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->cipher_key,
+ ctx->cipher_key_len, aead_req->iv, alg->cra_aead.ivsize,
+ ctx->hash_ctx, ctx->hash_key_len);
+
+ /* Set the source and destination DDT pointers. */
+ writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
+ writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
+ writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
+
+ assoc_len = aead_req->assoclen;
+ proc_len = aead_req->cryptlen + assoc_len;
+
+ /*
+ * If we aren't generating an IV, then we need to include the IV in the
+ * associated data so that it is included in the hash.
+ */
+ if (!req->giv) {
+ assoc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req));
+ proc_len += crypto_aead_ivsize(crypto_aead_reqtfm(aead_req));
+ } else
+ proc_len += req->giv_len;
+
+ /*
+ * If we are decrypting, we need to take the length of the ICV out of
+ * the processing length.
+ */
+ if (!req->is_encrypt)
+ proc_len -= ctx->auth_size;
+
+ writel(proc_len, engine->regs + SPA_PROC_LEN_REG_OFFSET);
+ writel(assoc_len, engine->regs + SPA_AAD_LEN_REG_OFFSET);
+ writel(ctx->auth_size, engine->regs + SPA_ICV_LEN_REG_OFFSET);
+ writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
+ writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
+
+ ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
+ (1 << SPA_CTRL_ICV_APPEND);
+ if (req->is_encrypt)
+ ctrl |= (1 << SPA_CTRL_ENCRYPT_IDX) | (1 << SPA_CTRL_AAD_COPY);
+ else
+ ctrl |= (1 << SPA_CTRL_KEY_EXP);
+
+ mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
+
+ writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
+
+ return -EINPROGRESS;
+}
+
+static int spacc_req_submit(struct spacc_req *req);
+
+static void spacc_push(struct spacc_engine *engine)
+{
+ struct spacc_req *req;
+
+ while (!list_empty(&engine->pending) &&
+ engine->in_flight + 1 <= engine->fifo_sz) {
+
+ ++engine->in_flight;
+ req = list_first_entry(&engine->pending, struct spacc_req,
+ list);
+ list_move_tail(&req->list, &engine->in_progress);
+
+ req->result = spacc_req_submit(req);
+ }
+}
+
+/*
+ * Setup an AEAD request for processing. This will configure the engine, load
+ * the context and then start the packet processing.
+ *
+ * @giv Pointer to destination address for a generated IV. If the
+ * request does not need to generate an IV then this should be set to NULL.
+ */
+static int spacc_aead_setup(struct aead_request *req, u8 *giv,
+ unsigned alg_type, bool is_encrypt)
+{
+ struct crypto_alg *alg = req->base.tfm->__crt_alg;
+ struct spacc_engine *engine = to_spacc_alg(alg)->engine;
+ struct spacc_req *dev_req = aead_request_ctx(req);
+ int err = -EINPROGRESS;
+ unsigned long flags;
+ unsigned ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
+
+ dev_req->giv = giv;
+ dev_req->giv_len = ivsize;
+ dev_req->req = &req->base;
+ dev_req->is_encrypt = is_encrypt;
+ dev_req->result = -EBUSY;
+ dev_req->engine = engine;
+ dev_req->complete = spacc_aead_complete;
+
+ if (unlikely(spacc_aead_need_fallback(dev_req)))
+ return spacc_aead_do_fallback(req, alg_type, is_encrypt);
+
+ spacc_aead_make_ddts(dev_req, dev_req->giv);
+
+ err = -EINPROGRESS;
+ spin_lock_irqsave(&engine->hw_lock, flags);
+ if (unlikely(spacc_fifo_cmd_full(engine)) ||
+ engine->in_flight + 1 > engine->fifo_sz) {
+ if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ err = -EBUSY;
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+ goto out_free_ddts;
+ }
+ list_add_tail(&dev_req->list, &engine->pending);
+ } else {
+ list_add_tail(&dev_req->list, &engine->pending);
+ spacc_push(engine);
+ }
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+
+ goto out;
+
+out_free_ddts:
+ spacc_aead_free_ddts(dev_req);
+out:
+ return err;
+}
+
+static int spacc_aead_encrypt(struct aead_request *req)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+ struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
+
+ return spacc_aead_setup(req, NULL, alg->type, 1);
+}
+
+static int spacc_aead_givencrypt(struct aead_givcrypt_request *req)
+{
+ struct crypto_aead *tfm = aead_givcrypt_reqtfm(req);
+ struct spacc_aead_ctx *ctx = crypto_aead_ctx(tfm);
+ size_t ivsize = crypto_aead_ivsize(tfm);
+ struct spacc_alg *alg = to_spacc_alg(tfm->base.__crt_alg);
+ unsigned len;
+ __be64 seq;
+
+ memcpy(req->areq.iv, ctx->salt, ivsize);
+ len = ivsize;
+ if (ivsize > sizeof(u64)) {
+ memset(req->giv, 0, ivsize - sizeof(u64));
+ len = sizeof(u64);
+ }
+ seq = cpu_to_be64(req->seq);
+ memcpy(req->giv + ivsize - len, &seq, len);
+
+ return spacc_aead_setup(&req->areq, req->giv, alg->type, 1);
+}
+
+static int spacc_aead_decrypt(struct aead_request *req)
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_aead_tfm(aead);
+ struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
+
+ return spacc_aead_setup(req, NULL, alg->type, 0);
+}
+
+/*
+ * Initialise a new AEAD context. This is responsible for allocating the
+ * fallback cipher and initialising the context.
+ */
+static int spacc_aead_cra_init(struct crypto_tfm *tfm)
+{
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+ struct spacc_engine *engine = spacc_alg->engine;
+
+ ctx->generic.flags = spacc_alg->type;
+ ctx->generic.engine = engine;
+ ctx->sw_cipher = crypto_alloc_aead(alg->cra_name, 0,
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(ctx->sw_cipher)) {
+ dev_warn(engine->dev, "failed to allocate fallback for %s\n",
+ alg->cra_name);
+ ctx->sw_cipher = NULL;
+ }
+ ctx->generic.key_offs = spacc_alg->key_offs;
+ ctx->generic.iv_offs = spacc_alg->iv_offs;
+
+ get_random_bytes(ctx->salt, sizeof(ctx->salt));
+
+ tfm->crt_aead.reqsize = sizeof(struct spacc_req);
+
+ return 0;
+}
+
+/*
+ * Destructor for an AEAD context. This is called when the transform is freed
+ * and must free the fallback cipher.
+ */
+static void spacc_aead_cra_exit(struct crypto_tfm *tfm)
+{
+ struct spacc_aead_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (ctx->sw_cipher)
+ crypto_free_aead(ctx->sw_cipher);
+ ctx->sw_cipher = NULL;
+}
+
+/*
+ * Set the DES key for a block cipher transform. This also performs weak key
+ * checking if the transform has requested it.
+ */
+static int spacc_des_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+ u32 tmp[DES_EXPKEY_WORDS];
+
+ if (len > DES3_EDE_KEY_SIZE) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ if (unlikely(!des_ekey(tmp, key)) &&
+ (crypto_ablkcipher_get_flags(cipher) & CRYPTO_TFM_REQ_WEAK_KEY)) {
+ tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ return -EINVAL;
+ }
+
+ memcpy(ctx->key, key, len);
+ ctx->key_len = len;
+
+ return 0;
+}
+
+/*
+ * Set the key for an AES block cipher. Some key lengths are not supported in
+ * hardware so this must also check whether a fallback is needed.
+ */
+static int spacc_aes_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
+ unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err = 0;
+
+ if (len > AES_MAX_KEY_SIZE) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ /*
+ * IPSec engine only supports 128 and 256 bit AES keys. If we get a
+ * request for any other size (192 bits) then we need to do a software
+ * fallback.
+ */
+ if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 &&
+ ctx->sw_cipher) {
+ /*
+ * Set the fallback transform to use the same request flags as
+ * the hardware transform.
+ */
+ ctx->sw_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
+ ctx->sw_cipher->base.crt_flags |=
+ cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK;
+
+ err = crypto_ablkcipher_setkey(ctx->sw_cipher, key, len);
+ if (err)
+ goto sw_setkey_failed;
+ } else if (len != AES_KEYSIZE_128 && len != AES_KEYSIZE_256 &&
+ !ctx->sw_cipher)
+ err = -EINVAL;
+
+ memcpy(ctx->key, key, len);
+ ctx->key_len = len;
+
+sw_setkey_failed:
+ if (err && ctx->sw_cipher) {
+ tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
+ tfm->crt_flags |=
+ ctx->sw_cipher->base.crt_flags & CRYPTO_TFM_RES_MASK;
+ }
+
+ return err;
+}
+
+static int spacc_kasumi_f8_setkey(struct crypto_ablkcipher *cipher,
+ const u8 *key, unsigned int len)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err = 0;
+
+ if (len > AES_MAX_KEY_SIZE) {
+ crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ err = -EINVAL;
+ goto out;
+ }
+
+ memcpy(ctx->key, key, len);
+ ctx->key_len = len;
+
+out:
+ return err;
+}
+
+static int spacc_ablk_need_fallback(struct spacc_req *req)
+{
+ struct spacc_ablk_ctx *ctx;
+ struct crypto_tfm *tfm = req->req->tfm;
+ struct crypto_alg *alg = req->req->tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+
+ ctx = crypto_tfm_ctx(tfm);
+
+ return (spacc_alg->ctrl_default & SPACC_CRYPTO_ALG_MASK) ==
+ SPA_CTRL_CIPH_ALG_AES &&
+ ctx->key_len != AES_KEYSIZE_128 &&
+ ctx->key_len != AES_KEYSIZE_256;
+}
+
+static void spacc_ablk_complete(struct spacc_req *req)
+{
+ struct ablkcipher_request *ablk_req =
+ container_of(req->req, struct ablkcipher_request, base);
+
+ if (ablk_req->src != ablk_req->dst) {
+ spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src,
+ ablk_req->nbytes, DMA_TO_DEVICE);
+ spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
+ ablk_req->nbytes, DMA_FROM_DEVICE);
+ } else
+ spacc_free_ddt(req, req->dst_ddt, req->dst_addr, ablk_req->dst,
+ ablk_req->nbytes, DMA_BIDIRECTIONAL);
+
+ req->req->complete(req->req, req->result);
+}
+
+static int spacc_ablk_submit(struct spacc_req *req)
+{
+ struct crypto_tfm *tfm = req->req->tfm;
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req);
+ struct crypto_alg *alg = req->req->tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+ struct spacc_engine *engine = ctx->generic.engine;
+ u32 ctrl;
+
+ req->ctx_id = spacc_load_ctx(&ctx->generic, ctx->key,
+ ctx->key_len, ablk_req->info, alg->cra_ablkcipher.ivsize,
+ NULL, 0);
+
+ writel(req->src_addr, engine->regs + SPA_SRC_PTR_REG_OFFSET);
+ writel(req->dst_addr, engine->regs + SPA_DST_PTR_REG_OFFSET);
+ writel(0, engine->regs + SPA_OFFSET_REG_OFFSET);
+
+ writel(ablk_req->nbytes, engine->regs + SPA_PROC_LEN_REG_OFFSET);
+ writel(0, engine->regs + SPA_ICV_OFFSET_REG_OFFSET);
+ writel(0, engine->regs + SPA_AUX_INFO_REG_OFFSET);
+ writel(0, engine->regs + SPA_AAD_LEN_REG_OFFSET);
+
+ ctrl = spacc_alg->ctrl_default | (req->ctx_id << SPA_CTRL_CTX_IDX) |
+ (req->is_encrypt ? (1 << SPA_CTRL_ENCRYPT_IDX) :
+ (1 << SPA_CTRL_KEY_EXP));
+
+ mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
+
+ writel(ctrl, engine->regs + SPA_CTRL_REG_OFFSET);
+
+ return -EINPROGRESS;
+}
+
+static int spacc_ablk_do_fallback(struct ablkcipher_request *req,
+ unsigned alg_type, bool is_encrypt)
+{
+ struct crypto_tfm *old_tfm =
+ crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(old_tfm);
+ int err;
+
+ if (!ctx->sw_cipher)
+ return -EINVAL;
+
+ /*
+ * Change the request to use the software fallback transform, and once
+ * the ciphering has completed, put the old transform back into the
+ * request.
+ */
+ ablkcipher_request_set_tfm(req, ctx->sw_cipher);
+ err = is_encrypt ? crypto_ablkcipher_encrypt(req) :
+ crypto_ablkcipher_decrypt(req);
+ ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(old_tfm));
+
+ return err;
+}
+
+static int spacc_ablk_setup(struct ablkcipher_request *req, unsigned alg_type,
+ bool is_encrypt)
+{
+ struct crypto_alg *alg = req->base.tfm->__crt_alg;
+ struct spacc_engine *engine = to_spacc_alg(alg)->engine;
+ struct spacc_req *dev_req = ablkcipher_request_ctx(req);
+ unsigned long flags;
+ int err = -ENOMEM;
+
+ dev_req->req = &req->base;
+ dev_req->is_encrypt = is_encrypt;
+ dev_req->engine = engine;
+ dev_req->complete = spacc_ablk_complete;
+ dev_req->result = -EINPROGRESS;
+
+ if (unlikely(spacc_ablk_need_fallback(dev_req)))
+ return spacc_ablk_do_fallback(req, alg_type, is_encrypt);
+
+ /*
+ * Create the DDT's for the engine. If we share the same source and
+ * destination then we can optimize by reusing the DDT's.
+ */
+ if (req->src != req->dst) {
+ dev_req->src_ddt = spacc_sg_to_ddt(engine, req->src,
+ req->nbytes, DMA_TO_DEVICE, &dev_req->src_addr);
+ if (!dev_req->src_ddt)
+ goto out;
+
+ dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
+ req->nbytes, DMA_FROM_DEVICE, &dev_req->dst_addr);
+ if (!dev_req->dst_ddt)
+ goto out_free_src;
+ } else {
+ dev_req->dst_ddt = spacc_sg_to_ddt(engine, req->dst,
+ req->nbytes, DMA_BIDIRECTIONAL, &dev_req->dst_addr);
+ if (!dev_req->dst_ddt)
+ goto out;
+
+ dev_req->src_ddt = NULL;
+ dev_req->src_addr = dev_req->dst_addr;
+ }
+
+ err = -EINPROGRESS;
+ spin_lock_irqsave(&engine->hw_lock, flags);
+ /*
+ * Check if the engine will accept the operation now. If it won't then
+ * we either stick it on the end of a pending list if we can backlog,
+ * or bailout with an error if not.
+ */
+ if (unlikely(spacc_fifo_cmd_full(engine)) ||
+ engine->in_flight + 1 > engine->fifo_sz) {
+ if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
+ err = -EBUSY;
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+ goto out_free_ddts;
+ }
+ list_add_tail(&dev_req->list, &engine->pending);
+ } else {
+ list_add_tail(&dev_req->list, &engine->pending);
+ spacc_push(engine);
+ }
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+
+ goto out;
+
+out_free_ddts:
+ spacc_free_ddt(dev_req, dev_req->dst_ddt, dev_req->dst_addr, req->dst,
+ req->nbytes, req->src == req->dst ?
+ DMA_BIDIRECTIONAL : DMA_FROM_DEVICE);
+out_free_src:
+ if (req->src != req->dst)
+ spacc_free_ddt(dev_req, dev_req->src_ddt, dev_req->src_addr,
+ req->src, req->nbytes, DMA_TO_DEVICE);
+out:
+ return err;
+}
+
+static int spacc_ablk_cra_init(struct crypto_tfm *tfm)
+{
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct spacc_alg *spacc_alg = to_spacc_alg(alg);
+ struct spacc_engine *engine = spacc_alg->engine;
+
+ ctx->generic.flags = spacc_alg->type;
+ ctx->generic.engine = engine;
+ if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
+ ctx->sw_cipher = crypto_alloc_ablkcipher(alg->cra_name, 0,
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(ctx->sw_cipher)) {
+ dev_warn(engine->dev, "failed to allocate fallback for %s\n",
+ alg->cra_name);
+ ctx->sw_cipher = NULL;
+ }
+ }
+ ctx->generic.key_offs = spacc_alg->key_offs;
+ ctx->generic.iv_offs = spacc_alg->iv_offs;
+
+ tfm->crt_ablkcipher.reqsize = sizeof(struct spacc_req);
+
+ return 0;
+}
+
+static void spacc_ablk_cra_exit(struct crypto_tfm *tfm)
+{
+ struct spacc_ablk_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (ctx->sw_cipher)
+ crypto_free_ablkcipher(ctx->sw_cipher);
+ ctx->sw_cipher = NULL;
+}
+
+static int spacc_ablk_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
+
+ return spacc_ablk_setup(req, alg->type, 1);
+}
+
+static int spacc_ablk_decrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(req);
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct spacc_alg *alg = to_spacc_alg(tfm->__crt_alg);
+
+ return spacc_ablk_setup(req, alg->type, 0);
+}
+
+static inline int spacc_fifo_stat_empty(struct spacc_engine *engine)
+{
+ return readl(engine->regs + SPA_FIFO_STAT_REG_OFFSET) &
+ SPA_FIFO_STAT_EMPTY;
+}
+
+static void spacc_process_done(struct spacc_engine *engine)
+{
+ struct spacc_req *req;
+ unsigned long flags;
+
+ spin_lock_irqsave(&engine->hw_lock, flags);
+
+ while (!spacc_fifo_stat_empty(engine)) {
+ req = list_first_entry(&engine->in_progress, struct spacc_req,
+ list);
+ list_move_tail(&req->list, &engine->completed);
+ --engine->in_flight;
+
+ /* POP the status register. */
+ writel(~0, engine->regs + SPA_STAT_POP_REG_OFFSET);
+ req->result = (readl(engine->regs + SPA_STATUS_REG_OFFSET) &
+ SPA_STATUS_RES_CODE_MASK) >> SPA_STATUS_RES_CODE_OFFSET;
+
+ /*
+ * Convert the SPAcc error status into the standard POSIX error
+ * codes.
+ */
+ if (unlikely(req->result)) {
+ switch (req->result) {
+ case SPA_STATUS_ICV_FAIL:
+ req->result = -EBADMSG;
+ break;
+
+ case SPA_STATUS_MEMORY_ERROR:
+ dev_warn(engine->dev,
+ "memory error triggered\n");
+ req->result = -EFAULT;
+ break;
+
+ case SPA_STATUS_BLOCK_ERROR:
+ dev_warn(engine->dev,
+ "block error triggered\n");
+ req->result = -EIO;
+ break;
+ }
+ }
+ }
+
+ tasklet_schedule(&engine->complete);
+
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+}
+
+static irqreturn_t spacc_spacc_irq(int irq, void *dev)
+{
+ struct spacc_engine *engine = (struct spacc_engine *)dev;
+ u32 spacc_irq_stat = readl(engine->regs + SPA_IRQ_STAT_REG_OFFSET);
+
+ writel(spacc_irq_stat, engine->regs + SPA_IRQ_STAT_REG_OFFSET);
+ spacc_process_done(engine);
+
+ return IRQ_HANDLED;
+}
+
+static void spacc_packet_timeout(unsigned long data)
+{
+ struct spacc_engine *engine = (struct spacc_engine *)data;
+
+ spacc_process_done(engine);
+}
+
+static int spacc_req_submit(struct spacc_req *req)
+{
+ struct crypto_alg *alg = req->req->tfm->__crt_alg;
+
+ if (CRYPTO_ALG_TYPE_AEAD == (CRYPTO_ALG_TYPE_MASK & alg->cra_flags))
+ return spacc_aead_submit(req);
+ else
+ return spacc_ablk_submit(req);
+}
+
+static void spacc_spacc_complete(unsigned long data)
+{
+ struct spacc_engine *engine = (struct spacc_engine *)data;
+ struct spacc_req *req, *tmp;
+ unsigned long flags;
+ LIST_HEAD(completed);
+
+ spin_lock_irqsave(&engine->hw_lock, flags);
+
+ list_splice_init(&engine->completed, &completed);
+ spacc_push(engine);
+ if (engine->in_flight)
+ mod_timer(&engine->packet_timeout, jiffies + PACKET_TIMEOUT);
+
+ spin_unlock_irqrestore(&engine->hw_lock, flags);
+
+ list_for_each_entry_safe(req, tmp, &completed, list) {
+ list_del(&req->list);
+ req->complete(req);
+ }
+}
+
+#ifdef CONFIG_PM
+static int spacc_suspend(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct spacc_engine *engine = platform_get_drvdata(pdev);
+
+ /*
+ * We only support standby mode. All we have to do is gate the clock to
+ * the spacc. The hardware will preserve state until we turn it back
+ * on again.
+ */
+ clk_disable(engine->clk);
+
+ return 0;
+}
+
+static int spacc_resume(struct device *dev)
+{
+ struct platform_device *pdev = to_platform_device(dev);
+ struct spacc_engine *engine = platform_get_drvdata(pdev);
+
+ return clk_enable(engine->clk);
+}
+
+static const struct dev_pm_ops spacc_pm_ops = {
+ .suspend = spacc_suspend,
+ .resume = spacc_resume,
+};
+#endif /* CONFIG_PM */
+
+static inline struct spacc_engine *spacc_dev_to_engine(struct device *dev)
+{
+ return dev ? platform_get_drvdata(to_platform_device(dev)) : NULL;
+}
+
+static ssize_t spacc_stat_irq_thresh_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct spacc_engine *engine = spacc_dev_to_engine(dev);
+
+ return snprintf(buf, PAGE_SIZE, "%u\n", engine->stat_irq_thresh);
+}
+
+static ssize_t spacc_stat_irq_thresh_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t len)
+{
+ struct spacc_engine *engine = spacc_dev_to_engine(dev);
+ unsigned long thresh;
+
+ if (strict_strtoul(buf, 0, &thresh))
+ return -EINVAL;
+
+ thresh = clamp(thresh, 1UL, engine->fifo_sz - 1);
+
+ engine->stat_irq_thresh = thresh;
+ writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
+ engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
+
+ return len;
+}
+static DEVICE_ATTR(stat_irq_thresh, 0644, spacc_stat_irq_thresh_show,
+ spacc_stat_irq_thresh_store);
+
+static struct spacc_alg ipsec_engine_algs[] = {
+ {
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC,
+ .key_offs = 0,
+ .iv_offs = AES_MAX_KEY_SIZE,
+ .alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_aes_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .key_offs = 0,
+ .iv_offs = AES_MAX_KEY_SIZE,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_ECB,
+ .alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_aes_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
+ .alg = {
+ .cra_name = "cbc(des)",
+ .cra_driver_name = "cbc-des-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_des_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ .ivsize = DES_BLOCK_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
+ .alg = {
+ .cra_name = "ecb(des)",
+ .cra_driver_name = "ecb-des-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_des_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = DES_KEY_SIZE,
+ .max_keysize = DES_KEY_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC,
+ .alg = {
+ .cra_name = "cbc(des3_ede)",
+ .cra_driver_name = "cbc-des3-ede-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_des_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_ECB,
+ .alg = {
+ .cra_name = "ecb(des3_ede)",
+ .cra_driver_name = "ecb-des3-ede-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_des_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+ {
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC,
+ .key_offs = 0,
+ .iv_offs = AES_MAX_KEY_SIZE,
+ .alg = {
+ .cra_name = "authenc(hmac(sha1),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-sha1-cbc-aes-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+ {
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_SHA256 |
+ SPA_CTRL_HASH_MODE_HMAC,
+ .key_offs = 0,
+ .iv_offs = AES_MAX_KEY_SIZE,
+ .alg = {
+ .cra_name = "authenc(hmac(sha256),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-sha256-cbc-aes-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+ {
+ .key_offs = 0,
+ .iv_offs = AES_MAX_KEY_SIZE,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC,
+ .alg = {
+ .cra_name = "authenc(hmac(md5),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-md5-cbc-aes-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_SHA | SPA_CTRL_HASH_MODE_HMAC,
+ .alg = {
+ .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-sha1-cbc-3des-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_AES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_SHA256 |
+ SPA_CTRL_HASH_MODE_HMAC,
+ .alg = {
+ .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-sha256-cbc-3des-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+ {
+ .key_offs = DES_BLOCK_SIZE,
+ .iv_offs = 0,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_DES | SPA_CTRL_CIPH_MODE_CBC |
+ SPA_CTRL_HASH_ALG_MD5 | SPA_CTRL_HASH_MODE_HMAC,
+ .alg = {
+ .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-md5-cbc-3des-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct spacc_aead_ctx),
+ .cra_type = &crypto_aead_type,
+ .cra_module = THIS_MODULE,
+ .cra_aead = {
+ .setkey = spacc_aead_setkey,
+ .setauthsize = spacc_aead_setauthsize,
+ .encrypt = spacc_aead_encrypt,
+ .decrypt = spacc_aead_decrypt,
+ .givencrypt = spacc_aead_givencrypt,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ },
+ .cra_init = spacc_aead_cra_init,
+ .cra_exit = spacc_aead_cra_exit,
+ },
+ },
+};
+
+static struct spacc_alg l2_engine_algs[] = {
+ {
+ .key_offs = 0,
+ .iv_offs = SPACC_CRYPTO_KASUMI_F8_KEY_LEN,
+ .ctrl_default = SPA_CTRL_CIPH_ALG_KASUMI |
+ SPA_CTRL_CIPH_MODE_F8,
+ .alg = {
+ .cra_name = "f8(kasumi)",
+ .cra_driver_name = "f8-kasumi-picoxcell",
+ .cra_priority = SPACC_CRYPTO_ALG_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_GIVCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 8,
+ .cra_ctxsize = sizeof(struct spacc_ablk_ctx),
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_ablkcipher = {
+ .setkey = spacc_kasumi_f8_setkey,
+ .encrypt = spacc_ablk_encrypt,
+ .decrypt = spacc_ablk_decrypt,
+ .min_keysize = 16,
+ .max_keysize = 16,
+ .ivsize = 8,
+ },
+ .cra_init = spacc_ablk_cra_init,
+ .cra_exit = spacc_ablk_cra_exit,
+ },
+ },
+};
+
+#ifdef CONFIG_OF
+static const struct of_device_id spacc_of_id_table[] = {
+ { .compatible = "picochip,spacc-ipsec" },
+ { .compatible = "picochip,spacc-l2" },
+ {}
+};
+#else /* CONFIG_OF */
+#define spacc_of_id_table NULL
+#endif /* CONFIG_OF */
+
+static bool spacc_is_compatible(struct platform_device *pdev,
+ const char *spacc_type)
+{
+ const struct platform_device_id *platid = platform_get_device_id(pdev);
+
+ if (platid && !strcmp(platid->name, spacc_type))
+ return true;
+
+#ifdef CONFIG_OF
+ if (of_device_is_compatible(pdev->dev.of_node, spacc_type))
+ return true;
+#endif /* CONFIG_OF */
+
+ return false;
+}
+
+static int __devinit spacc_probe(struct platform_device *pdev)
+{
+ int i, err, ret = -EINVAL;
+ struct resource *mem, *irq;
+ struct spacc_engine *engine = devm_kzalloc(&pdev->dev, sizeof(*engine),
+ GFP_KERNEL);
+ if (!engine)
+ return -ENOMEM;
+
+ if (spacc_is_compatible(pdev, "picochip,spacc-ipsec")) {
+ engine->max_ctxs = SPACC_CRYPTO_IPSEC_MAX_CTXS;
+ engine->cipher_pg_sz = SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ;
+ engine->hash_pg_sz = SPACC_CRYPTO_IPSEC_HASH_PG_SZ;
+ engine->fifo_sz = SPACC_CRYPTO_IPSEC_FIFO_SZ;
+ engine->algs = ipsec_engine_algs;
+ engine->num_algs = ARRAY_SIZE(ipsec_engine_algs);
+ } else if (spacc_is_compatible(pdev, "picochip,spacc-l2")) {
+ engine->max_ctxs = SPACC_CRYPTO_L2_MAX_CTXS;
+ engine->cipher_pg_sz = SPACC_CRYPTO_L2_CIPHER_PG_SZ;
+ engine->hash_pg_sz = SPACC_CRYPTO_L2_HASH_PG_SZ;
+ engine->fifo_sz = SPACC_CRYPTO_L2_FIFO_SZ;
+ engine->algs = l2_engine_algs;
+ engine->num_algs = ARRAY_SIZE(l2_engine_algs);
+ } else {
+ return -EINVAL;
+ }
+
+ engine->name = dev_name(&pdev->dev);
+
+ mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ irq = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!mem || !irq) {
+ dev_err(&pdev->dev, "no memory/irq resource for engine\n");
+ return -ENXIO;
+ }
+
+ if (!devm_request_mem_region(&pdev->dev, mem->start, resource_size(mem),
+ engine->name))
+ return -ENOMEM;
+
+ engine->regs = devm_ioremap(&pdev->dev, mem->start, resource_size(mem));
+ if (!engine->regs) {
+ dev_err(&pdev->dev, "memory map failed\n");
+ return -ENOMEM;
+ }
+
+ if (devm_request_irq(&pdev->dev, irq->start, spacc_spacc_irq, 0,
+ engine->name, engine)) {
+ dev_err(engine->dev, "failed to request IRQ\n");
+ return -EBUSY;
+ }
+
+ engine->dev = &pdev->dev;
+ engine->cipher_ctx_base = engine->regs + SPA_CIPH_KEY_BASE_REG_OFFSET;
+ engine->hash_key_base = engine->regs + SPA_HASH_KEY_BASE_REG_OFFSET;
+
+ engine->req_pool = dmam_pool_create(engine->name, engine->dev,
+ MAX_DDT_LEN * sizeof(struct spacc_ddt), 8, SZ_64K);
+ if (!engine->req_pool)
+ return -ENOMEM;
+
+ spin_lock_init(&engine->hw_lock);
+
+ engine->clk = clk_get(&pdev->dev, "ref");
+ if (IS_ERR(engine->clk)) {
+ dev_info(&pdev->dev, "clk unavailable\n");
+ device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+ return PTR_ERR(engine->clk);
+ }
+
+ if (clk_enable(engine->clk)) {
+ dev_info(&pdev->dev, "unable to enable clk\n");
+ clk_put(engine->clk);
+ return -EIO;
+ }
+
+ err = device_create_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+ if (err) {
+ clk_disable(engine->clk);
+ clk_put(engine->clk);
+ return err;
+ }
+
+
+ /*
+ * Use an IRQ threshold of 50% as a default. This seems to be a
+ * reasonable trade off of latency against throughput but can be
+ * changed at runtime.
+ */
+ engine->stat_irq_thresh = (engine->fifo_sz / 2);
+
+ /*
+ * Configure the interrupts. We only use the STAT_CNT interrupt as we
+ * only submit a new packet for processing when we complete another in
+ * the queue. This minimizes time spent in the interrupt handler.
+ */
+ writel(engine->stat_irq_thresh << SPA_IRQ_CTRL_STAT_CNT_OFFSET,
+ engine->regs + SPA_IRQ_CTRL_REG_OFFSET);
+ writel(SPA_IRQ_EN_STAT_EN | SPA_IRQ_EN_GLBL_EN,
+ engine->regs + SPA_IRQ_EN_REG_OFFSET);
+
+ setup_timer(&engine->packet_timeout, spacc_packet_timeout,
+ (unsigned long)engine);
+
+ INIT_LIST_HEAD(&engine->pending);
+ INIT_LIST_HEAD(&engine->completed);
+ INIT_LIST_HEAD(&engine->in_progress);
+ engine->in_flight = 0;
+ tasklet_init(&engine->complete, spacc_spacc_complete,
+ (unsigned long)engine);
+
+ platform_set_drvdata(pdev, engine);
+
+ INIT_LIST_HEAD(&engine->registered_algs);
+ for (i = 0; i < engine->num_algs; ++i) {
+ engine->algs[i].engine = engine;
+ err = crypto_register_alg(&engine->algs[i].alg);
+ if (!err) {
+ list_add_tail(&engine->algs[i].entry,
+ &engine->registered_algs);
+ ret = 0;
+ }
+ if (err)
+ dev_err(engine->dev, "failed to register alg \"%s\"\n",
+ engine->algs[i].alg.cra_name);
+ else
+ dev_dbg(engine->dev, "registered alg \"%s\"\n",
+ engine->algs[i].alg.cra_name);
+ }
+
+ return ret;
+}
+
+static int __devexit spacc_remove(struct platform_device *pdev)
+{
+ struct spacc_alg *alg, *next;
+ struct spacc_engine *engine = platform_get_drvdata(pdev);
+
+ del_timer_sync(&engine->packet_timeout);
+ device_remove_file(&pdev->dev, &dev_attr_stat_irq_thresh);
+
+ list_for_each_entry_safe(alg, next, &engine->registered_algs, entry) {
+ list_del(&alg->entry);
+ crypto_unregister_alg(&alg->alg);
+ }
+
+ clk_disable(engine->clk);
+ clk_put(engine->clk);
+
+ return 0;
+}
+
+static const struct platform_device_id spacc_id_table[] = {
+ { "picochip,spacc-ipsec", },
+ { "picochip,spacc-l2", },
+};
+
+static struct platform_driver spacc_driver = {
+ .probe = spacc_probe,
+ .remove = __devexit_p(spacc_remove),
+ .driver = {
+ .name = "picochip,spacc",
+#ifdef CONFIG_PM
+ .pm = &spacc_pm_ops,
+#endif /* CONFIG_PM */
+ .of_match_table = spacc_of_id_table,
+ },
+ .id_table = spacc_id_table,
+};
+
+module_platform_driver(spacc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Jamie Iles");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto_regs.h b/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto_regs.h
new file mode 100644
index 0000000..af93442
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/picoxcell_crypto_regs.h
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2010 Picochip Ltd., Jamie Iles
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+#ifndef __PICOXCELL_CRYPTO_REGS_H__
+#define __PICOXCELL_CRYPTO_REGS_H__
+
+#define SPA_STATUS_OK 0
+#define SPA_STATUS_ICV_FAIL 1
+#define SPA_STATUS_MEMORY_ERROR 2
+#define SPA_STATUS_BLOCK_ERROR 3
+
+#define SPA_IRQ_CTRL_STAT_CNT_OFFSET 16
+#define SPA_IRQ_STAT_STAT_MASK (1 << 4)
+#define SPA_FIFO_STAT_STAT_OFFSET 16
+#define SPA_FIFO_STAT_STAT_CNT_MASK (0x3F << SPA_FIFO_STAT_STAT_OFFSET)
+#define SPA_STATUS_RES_CODE_OFFSET 24
+#define SPA_STATUS_RES_CODE_MASK (0x3 << SPA_STATUS_RES_CODE_OFFSET)
+#define SPA_KEY_SZ_CTX_INDEX_OFFSET 8
+#define SPA_KEY_SZ_CIPHER_OFFSET 31
+
+#define SPA_IRQ_EN_REG_OFFSET 0x00000000
+#define SPA_IRQ_STAT_REG_OFFSET 0x00000004
+#define SPA_IRQ_CTRL_REG_OFFSET 0x00000008
+#define SPA_FIFO_STAT_REG_OFFSET 0x0000000C
+#define SPA_SDMA_BRST_SZ_REG_OFFSET 0x00000010
+#define SPA_SRC_PTR_REG_OFFSET 0x00000020
+#define SPA_DST_PTR_REG_OFFSET 0x00000024
+#define SPA_OFFSET_REG_OFFSET 0x00000028
+#define SPA_AAD_LEN_REG_OFFSET 0x0000002C
+#define SPA_PROC_LEN_REG_OFFSET 0x00000030
+#define SPA_ICV_LEN_REG_OFFSET 0x00000034
+#define SPA_ICV_OFFSET_REG_OFFSET 0x00000038
+#define SPA_SW_CTRL_REG_OFFSET 0x0000003C
+#define SPA_CTRL_REG_OFFSET 0x00000040
+#define SPA_AUX_INFO_REG_OFFSET 0x0000004C
+#define SPA_STAT_POP_REG_OFFSET 0x00000050
+#define SPA_STATUS_REG_OFFSET 0x00000054
+#define SPA_KEY_SZ_REG_OFFSET 0x00000100
+#define SPA_CIPH_KEY_BASE_REG_OFFSET 0x00004000
+#define SPA_HASH_KEY_BASE_REG_OFFSET 0x00008000
+#define SPA_RC4_CTX_BASE_REG_OFFSET 0x00020000
+
+#define SPA_IRQ_EN_REG_RESET 0x00000000
+#define SPA_IRQ_CTRL_REG_RESET 0x00000000
+#define SPA_FIFO_STAT_REG_RESET 0x00000000
+#define SPA_SDMA_BRST_SZ_REG_RESET 0x00000000
+#define SPA_SRC_PTR_REG_RESET 0x00000000
+#define SPA_DST_PTR_REG_RESET 0x00000000
+#define SPA_OFFSET_REG_RESET 0x00000000
+#define SPA_AAD_LEN_REG_RESET 0x00000000
+#define SPA_PROC_LEN_REG_RESET 0x00000000
+#define SPA_ICV_LEN_REG_RESET 0x00000000
+#define SPA_ICV_OFFSET_REG_RESET 0x00000000
+#define SPA_SW_CTRL_REG_RESET 0x00000000
+#define SPA_CTRL_REG_RESET 0x00000000
+#define SPA_AUX_INFO_REG_RESET 0x00000000
+#define SPA_STAT_POP_REG_RESET 0x00000000
+#define SPA_STATUS_REG_RESET 0x00000000
+#define SPA_KEY_SZ_REG_RESET 0x00000000
+
+#define SPA_CTRL_HASH_ALG_IDX 4
+#define SPA_CTRL_CIPH_MODE_IDX 8
+#define SPA_CTRL_HASH_MODE_IDX 12
+#define SPA_CTRL_CTX_IDX 16
+#define SPA_CTRL_ENCRYPT_IDX 24
+#define SPA_CTRL_AAD_COPY 25
+#define SPA_CTRL_ICV_PT 26
+#define SPA_CTRL_ICV_ENC 27
+#define SPA_CTRL_ICV_APPEND 28
+#define SPA_CTRL_KEY_EXP 29
+
+#define SPA_KEY_SZ_CXT_IDX 8
+#define SPA_KEY_SZ_CIPHER_IDX 31
+
+#define SPA_IRQ_EN_CMD0_EN (1 << 0)
+#define SPA_IRQ_EN_STAT_EN (1 << 4)
+#define SPA_IRQ_EN_GLBL_EN (1 << 31)
+
+#define SPA_CTRL_CIPH_ALG_NULL 0x00
+#define SPA_CTRL_CIPH_ALG_DES 0x01
+#define SPA_CTRL_CIPH_ALG_AES 0x02
+#define SPA_CTRL_CIPH_ALG_RC4 0x03
+#define SPA_CTRL_CIPH_ALG_MULTI2 0x04
+#define SPA_CTRL_CIPH_ALG_KASUMI 0x05
+
+#define SPA_CTRL_HASH_ALG_NULL (0x00 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_MD5 (0x01 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_SHA (0x02 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_SHA224 (0x03 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_SHA256 (0x04 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_SHA384 (0x05 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_SHA512 (0x06 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_AESMAC (0x07 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_AESCMAC (0x08 << SPA_CTRL_HASH_ALG_IDX)
+#define SPA_CTRL_HASH_ALG_KASF9 (0x09 << SPA_CTRL_HASH_ALG_IDX)
+
+#define SPA_CTRL_CIPH_MODE_NULL (0x00 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_ECB (0x00 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_CBC (0x01 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_CTR (0x02 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_CCM (0x03 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_GCM (0x05 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_OFB (0x07 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_CFB (0x08 << SPA_CTRL_CIPH_MODE_IDX)
+#define SPA_CTRL_CIPH_MODE_F8 (0x09 << SPA_CTRL_CIPH_MODE_IDX)
+
+#define SPA_CTRL_HASH_MODE_RAW (0x00 << SPA_CTRL_HASH_MODE_IDX)
+#define SPA_CTRL_HASH_MODE_SSLMAC (0x01 << SPA_CTRL_HASH_MODE_IDX)
+#define SPA_CTRL_HASH_MODE_HMAC (0x02 << SPA_CTRL_HASH_MODE_IDX)
+
+#define SPA_FIFO_STAT_EMPTY (1 << 31)
+#define SPA_FIFO_CMD_FULL (1 << 7)
+
+#endif /* __PICOXCELL_CRYPTO_REGS_H__ */
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/s5p-sss.c b/ap/os/linux/linux-3.4.x/drivers/crypto/s5p-sss.c
new file mode 100644
index 0000000..bc986f8
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/s5p-sss.c
@@ -0,0 +1,692 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for Samsung S5PV210 HW acceleration.
+ *
+ * Copyright (C) 2011 NetUP Inc. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation.
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/crypto.h>
+#include <linux/interrupt.h>
+
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/ctr.h>
+
+#include <plat/cpu.h>
+#include <plat/dma.h>
+
+#define _SBF(s, v) ((v) << (s))
+#define _BIT(b) _SBF(b, 1)
+
+/* Feed control registers */
+#define SSS_REG_FCINTSTAT 0x0000
+#define SSS_FCINTSTAT_BRDMAINT _BIT(3)
+#define SSS_FCINTSTAT_BTDMAINT _BIT(2)
+#define SSS_FCINTSTAT_HRDMAINT _BIT(1)
+#define SSS_FCINTSTAT_PKDMAINT _BIT(0)
+
+#define SSS_REG_FCINTENSET 0x0004
+#define SSS_FCINTENSET_BRDMAINTENSET _BIT(3)
+#define SSS_FCINTENSET_BTDMAINTENSET _BIT(2)
+#define SSS_FCINTENSET_HRDMAINTENSET _BIT(1)
+#define SSS_FCINTENSET_PKDMAINTENSET _BIT(0)
+
+#define SSS_REG_FCINTENCLR 0x0008
+#define SSS_FCINTENCLR_BRDMAINTENCLR _BIT(3)
+#define SSS_FCINTENCLR_BTDMAINTENCLR _BIT(2)
+#define SSS_FCINTENCLR_HRDMAINTENCLR _BIT(1)
+#define SSS_FCINTENCLR_PKDMAINTENCLR _BIT(0)
+
+#define SSS_REG_FCINTPEND 0x000C
+#define SSS_FCINTPEND_BRDMAINTP _BIT(3)
+#define SSS_FCINTPEND_BTDMAINTP _BIT(2)
+#define SSS_FCINTPEND_HRDMAINTP _BIT(1)
+#define SSS_FCINTPEND_PKDMAINTP _BIT(0)
+
+#define SSS_REG_FCFIFOSTAT 0x0010
+#define SSS_FCFIFOSTAT_BRFIFOFUL _BIT(7)
+#define SSS_FCFIFOSTAT_BRFIFOEMP _BIT(6)
+#define SSS_FCFIFOSTAT_BTFIFOFUL _BIT(5)
+#define SSS_FCFIFOSTAT_BTFIFOEMP _BIT(4)
+#define SSS_FCFIFOSTAT_HRFIFOFUL _BIT(3)
+#define SSS_FCFIFOSTAT_HRFIFOEMP _BIT(2)
+#define SSS_FCFIFOSTAT_PKFIFOFUL _BIT(1)
+#define SSS_FCFIFOSTAT_PKFIFOEMP _BIT(0)
+
+#define SSS_REG_FCFIFOCTRL 0x0014
+#define SSS_FCFIFOCTRL_DESSEL _BIT(2)
+#define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00)
+#define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01)
+#define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02)
+
+#define SSS_REG_FCBRDMAS 0x0020
+#define SSS_REG_FCBRDMAL 0x0024
+#define SSS_REG_FCBRDMAC 0x0028
+#define SSS_FCBRDMAC_BYTESWAP _BIT(1)
+#define SSS_FCBRDMAC_FLUSH _BIT(0)
+
+#define SSS_REG_FCBTDMAS 0x0030
+#define SSS_REG_FCBTDMAL 0x0034
+#define SSS_REG_FCBTDMAC 0x0038
+#define SSS_FCBTDMAC_BYTESWAP _BIT(1)
+#define SSS_FCBTDMAC_FLUSH _BIT(0)
+
+#define SSS_REG_FCHRDMAS 0x0040
+#define SSS_REG_FCHRDMAL 0x0044
+#define SSS_REG_FCHRDMAC 0x0048
+#define SSS_FCHRDMAC_BYTESWAP _BIT(1)
+#define SSS_FCHRDMAC_FLUSH _BIT(0)
+
+#define SSS_REG_FCPKDMAS 0x0050
+#define SSS_REG_FCPKDMAL 0x0054
+#define SSS_REG_FCPKDMAC 0x0058
+#define SSS_FCPKDMAC_BYTESWAP _BIT(3)
+#define SSS_FCPKDMAC_DESCEND _BIT(2)
+#define SSS_FCPKDMAC_TRANSMIT _BIT(1)
+#define SSS_FCPKDMAC_FLUSH _BIT(0)
+
+#define SSS_REG_FCPKDMAO 0x005C
+
+/* AES registers */
+#define SSS_REG_AES_CONTROL 0x4000
+#define SSS_AES_BYTESWAP_DI _BIT(11)
+#define SSS_AES_BYTESWAP_DO _BIT(10)
+#define SSS_AES_BYTESWAP_IV _BIT(9)
+#define SSS_AES_BYTESWAP_CNT _BIT(8)
+#define SSS_AES_BYTESWAP_KEY _BIT(7)
+#define SSS_AES_KEY_CHANGE_MODE _BIT(6)
+#define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00)
+#define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01)
+#define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02)
+#define SSS_AES_FIFO_MODE _BIT(3)
+#define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00)
+#define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01)
+#define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02)
+#define SSS_AES_MODE_DECRYPT _BIT(0)
+
+#define SSS_REG_AES_STATUS 0x4004
+#define SSS_AES_BUSY _BIT(2)
+#define SSS_AES_INPUT_READY _BIT(1)
+#define SSS_AES_OUTPUT_READY _BIT(0)
+
+#define SSS_REG_AES_IN_DATA(s) (0x4010 + (s << 2))
+#define SSS_REG_AES_OUT_DATA(s) (0x4020 + (s << 2))
+#define SSS_REG_AES_IV_DATA(s) (0x4030 + (s << 2))
+#define SSS_REG_AES_CNT_DATA(s) (0x4040 + (s << 2))
+#define SSS_REG_AES_KEY_DATA(s) (0x4080 + (s << 2))
+
+#define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg))
+#define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg))
+#define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg))
+
+/* HW engine modes */
+#define FLAGS_AES_DECRYPT _BIT(0)
+#define FLAGS_AES_MODE_MASK _SBF(1, 0x03)
+#define FLAGS_AES_CBC _SBF(1, 0x01)
+#define FLAGS_AES_CTR _SBF(1, 0x02)
+
+#define AES_KEY_LEN 16
+#define CRYPTO_QUEUE_LEN 1
+
+struct s5p_aes_reqctx {
+ unsigned long mode;
+};
+
+struct s5p_aes_ctx {
+ struct s5p_aes_dev *dev;
+
+ uint8_t aes_key[AES_MAX_KEY_SIZE];
+ uint8_t nonce[CTR_RFC3686_NONCE_SIZE];
+ int keylen;
+};
+
+struct s5p_aes_dev {
+ struct device *dev;
+ struct clk *clk;
+ void __iomem *ioaddr;
+ int irq_hash;
+ int irq_fc;
+
+ struct ablkcipher_request *req;
+ struct s5p_aes_ctx *ctx;
+ struct scatterlist *sg_src;
+ struct scatterlist *sg_dst;
+
+ struct tasklet_struct tasklet;
+ struct crypto_queue queue;
+ bool busy;
+ spinlock_t lock;
+};
+
+static struct s5p_aes_dev *s5p_dev;
+
+static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
+{
+ SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));
+ SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));
+}
+
+static void s5p_set_dma_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
+{
+ SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));
+ SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));
+}
+
+static void s5p_aes_complete(struct s5p_aes_dev *dev, int err)
+{
+ /* holding a lock outside */
+ dev->req->base.complete(&dev->req->base, err);
+ dev->busy = false;
+}
+
+static void s5p_unset_outdata(struct s5p_aes_dev *dev)
+{
+ dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);
+}
+
+static void s5p_unset_indata(struct s5p_aes_dev *dev)
+{
+ dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);
+}
+
+static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
+{
+ int err;
+
+ if (!IS_ALIGNED(sg_dma_len(sg), AES_BLOCK_SIZE)) {
+ err = -EINVAL;
+ goto exit;
+ }
+ if (!sg_dma_len(sg)) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ err = dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE);
+ if (!err) {
+ err = -ENOMEM;
+ goto exit;
+ }
+
+ dev->sg_dst = sg;
+ err = 0;
+
+ exit:
+ return err;
+}
+
+static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
+{
+ int err;
+
+ if (!IS_ALIGNED(sg_dma_len(sg), AES_BLOCK_SIZE)) {
+ err = -EINVAL;
+ goto exit;
+ }
+ if (!sg_dma_len(sg)) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ err = dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE);
+ if (!err) {
+ err = -ENOMEM;
+ goto exit;
+ }
+
+ dev->sg_src = sg;
+ err = 0;
+
+ exit:
+ return err;
+}
+
+static void s5p_aes_tx(struct s5p_aes_dev *dev)
+{
+ int err = 0;
+
+ s5p_unset_outdata(dev);
+
+ if (!sg_is_last(dev->sg_dst)) {
+ err = s5p_set_outdata(dev, sg_next(dev->sg_dst));
+ if (err) {
+ s5p_aes_complete(dev, err);
+ return;
+ }
+
+ s5p_set_dma_outdata(dev, dev->sg_dst);
+ } else
+ s5p_aes_complete(dev, err);
+}
+
+static void s5p_aes_rx(struct s5p_aes_dev *dev)
+{
+ int err;
+
+ s5p_unset_indata(dev);
+
+ if (!sg_is_last(dev->sg_src)) {
+ err = s5p_set_indata(dev, sg_next(dev->sg_src));
+ if (err) {
+ s5p_aes_complete(dev, err);
+ return;
+ }
+
+ s5p_set_dma_indata(dev, dev->sg_src);
+ }
+}
+
+static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
+{
+ struct platform_device *pdev = dev_id;
+ struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
+ uint32_t status;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+
+ if (irq == dev->irq_fc) {
+ status = SSS_READ(dev, FCINTSTAT);
+ if (status & SSS_FCINTSTAT_BRDMAINT)
+ s5p_aes_rx(dev);
+ if (status & SSS_FCINTSTAT_BTDMAINT)
+ s5p_aes_tx(dev);
+
+ SSS_WRITE(dev, FCINTPEND, status);
+ }
+
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ return IRQ_HANDLED;
+}
+
+static void s5p_set_aes(struct s5p_aes_dev *dev,
+ uint8_t *key, uint8_t *iv, unsigned int keylen)
+{
+ void __iomem *keystart;
+
+ memcpy(dev->ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10);
+
+ if (keylen == AES_KEYSIZE_256)
+ keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(0);
+ else if (keylen == AES_KEYSIZE_192)
+ keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(2);
+ else
+ keystart = dev->ioaddr + SSS_REG_AES_KEY_DATA(4);
+
+ memcpy(keystart, key, keylen);
+}
+
+static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
+{
+ struct ablkcipher_request *req = dev->req;
+
+ uint32_t aes_control;
+ int err;
+ unsigned long flags;
+
+ aes_control = SSS_AES_KEY_CHANGE_MODE;
+ if (mode & FLAGS_AES_DECRYPT)
+ aes_control |= SSS_AES_MODE_DECRYPT;
+
+ if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC)
+ aes_control |= SSS_AES_CHAIN_MODE_CBC;
+ else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR)
+ aes_control |= SSS_AES_CHAIN_MODE_CTR;
+
+ if (dev->ctx->keylen == AES_KEYSIZE_192)
+ aes_control |= SSS_AES_KEY_SIZE_192;
+ else if (dev->ctx->keylen == AES_KEYSIZE_256)
+ aes_control |= SSS_AES_KEY_SIZE_256;
+
+ aes_control |= SSS_AES_FIFO_MODE;
+
+ /* as a variant it is possible to use byte swapping on DMA side */
+ aes_control |= SSS_AES_BYTESWAP_DI
+ | SSS_AES_BYTESWAP_DO
+ | SSS_AES_BYTESWAP_IV
+ | SSS_AES_BYTESWAP_KEY
+ | SSS_AES_BYTESWAP_CNT;
+
+ spin_lock_irqsave(&dev->lock, flags);
+
+ SSS_WRITE(dev, FCINTENCLR,
+ SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);
+ SSS_WRITE(dev, FCFIFOCTRL, 0x00);
+
+ err = s5p_set_indata(dev, req->src);
+ if (err)
+ goto indata_error;
+
+ err = s5p_set_outdata(dev, req->dst);
+ if (err)
+ goto outdata_error;
+
+ SSS_WRITE(dev, AES_CONTROL, aes_control);
+ s5p_set_aes(dev, dev->ctx->aes_key, req->info, dev->ctx->keylen);
+
+ s5p_set_dma_indata(dev, req->src);
+ s5p_set_dma_outdata(dev, req->dst);
+
+ SSS_WRITE(dev, FCINTENSET,
+ SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);
+
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ return;
+
+ outdata_error:
+ s5p_unset_indata(dev);
+
+ indata_error:
+ s5p_aes_complete(dev, err);
+ spin_unlock_irqrestore(&dev->lock, flags);
+}
+
+static void s5p_tasklet_cb(unsigned long data)
+{
+ struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;
+ struct crypto_async_request *async_req, *backlog;
+ struct s5p_aes_reqctx *reqctx;
+ unsigned long flags;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ backlog = crypto_get_backlog(&dev->queue);
+ async_req = crypto_dequeue_request(&dev->queue);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ if (!async_req)
+ return;
+
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
+ dev->req = ablkcipher_request_cast(async_req);
+ dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);
+ reqctx = ablkcipher_request_ctx(dev->req);
+
+ s5p_aes_crypt_start(dev, reqctx->mode);
+}
+
+static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
+ struct ablkcipher_request *req)
+{
+ unsigned long flags;
+ int err;
+
+ spin_lock_irqsave(&dev->lock, flags);
+ if (dev->busy) {
+ err = -EAGAIN;
+ spin_unlock_irqrestore(&dev->lock, flags);
+ goto exit;
+ }
+ dev->busy = true;
+
+ err = ablkcipher_enqueue_request(&dev->queue, req);
+ spin_unlock_irqrestore(&dev->lock, flags);
+
+ tasklet_schedule(&dev->tasklet);
+
+ exit:
+ return err;
+}
+
+static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
+ struct s5p_aes_dev *dev = ctx->dev;
+
+ if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
+ pr_err("request size is not exact amount of AES blocks\n");
+ return -EINVAL;
+ }
+
+ reqctx->mode = mode;
+
+ return s5p_aes_handle_req(dev, req);
+}
+
+static int s5p_aes_setkey(struct crypto_ablkcipher *cipher,
+ const uint8_t *key, unsigned int keylen)
+{
+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
+ struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ if (keylen != AES_KEYSIZE_128 &&
+ keylen != AES_KEYSIZE_192 &&
+ keylen != AES_KEYSIZE_256)
+ return -EINVAL;
+
+ memcpy(ctx->aes_key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static int s5p_aes_ecb_encrypt(struct ablkcipher_request *req)
+{
+ return s5p_aes_crypt(req, 0);
+}
+
+static int s5p_aes_ecb_decrypt(struct ablkcipher_request *req)
+{
+ return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);
+}
+
+static int s5p_aes_cbc_encrypt(struct ablkcipher_request *req)
+{
+ return s5p_aes_crypt(req, FLAGS_AES_CBC);
+}
+
+static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req)
+{
+ return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
+}
+
+static int s5p_aes_cra_init(struct crypto_tfm *tfm)
+{
+ struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ ctx->dev = s5p_dev;
+ tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_aes_reqctx);
+
+ return 0;
+}
+
+static struct crypto_alg algs[] = {
+ {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-s5p",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct s5p_aes_ctx),
+ .cra_alignmask = 0x0f,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = s5p_aes_cra_init,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = s5p_aes_setkey,
+ .encrypt = s5p_aes_ecb_encrypt,
+ .decrypt = s5p_aes_ecb_decrypt,
+ }
+ },
+ {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-s5p",
+ .cra_priority = 100,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct s5p_aes_ctx),
+ .cra_alignmask = 0x0f,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_init = s5p_aes_cra_init,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = s5p_aes_setkey,
+ .encrypt = s5p_aes_cbc_encrypt,
+ .decrypt = s5p_aes_cbc_decrypt,
+ }
+ },
+};
+
+static int s5p_aes_probe(struct platform_device *pdev)
+{
+ int i, j, err = -ENODEV;
+ struct s5p_aes_dev *pdata;
+ struct device *dev = &pdev->dev;
+ struct resource *res;
+
+ if (s5p_dev)
+ return -EEXIST;
+
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res)
+ return -ENODEV;
+
+ pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
+ if (!pdata)
+ return -ENOMEM;
+
+ if (!devm_request_mem_region(dev, res->start,
+ resource_size(res), pdev->name))
+ return -EBUSY;
+
+ pdata->clk = clk_get(dev, "secss");
+ if (IS_ERR(pdata->clk)) {
+ dev_err(dev, "failed to find secss clock source\n");
+ return -ENOENT;
+ }
+
+ clk_enable(pdata->clk);
+
+ spin_lock_init(&pdata->lock);
+ pdata->ioaddr = devm_ioremap(dev, res->start,
+ resource_size(res));
+
+ pdata->irq_hash = platform_get_irq_byname(pdev, "hash");
+ if (pdata->irq_hash < 0) {
+ err = pdata->irq_hash;
+ dev_warn(dev, "hash interrupt is not available.\n");
+ goto err_irq;
+ }
+ err = devm_request_irq(dev, pdata->irq_hash, s5p_aes_interrupt,
+ IRQF_SHARED, pdev->name, pdev);
+ if (err < 0) {
+ dev_warn(dev, "hash interrupt is not available.\n");
+ goto err_irq;
+ }
+
+ pdata->irq_fc = platform_get_irq_byname(pdev, "feed control");
+ if (pdata->irq_fc < 0) {
+ err = pdata->irq_fc;
+ dev_warn(dev, "feed control interrupt is not available.\n");
+ goto err_irq;
+ }
+ err = devm_request_irq(dev, pdata->irq_fc, s5p_aes_interrupt,
+ IRQF_SHARED, pdev->name, pdev);
+ if (err < 0) {
+ dev_warn(dev, "feed control interrupt is not available.\n");
+ goto err_irq;
+ }
+
+ pdata->dev = dev;
+ platform_set_drvdata(pdev, pdata);
+ s5p_dev = pdata;
+
+ tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);
+ crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++) {
+ INIT_LIST_HEAD(&algs[i].cra_list);
+ err = crypto_register_alg(&algs[i]);
+ if (err)
+ goto err_algs;
+ }
+
+ pr_info("s5p-sss driver registered\n");
+
+ return 0;
+
+ err_algs:
+ dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name, err);
+
+ for (j = 0; j < i; j++)
+ crypto_unregister_alg(&algs[j]);
+
+ tasklet_kill(&pdata->tasklet);
+
+ err_irq:
+ clk_disable(pdata->clk);
+ clk_put(pdata->clk);
+
+ s5p_dev = NULL;
+ platform_set_drvdata(pdev, NULL);
+
+ return err;
+}
+
+static int s5p_aes_remove(struct platform_device *pdev)
+{
+ struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);
+ int i;
+
+ if (!pdata)
+ return -ENODEV;
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++)
+ crypto_unregister_alg(&algs[i]);
+
+ tasklet_kill(&pdata->tasklet);
+
+ clk_disable(pdata->clk);
+ clk_put(pdata->clk);
+
+ s5p_dev = NULL;
+ platform_set_drvdata(pdev, NULL);
+
+ return 0;
+}
+
+static struct platform_driver s5p_aes_crypto = {
+ .probe = s5p_aes_probe,
+ .remove = s5p_aes_remove,
+ .driver = {
+ .owner = THIS_MODULE,
+ .name = "s5p-secss",
+ },
+};
+
+module_platform_driver(s5p_aes_crypto);
+
+MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.c b/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.c
new file mode 100644
index 0000000..a759fdc
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.c
@@ -0,0 +1,2916 @@
+/*
+ * talitos - Freescale Integrated Security Engine (SEC) device driver
+ *
+ * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Scatterlist Crypto API glue code copied from files with the following:
+ * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
+ *
+ * Crypto algorithm registration code copied from hifn driver:
+ * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/of_platform.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/rtnetlink.h>
+#include <linux/slab.h>
+
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
+#include <crypto/des.h>
+#include <crypto/sha.h>
+#include <crypto/md5.h>
+#include <crypto/aead.h>
+#include <crypto/authenc.h>
+#include <crypto/skcipher.h>
+#include <crypto/hash.h>
+#include <crypto/internal/hash.h>
+#include <crypto/scatterwalk.h>
+
+#include "talitos.h"
+
+#define TALITOS_TIMEOUT 100000
+#define TALITOS_MAX_DATA_LEN 65535
+
+#define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
+#define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
+#define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
+
+/* descriptor pointer entry */
+struct talitos_ptr {
+ __be16 len; /* length */
+ u8 j_extent; /* jump to sg link table and/or extent */
+ u8 eptr; /* extended address */
+ __be32 ptr; /* address */
+};
+
+static const struct talitos_ptr zero_entry = {
+ .len = 0,
+ .j_extent = 0,
+ .eptr = 0,
+ .ptr = 0
+};
+
+/* descriptor */
+struct talitos_desc {
+ __be32 hdr; /* header high bits */
+ __be32 hdr_lo; /* header low bits */
+ struct talitos_ptr ptr[7]; /* ptr/len pair array */
+};
+
+/**
+ * talitos_request - descriptor submission request
+ * @desc: descriptor pointer (kernel virtual)
+ * @dma_desc: descriptor's physical bus address
+ * @callback: whom to call when descriptor processing is done
+ * @context: caller context (optional)
+ */
+struct talitos_request {
+ struct talitos_desc *desc;
+ dma_addr_t dma_desc;
+ void (*callback) (struct device *dev, struct talitos_desc *desc,
+ void *context, int error);
+ void *context;
+};
+
+/* per-channel fifo management */
+struct talitos_channel {
+ void __iomem *reg;
+
+ /* request fifo */
+ struct talitos_request *fifo;
+
+ /* number of requests pending in channel h/w fifo */
+ atomic_t submit_count ____cacheline_aligned;
+
+ /* request submission (head) lock */
+ spinlock_t head_lock ____cacheline_aligned;
+ /* index to next free descriptor request */
+ int head;
+
+ /* request release (tail) lock */
+ spinlock_t tail_lock ____cacheline_aligned;
+ /* index to next in-progress/done descriptor request */
+ int tail;
+};
+
+struct talitos_private {
+ struct device *dev;
+ struct platform_device *ofdev;
+ void __iomem *reg;
+ int irq[2];
+
+ /* SEC global registers lock */
+ spinlock_t reg_lock ____cacheline_aligned;
+
+ /* SEC version geometry (from device tree node) */
+ unsigned int num_channels;
+ unsigned int chfifo_len;
+ unsigned int exec_units;
+ unsigned int desc_types;
+
+ /* SEC Compatibility info */
+ unsigned long features;
+
+ /*
+ * length of the request fifo
+ * fifo_len is chfifo_len rounded up to next power of 2
+ * so we can use bitwise ops to wrap
+ */
+ unsigned int fifo_len;
+
+ struct talitos_channel *chan;
+
+ /* next channel to be assigned next incoming descriptor */
+ atomic_t last_chan ____cacheline_aligned;
+
+ /* request callback tasklet */
+ struct tasklet_struct done_task[2];
+
+ /* list of registered algorithms */
+ struct list_head alg_list;
+
+ /* hwrng device */
+ struct hwrng rng;
+};
+
+/* .features flag */
+#define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
+#define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
+#define TALITOS_FTR_SHA224_HWINIT 0x00000004
+#define TALITOS_FTR_HMAC_OK 0x00000008
+
+static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr)
+{
+ talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
+ talitos_ptr->eptr = upper_32_bits(dma_addr);
+}
+
+/*
+ * map virtual single (contiguous) pointer to h/w descriptor pointer
+ */
+static void map_single_talitos_ptr(struct device *dev,
+ struct talitos_ptr *talitos_ptr,
+ unsigned short len, void *data,
+ unsigned char extent,
+ enum dma_data_direction dir)
+{
+ dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
+
+ talitos_ptr->len = cpu_to_be16(len);
+ to_talitos_ptr(talitos_ptr, dma_addr);
+ talitos_ptr->j_extent = extent;
+}
+
+/*
+ * unmap bus single (contiguous) h/w descriptor pointer
+ */
+static void unmap_single_talitos_ptr(struct device *dev,
+ struct talitos_ptr *talitos_ptr,
+ enum dma_data_direction dir)
+{
+ dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
+ be16_to_cpu(talitos_ptr->len), dir);
+}
+
+static int reset_channel(struct device *dev, int ch)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ unsigned int timeout = TALITOS_TIMEOUT;
+
+ setbits32(priv->chan[ch].reg + TALITOS_CCCR, TALITOS_CCCR_RESET);
+
+ while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) & TALITOS_CCCR_RESET)
+ && --timeout)
+ cpu_relax();
+
+ if (timeout == 0) {
+ dev_err(dev, "failed to reset channel %d\n", ch);
+ return -EIO;
+ }
+
+ /* set 36-bit addressing, done writeback enable and done IRQ enable */
+ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
+ TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
+
+ /* and ICCR writeback, if available */
+ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
+ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
+ TALITOS_CCCR_LO_IWSE);
+
+ return 0;
+}
+
+static int reset_device(struct device *dev)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ unsigned int timeout = TALITOS_TIMEOUT;
+ u32 mcr = TALITOS_MCR_SWR;
+
+ setbits32(priv->reg + TALITOS_MCR, mcr);
+
+ while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
+ && --timeout)
+ cpu_relax();
+
+ if (priv->irq[1]) {
+ mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
+ setbits32(priv->reg + TALITOS_MCR, mcr);
+ }
+
+ if (timeout == 0) {
+ dev_err(dev, "failed to reset device\n");
+ return -EIO;
+ }
+
+ return 0;
+}
+
+/*
+ * Reset and initialize the device
+ */
+static int init_device(struct device *dev)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ int ch, err;
+
+ /*
+ * Master reset
+ * errata documentation: warning: certain SEC interrupts
+ * are not fully cleared by writing the MCR:SWR bit,
+ * set bit twice to completely reset
+ */
+ err = reset_device(dev);
+ if (err)
+ return err;
+
+ err = reset_device(dev);
+ if (err)
+ return err;
+
+ /* reset channels */
+ for (ch = 0; ch < priv->num_channels; ch++) {
+ err = reset_channel(dev, ch);
+ if (err)
+ return err;
+ }
+
+ /* enable channel done and error interrupts */
+ setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
+ setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
+
+ /* disable integrity check error interrupts (use writeback instead) */
+ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
+ setbits32(priv->reg + TALITOS_MDEUICR_LO,
+ TALITOS_MDEUICR_LO_ICE);
+
+ return 0;
+}
+
+/**
+ * talitos_submit - submits a descriptor to the device for processing
+ * @dev: the SEC device to be used
+ * @ch: the SEC device channel to be used
+ * @desc: the descriptor to be processed by the device
+ * @callback: whom to call when processing is complete
+ * @context: a handle for use by caller (optional)
+ *
+ * desc must contain valid dma-mapped (bus physical) address pointers.
+ * callback must check err and feedback in descriptor header
+ * for device processing status.
+ */
+static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
+ void (*callback)(struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error),
+ void *context)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ struct talitos_request *request;
+ unsigned long flags;
+ int head;
+
+ spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
+
+ if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
+ /* h/w fifo is full */
+ spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
+ return -EAGAIN;
+ }
+
+ head = priv->chan[ch].head;
+ request = &priv->chan[ch].fifo[head];
+
+ /* map descriptor and save caller data */
+ request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
+ DMA_BIDIRECTIONAL);
+ request->callback = callback;
+ request->context = context;
+
+ /* increment fifo head */
+ priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
+
+ smp_wmb();
+ request->desc = desc;
+
+ /* GO! */
+ wmb();
+ out_be32(priv->chan[ch].reg + TALITOS_FF,
+ upper_32_bits(request->dma_desc));
+ out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
+ lower_32_bits(request->dma_desc));
+
+ spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
+
+ return -EINPROGRESS;
+}
+
+/*
+ * process what was done, notify callback of error if not
+ */
+static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ struct talitos_request *request, saved_req;
+ unsigned long flags;
+ int tail, status;
+
+ spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
+
+ tail = priv->chan[ch].tail;
+ while (priv->chan[ch].fifo[tail].desc) {
+ request = &priv->chan[ch].fifo[tail];
+
+ /* descriptors with their done bits set don't get the error */
+ rmb();
+ if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
+ status = 0;
+ else
+ if (!error)
+ break;
+ else
+ status = error;
+
+ dma_unmap_single(dev, request->dma_desc,
+ sizeof(struct talitos_desc),
+ DMA_BIDIRECTIONAL);
+
+ /* copy entries so we can call callback outside lock */
+ saved_req.desc = request->desc;
+ saved_req.callback = request->callback;
+ saved_req.context = request->context;
+
+ /* release request entry in fifo */
+ smp_wmb();
+ request->desc = NULL;
+
+ /* increment fifo tail */
+ priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
+
+ spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
+
+ atomic_dec(&priv->chan[ch].submit_count);
+
+ saved_req.callback(dev, saved_req.desc, saved_req.context,
+ status);
+ /* channel may resume processing in single desc error case */
+ if (error && !reset_ch && status == error)
+ return;
+ spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
+ tail = priv->chan[ch].tail;
+ }
+
+ spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
+}
+
+/*
+ * process completed requests for channels that have done status
+ */
+#define DEF_TALITOS_DONE(name, ch_done_mask) \
+static void talitos_done_##name(unsigned long data) \
+{ \
+ struct device *dev = (struct device *)data; \
+ struct talitos_private *priv = dev_get_drvdata(dev); \
+ unsigned long flags; \
+ \
+ if (ch_done_mask & 1) \
+ flush_channel(dev, 0, 0, 0); \
+ if (priv->num_channels == 1) \
+ goto out; \
+ if (ch_done_mask & (1 << 2)) \
+ flush_channel(dev, 1, 0, 0); \
+ if (ch_done_mask & (1 << 4)) \
+ flush_channel(dev, 2, 0, 0); \
+ if (ch_done_mask & (1 << 6)) \
+ flush_channel(dev, 3, 0, 0); \
+ \
+out: \
+ /* At this point, all completed channels have been processed */ \
+ /* Unmask done interrupts for channels completed later on. */ \
+ spin_lock_irqsave(&priv->reg_lock, flags); \
+ setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
+ setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \
+ spin_unlock_irqrestore(&priv->reg_lock, flags); \
+}
+DEF_TALITOS_DONE(4ch, TALITOS_ISR_4CHDONE)
+DEF_TALITOS_DONE(ch0_2, TALITOS_ISR_CH_0_2_DONE)
+DEF_TALITOS_DONE(ch1_3, TALITOS_ISR_CH_1_3_DONE)
+
+/*
+ * locate current (offending) descriptor
+ */
+static u32 current_desc_hdr(struct device *dev, int ch)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ int tail = priv->chan[ch].tail;
+ dma_addr_t cur_desc;
+
+ cur_desc = in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
+
+ while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
+ tail = (tail + 1) & (priv->fifo_len - 1);
+ if (tail == priv->chan[ch].tail) {
+ dev_err(dev, "couldn't locate current descriptor\n");
+ return 0;
+ }
+ }
+
+ return priv->chan[ch].fifo[tail].desc->hdr;
+}
+
+/*
+ * user diagnostics; report root cause of error based on execution unit status
+ */
+static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ int i;
+
+ if (!desc_hdr)
+ desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
+
+ switch (desc_hdr & DESC_HDR_SEL0_MASK) {
+ case DESC_HDR_SEL0_AFEU:
+ dev_err(dev, "AFEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_AFEUISR),
+ in_be32(priv->reg + TALITOS_AFEUISR_LO));
+ break;
+ case DESC_HDR_SEL0_DEU:
+ dev_err(dev, "DEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_DEUISR),
+ in_be32(priv->reg + TALITOS_DEUISR_LO));
+ break;
+ case DESC_HDR_SEL0_MDEUA:
+ case DESC_HDR_SEL0_MDEUB:
+ dev_err(dev, "MDEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_MDEUISR),
+ in_be32(priv->reg + TALITOS_MDEUISR_LO));
+ break;
+ case DESC_HDR_SEL0_RNG:
+ dev_err(dev, "RNGUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_RNGUISR),
+ in_be32(priv->reg + TALITOS_RNGUISR_LO));
+ break;
+ case DESC_HDR_SEL0_PKEU:
+ dev_err(dev, "PKEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_PKEUISR),
+ in_be32(priv->reg + TALITOS_PKEUISR_LO));
+ break;
+ case DESC_HDR_SEL0_AESU:
+ dev_err(dev, "AESUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_AESUISR),
+ in_be32(priv->reg + TALITOS_AESUISR_LO));
+ break;
+ case DESC_HDR_SEL0_CRCU:
+ dev_err(dev, "CRCUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_CRCUISR),
+ in_be32(priv->reg + TALITOS_CRCUISR_LO));
+ break;
+ case DESC_HDR_SEL0_KEU:
+ dev_err(dev, "KEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_KEUISR),
+ in_be32(priv->reg + TALITOS_KEUISR_LO));
+ break;
+ }
+
+ switch (desc_hdr & DESC_HDR_SEL1_MASK) {
+ case DESC_HDR_SEL1_MDEUA:
+ case DESC_HDR_SEL1_MDEUB:
+ dev_err(dev, "MDEUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_MDEUISR),
+ in_be32(priv->reg + TALITOS_MDEUISR_LO));
+ break;
+ case DESC_HDR_SEL1_CRCU:
+ dev_err(dev, "CRCUISR 0x%08x_%08x\n",
+ in_be32(priv->reg + TALITOS_CRCUISR),
+ in_be32(priv->reg + TALITOS_CRCUISR_LO));
+ break;
+ }
+
+ for (i = 0; i < 8; i++)
+ dev_err(dev, "DESCBUF 0x%08x_%08x\n",
+ in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
+ in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
+}
+
+/*
+ * recover from error interrupts
+ */
+static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ unsigned int timeout = TALITOS_TIMEOUT;
+ int ch, error, reset_dev = 0, reset_ch = 0;
+ u32 v, v_lo;
+
+ for (ch = 0; ch < priv->num_channels; ch++) {
+ /* skip channels without errors */
+ if (!(isr & (1 << (ch * 2 + 1))))
+ continue;
+
+ error = -EINVAL;
+
+ v = in_be32(priv->chan[ch].reg + TALITOS_CCPSR);
+ v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
+
+ if (v_lo & TALITOS_CCPSR_LO_DOF) {
+ dev_err(dev, "double fetch fifo overflow error\n");
+ error = -EAGAIN;
+ reset_ch = 1;
+ }
+ if (v_lo & TALITOS_CCPSR_LO_SOF) {
+ /* h/w dropped descriptor */
+ dev_err(dev, "single fetch fifo overflow error\n");
+ error = -EAGAIN;
+ }
+ if (v_lo & TALITOS_CCPSR_LO_MDTE)
+ dev_err(dev, "master data transfer error\n");
+ if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
+ dev_err(dev, "s/g data length zero error\n");
+ if (v_lo & TALITOS_CCPSR_LO_FPZ)
+ dev_err(dev, "fetch pointer zero error\n");
+ if (v_lo & TALITOS_CCPSR_LO_IDH)
+ dev_err(dev, "illegal descriptor header error\n");
+ if (v_lo & TALITOS_CCPSR_LO_IEU)
+ dev_err(dev, "invalid execution unit error\n");
+ if (v_lo & TALITOS_CCPSR_LO_EU)
+ report_eu_error(dev, ch, current_desc_hdr(dev, ch));
+ if (v_lo & TALITOS_CCPSR_LO_GB)
+ dev_err(dev, "gather boundary error\n");
+ if (v_lo & TALITOS_CCPSR_LO_GRL)
+ dev_err(dev, "gather return/length error\n");
+ if (v_lo & TALITOS_CCPSR_LO_SB)
+ dev_err(dev, "scatter boundary error\n");
+ if (v_lo & TALITOS_CCPSR_LO_SRL)
+ dev_err(dev, "scatter return/length error\n");
+
+ flush_channel(dev, ch, error, reset_ch);
+
+ if (reset_ch) {
+ reset_channel(dev, ch);
+ } else {
+ setbits32(priv->chan[ch].reg + TALITOS_CCCR,
+ TALITOS_CCCR_CONT);
+ setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
+ while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
+ TALITOS_CCCR_CONT) && --timeout)
+ cpu_relax();
+ if (timeout == 0) {
+ dev_err(dev, "failed to restart channel %d\n",
+ ch);
+ reset_dev = 1;
+ }
+ }
+ }
+ if (reset_dev || isr & ~TALITOS_ISR_4CHERR || isr_lo) {
+ dev_err(dev, "done overflow, internal time out, or rngu error: "
+ "ISR 0x%08x_%08x\n", isr, isr_lo);
+
+ /* purge request queues */
+ for (ch = 0; ch < priv->num_channels; ch++)
+ flush_channel(dev, ch, -EIO, 1);
+
+ /* reset and reinitialize the device */
+ init_device(dev);
+ }
+}
+
+#define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
+static irqreturn_t talitos_interrupt_##name(int irq, void *data) \
+{ \
+ struct device *dev = data; \
+ struct talitos_private *priv = dev_get_drvdata(dev); \
+ u32 isr, isr_lo; \
+ unsigned long flags; \
+ \
+ spin_lock_irqsave(&priv->reg_lock, flags); \
+ isr = in_be32(priv->reg + TALITOS_ISR); \
+ isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
+ /* Acknowledge interrupt */ \
+ out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
+ out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
+ \
+ if (unlikely(isr & ch_err_mask || isr_lo)) { \
+ spin_unlock_irqrestore(&priv->reg_lock, flags); \
+ talitos_error(dev, isr & ch_err_mask, isr_lo); \
+ } \
+ else { \
+ if (likely(isr & ch_done_mask)) { \
+ /* mask further done interrupts. */ \
+ clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
+ /* done_task will unmask done interrupts at exit */ \
+ tasklet_schedule(&priv->done_task[tlet]); \
+ } \
+ spin_unlock_irqrestore(&priv->reg_lock, flags); \
+ } \
+ \
+ return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
+ IRQ_NONE; \
+}
+DEF_TALITOS_INTERRUPT(4ch, TALITOS_ISR_4CHDONE, TALITOS_ISR_4CHERR, 0)
+DEF_TALITOS_INTERRUPT(ch0_2, TALITOS_ISR_CH_0_2_DONE, TALITOS_ISR_CH_0_2_ERR, 0)
+DEF_TALITOS_INTERRUPT(ch1_3, TALITOS_ISR_CH_1_3_DONE, TALITOS_ISR_CH_1_3_ERR, 1)
+
+/*
+ * hwrng
+ */
+static int talitos_rng_data_present(struct hwrng *rng, int wait)
+{
+ struct device *dev = (struct device *)rng->priv;
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ u32 ofl;
+ int i;
+
+ for (i = 0; i < 20; i++) {
+ ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
+ TALITOS_RNGUSR_LO_OFL;
+ if (ofl || !wait)
+ break;
+ udelay(10);
+ }
+
+ return !!ofl;
+}
+
+static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
+{
+ struct device *dev = (struct device *)rng->priv;
+ struct talitos_private *priv = dev_get_drvdata(dev);
+
+ /* rng fifo requires 64-bit accesses */
+ *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
+ *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
+
+ return sizeof(u32);
+}
+
+static int talitos_rng_init(struct hwrng *rng)
+{
+ struct device *dev = (struct device *)rng->priv;
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ unsigned int timeout = TALITOS_TIMEOUT;
+
+ setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
+ while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
+ && --timeout)
+ cpu_relax();
+ if (timeout == 0) {
+ dev_err(dev, "failed to reset rng hw\n");
+ return -ENODEV;
+ }
+
+ /* start generating */
+ setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
+
+ return 0;
+}
+
+static int talitos_register_rng(struct device *dev)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+
+ priv->rng.name = dev_driver_string(dev),
+ priv->rng.init = talitos_rng_init,
+ priv->rng.data_present = talitos_rng_data_present,
+ priv->rng.data_read = talitos_rng_data_read,
+ priv->rng.priv = (unsigned long)dev;
+
+ return hwrng_register(&priv->rng);
+}
+
+static void talitos_unregister_rng(struct device *dev)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+
+ hwrng_unregister(&priv->rng);
+}
+
+/*
+ * crypto alg
+ */
+#define TALITOS_CRA_PRIORITY 3000
+#define TALITOS_MAX_KEY_SIZE 64
+#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
+
+#define MD5_BLOCK_SIZE 64
+
+struct talitos_ctx {
+ struct device *dev;
+ int ch;
+ __be32 desc_hdr_template;
+ u8 key[TALITOS_MAX_KEY_SIZE];
+ u8 iv[TALITOS_MAX_IV_LENGTH];
+ unsigned int keylen;
+ unsigned int enckeylen;
+ unsigned int authkeylen;
+ unsigned int authsize;
+};
+
+#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
+#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
+
+struct talitos_ahash_req_ctx {
+ u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
+ unsigned int hw_context_size;
+ u8 buf[HASH_MAX_BLOCK_SIZE];
+ u8 bufnext[HASH_MAX_BLOCK_SIZE];
+ unsigned int swinit;
+ unsigned int first;
+ unsigned int last;
+ unsigned int to_hash_later;
+ u64 nbuf;
+ struct scatterlist bufsl[2];
+ struct scatterlist *psrc;
+};
+
+static int aead_setauthsize(struct crypto_aead *authenc,
+ unsigned int authsize)
+{
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+
+ ctx->authsize = authsize;
+
+ return 0;
+}
+
+static int aead_setkey(struct crypto_aead *authenc,
+ const u8 *key, unsigned int keylen)
+{
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ struct rtattr *rta = (void *)key;
+ struct crypto_authenc_key_param *param;
+ unsigned int authkeylen;
+ unsigned int enckeylen;
+
+ if (!RTA_OK(rta, keylen))
+ goto badkey;
+
+ if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
+ goto badkey;
+
+ if (RTA_PAYLOAD(rta) < sizeof(*param))
+ goto badkey;
+
+ param = RTA_DATA(rta);
+ enckeylen = be32_to_cpu(param->enckeylen);
+
+ key += RTA_ALIGN(rta->rta_len);
+ keylen -= RTA_ALIGN(rta->rta_len);
+
+ if (keylen < enckeylen)
+ goto badkey;
+
+ authkeylen = keylen - enckeylen;
+
+ if (keylen > TALITOS_MAX_KEY_SIZE)
+ goto badkey;
+
+ memcpy(&ctx->key, key, keylen);
+
+ ctx->keylen = keylen;
+ ctx->enckeylen = enckeylen;
+ ctx->authkeylen = authkeylen;
+
+ return 0;
+
+badkey:
+ crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+}
+
+/*
+ * talitos_edesc - s/w-extended descriptor
+ * @src_nents: number of segments in input scatterlist
+ * @dst_nents: number of segments in output scatterlist
+ * @dma_len: length of dma mapped link_tbl space
+ * @dma_link_tbl: bus physical address of link_tbl
+ * @desc: h/w descriptor
+ * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
+ *
+ * if decrypting (with authcheck), or either one of src_nents or dst_nents
+ * is greater than 1, an integrity check value is concatenated to the end
+ * of link_tbl data
+ */
+struct talitos_edesc {
+ int src_nents;
+ int dst_nents;
+ int src_is_chained;
+ int dst_is_chained;
+ int dma_len;
+ dma_addr_t dma_link_tbl;
+ struct talitos_desc desc;
+ struct talitos_ptr link_tbl[0];
+};
+
+static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
+ unsigned int nents, enum dma_data_direction dir,
+ int chained)
+{
+ if (unlikely(chained))
+ while (sg) {
+ dma_map_sg(dev, sg, 1, dir);
+ sg = scatterwalk_sg_next(sg);
+ }
+ else
+ dma_map_sg(dev, sg, nents, dir);
+ return nents;
+}
+
+static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
+ enum dma_data_direction dir)
+{
+ while (sg) {
+ dma_unmap_sg(dev, sg, 1, dir);
+ sg = scatterwalk_sg_next(sg);
+ }
+}
+
+static void talitos_sg_unmap(struct device *dev,
+ struct talitos_edesc *edesc,
+ struct scatterlist *src,
+ struct scatterlist *dst)
+{
+ unsigned int src_nents = edesc->src_nents ? : 1;
+ unsigned int dst_nents = edesc->dst_nents ? : 1;
+
+ if (src != dst) {
+ if (edesc->src_is_chained)
+ talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
+ else
+ dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
+
+ if (dst) {
+ if (edesc->dst_is_chained)
+ talitos_unmap_sg_chain(dev, dst,
+ DMA_FROM_DEVICE);
+ else
+ dma_unmap_sg(dev, dst, dst_nents,
+ DMA_FROM_DEVICE);
+ }
+ } else
+ if (edesc->src_is_chained)
+ talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
+ else
+ dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
+}
+
+static void ipsec_esp_unmap(struct device *dev,
+ struct talitos_edesc *edesc,
+ struct aead_request *areq)
+{
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
+
+ dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
+
+ talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
+
+ if (edesc->dma_len)
+ dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+}
+
+/*
+ * ipsec_esp descriptor callbacks
+ */
+static void ipsec_esp_encrypt_done(struct device *dev,
+ struct talitos_desc *desc, void *context,
+ int err)
+{
+ struct aead_request *areq = context;
+ struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ struct talitos_edesc *edesc;
+ struct scatterlist *sg;
+ void *icvdata;
+
+ edesc = container_of(desc, struct talitos_edesc, desc);
+
+ ipsec_esp_unmap(dev, edesc, areq);
+
+ /* copy the generated ICV to dst */
+ if (edesc->dma_len) {
+ icvdata = &edesc->link_tbl[edesc->src_nents +
+ edesc->dst_nents + 2];
+ sg = sg_last(areq->dst, edesc->dst_nents);
+ memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
+ icvdata, ctx->authsize);
+ }
+
+ kfree(edesc);
+
+ aead_request_complete(areq, err);
+}
+
+static void ipsec_esp_decrypt_swauth_done(struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int err)
+{
+ struct aead_request *req = context;
+ struct crypto_aead *authenc = crypto_aead_reqtfm(req);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ struct talitos_edesc *edesc;
+ struct scatterlist *sg;
+ void *icvdata;
+
+ edesc = container_of(desc, struct talitos_edesc, desc);
+
+ ipsec_esp_unmap(dev, edesc, req);
+
+ if (!err) {
+ /* auth check */
+ if (edesc->dma_len)
+ icvdata = &edesc->link_tbl[edesc->src_nents +
+ edesc->dst_nents + 2];
+ else
+ icvdata = &edesc->link_tbl[0];
+
+ sg = sg_last(req->dst, edesc->dst_nents ? : 1);
+ err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
+ ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
+ }
+
+ kfree(edesc);
+
+ aead_request_complete(req, err);
+}
+
+static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int err)
+{
+ struct aead_request *req = context;
+ struct talitos_edesc *edesc;
+
+ edesc = container_of(desc, struct talitos_edesc, desc);
+
+ ipsec_esp_unmap(dev, edesc, req);
+
+ /* check ICV auth status */
+ if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
+ DESC_HDR_LO_ICCR1_PASS))
+ err = -EBADMSG;
+
+ kfree(edesc);
+
+ aead_request_complete(req, err);
+}
+
+/*
+ * convert scatterlist to SEC h/w link table format
+ * stop at cryptlen bytes
+ */
+static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
+ int cryptlen, struct talitos_ptr *link_tbl_ptr)
+{
+ int n_sg = sg_count;
+
+ while (n_sg--) {
+ to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg));
+ link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
+ link_tbl_ptr->j_extent = 0;
+ link_tbl_ptr++;
+ cryptlen -= sg_dma_len(sg);
+ sg = scatterwalk_sg_next(sg);
+ }
+
+ /* adjust (decrease) last one (or two) entry's len to cryptlen */
+ link_tbl_ptr--;
+ while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
+ /* Empty this entry, and move to previous one */
+ cryptlen += be16_to_cpu(link_tbl_ptr->len);
+ link_tbl_ptr->len = 0;
+ sg_count--;
+ link_tbl_ptr--;
+ }
+ link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
+ + cryptlen);
+
+ /* tag end of link table */
+ link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
+
+ return sg_count;
+}
+
+/*
+ * fill in and submit ipsec_esp descriptor
+ */
+static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
+ u8 *giv, u64 seq,
+ void (*callback) (struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error))
+{
+ struct crypto_aead *aead = crypto_aead_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_aead_ctx(aead);
+ struct device *dev = ctx->dev;
+ struct talitos_desc *desc = &edesc->desc;
+ unsigned int cryptlen = areq->cryptlen;
+ unsigned int authsize = ctx->authsize;
+ unsigned int ivsize = crypto_aead_ivsize(aead);
+ int sg_count, ret;
+ int sg_link_tbl_len;
+
+ /* hmac key */
+ map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
+ 0, DMA_TO_DEVICE);
+ /* hmac data */
+ map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
+ sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
+ /* cipher iv */
+ map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
+ DMA_TO_DEVICE);
+
+ /* cipher key */
+ map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
+ (char *)&ctx->key + ctx->authkeylen, 0,
+ DMA_TO_DEVICE);
+
+ /*
+ * cipher in
+ * map and adjust cipher len to aead request cryptlen.
+ * extent is bytes of HMAC postpended to ciphertext,
+ * typically 12 for ipsec
+ */
+ desc->ptr[4].len = cpu_to_be16(cryptlen);
+ desc->ptr[4].j_extent = authsize;
+
+ sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
+ (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
+ : DMA_TO_DEVICE,
+ edesc->src_is_chained);
+
+ if (sg_count == 1) {
+ to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src));
+ } else {
+ sg_link_tbl_len = cryptlen;
+
+ if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
+ sg_link_tbl_len = cryptlen + authsize;
+
+ sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
+ &edesc->link_tbl[0]);
+ if (sg_count > 1) {
+ desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
+ to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl);
+ dma_sync_single_for_device(dev, edesc->dma_link_tbl,
+ edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+ } else {
+ /* Only one segment now, so no link tbl needed */
+ to_talitos_ptr(&desc->ptr[4],
+ sg_dma_address(areq->src));
+ }
+ }
+
+ /* cipher out */
+ desc->ptr[5].len = cpu_to_be16(cryptlen);
+ desc->ptr[5].j_extent = authsize;
+
+ if (areq->src != areq->dst)
+ sg_count = talitos_map_sg(dev, areq->dst,
+ edesc->dst_nents ? : 1,
+ DMA_FROM_DEVICE,
+ edesc->dst_is_chained);
+
+ if (sg_count == 1) {
+ to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst));
+ } else {
+ struct talitos_ptr *link_tbl_ptr =
+ &edesc->link_tbl[edesc->src_nents + 1];
+
+ to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl +
+ (edesc->src_nents + 1) *
+ sizeof(struct talitos_ptr));
+ sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
+ link_tbl_ptr);
+
+ /* Add an entry to the link table for ICV data */
+ link_tbl_ptr += sg_count - 1;
+ link_tbl_ptr->j_extent = 0;
+ sg_count++;
+ link_tbl_ptr++;
+ link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
+ link_tbl_ptr->len = cpu_to_be16(authsize);
+
+ /* icv data follows link tables */
+ to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl +
+ (edesc->src_nents + edesc->dst_nents + 2) *
+ sizeof(struct talitos_ptr));
+ desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
+ dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
+ edesc->dma_len, DMA_BIDIRECTIONAL);
+ }
+
+ /* iv out */
+ map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
+ DMA_FROM_DEVICE);
+
+ ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
+ if (ret != -EINPROGRESS) {
+ ipsec_esp_unmap(dev, edesc, areq);
+ kfree(edesc);
+ }
+ return ret;
+}
+
+/*
+ * derive number of elements in scatterlist
+ */
+static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
+{
+ struct scatterlist *sg = sg_list;
+ int sg_nents = 0;
+
+ *chained = 0;
+ while (nbytes > 0) {
+ sg_nents++;
+ nbytes -= sg->length;
+ if (!sg_is_last(sg) && (sg + 1)->length == 0)
+ *chained = 1;
+ sg = scatterwalk_sg_next(sg);
+ }
+
+ return sg_nents;
+}
+
+/**
+ * sg_copy_end_to_buffer - Copy end data from SG list to a linear buffer
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy to
+ * @buflen: The number of bytes to copy
+ * @skip: The number of bytes to skip before copying.
+ * Note: skip + buflen should equal SG total size.
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+static size_t sg_copy_end_to_buffer(struct scatterlist *sgl, unsigned int nents,
+ void *buf, size_t buflen, unsigned int skip)
+{
+ unsigned int offset = 0;
+ unsigned int boffset = 0;
+ struct sg_mapping_iter miter;
+ unsigned long flags;
+ unsigned int sg_flags = SG_MITER_ATOMIC;
+ size_t total_buffer = buflen + skip;
+
+ sg_flags |= SG_MITER_FROM_SG;
+
+ sg_miter_start(&miter, sgl, nents, sg_flags);
+
+ local_irq_save(flags);
+
+ while (sg_miter_next(&miter) && offset < total_buffer) {
+ unsigned int len;
+ unsigned int ignore;
+
+ if ((offset + miter.length) > skip) {
+ if (offset < skip) {
+ /* Copy part of this segment */
+ ignore = skip - offset;
+ len = miter.length - ignore;
+ if (boffset + len > buflen)
+ len = buflen - boffset;
+ memcpy(buf + boffset, miter.addr + ignore, len);
+ } else {
+ /* Copy all of this segment (up to buflen) */
+ len = miter.length;
+ if (boffset + len > buflen)
+ len = buflen - boffset;
+ memcpy(buf + boffset, miter.addr, len);
+ }
+ boffset += len;
+ }
+ offset += miter.length;
+ }
+
+ sg_miter_stop(&miter);
+
+ local_irq_restore(flags);
+ return boffset;
+}
+
+/*
+ * allocate and map the extended descriptor
+ */
+static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
+ struct scatterlist *src,
+ struct scatterlist *dst,
+ int hash_result,
+ unsigned int cryptlen,
+ unsigned int authsize,
+ int icv_stashing,
+ u32 cryptoflags)
+{
+ struct talitos_edesc *edesc;
+ int src_nents, dst_nents, alloc_len, dma_len;
+ int src_chained, dst_chained = 0;
+ gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
+ GFP_ATOMIC;
+
+ if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
+ dev_err(dev, "length exceeds h/w max limit\n");
+ return ERR_PTR(-EINVAL);
+ }
+
+ src_nents = sg_count(src, cryptlen + authsize, &src_chained);
+ src_nents = (src_nents == 1) ? 0 : src_nents;
+
+ if (hash_result) {
+ dst_nents = 0;
+ } else {
+ if (dst == src) {
+ dst_nents = src_nents;
+ } else {
+ dst_nents = sg_count(dst, cryptlen + authsize,
+ &dst_chained);
+ dst_nents = (dst_nents == 1) ? 0 : dst_nents;
+ }
+ }
+
+ /*
+ * allocate space for base edesc plus the link tables,
+ * allowing for two separate entries for ICV and generated ICV (+ 2),
+ * and the ICV data itself
+ */
+ alloc_len = sizeof(struct talitos_edesc);
+ if (src_nents || dst_nents) {
+ dma_len = (src_nents + dst_nents + 2) *
+ sizeof(struct talitos_ptr) + authsize;
+ alloc_len += dma_len;
+ } else {
+ dma_len = 0;
+ alloc_len += icv_stashing ? authsize : 0;
+ }
+
+ edesc = kmalloc(alloc_len, GFP_DMA | flags);
+ if (!edesc) {
+ dev_err(dev, "could not allocate edescriptor\n");
+ return ERR_PTR(-ENOMEM);
+ }
+
+ edesc->src_nents = src_nents;
+ edesc->dst_nents = dst_nents;
+ edesc->src_is_chained = src_chained;
+ edesc->dst_is_chained = dst_chained;
+ edesc->dma_len = dma_len;
+ if (dma_len)
+ edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
+ edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+
+ return edesc;
+}
+
+static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
+ int icv_stashing)
+{
+ struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+
+ return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
+ areq->cryptlen, ctx->authsize, icv_stashing,
+ areq->base.flags);
+}
+
+static int aead_encrypt(struct aead_request *req)
+{
+ struct crypto_aead *authenc = crypto_aead_reqtfm(req);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ struct talitos_edesc *edesc;
+
+ /* allocate extended descriptor */
+ edesc = aead_edesc_alloc(req, 0);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* set encrypt */
+ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
+
+ return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
+}
+
+static int aead_decrypt(struct aead_request *req)
+{
+ struct crypto_aead *authenc = crypto_aead_reqtfm(req);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ unsigned int authsize = ctx->authsize;
+ struct talitos_private *priv = dev_get_drvdata(ctx->dev);
+ struct talitos_edesc *edesc;
+ struct scatterlist *sg;
+ void *icvdata;
+
+ req->cryptlen -= authsize;
+
+ /* allocate extended descriptor */
+ edesc = aead_edesc_alloc(req, 1);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
+ ((!edesc->src_nents && !edesc->dst_nents) ||
+ priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
+
+ /* decrypt and check the ICV */
+ edesc->desc.hdr = ctx->desc_hdr_template |
+ DESC_HDR_DIR_INBOUND |
+ DESC_HDR_MODE1_MDEU_CICV;
+
+ /* reset integrity check result bits */
+ edesc->desc.hdr_lo = 0;
+
+ return ipsec_esp(edesc, req, NULL, 0,
+ ipsec_esp_decrypt_hwauth_done);
+
+ }
+
+ /* Have to check the ICV with software */
+ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
+
+ /* stash incoming ICV for later cmp with ICV generated by the h/w */
+ if (edesc->dma_len)
+ icvdata = &edesc->link_tbl[edesc->src_nents +
+ edesc->dst_nents + 2];
+ else
+ icvdata = &edesc->link_tbl[0];
+
+ sg = sg_last(req->src, edesc->src_nents ? : 1);
+
+ memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
+ ctx->authsize);
+
+ return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
+}
+
+static int aead_givencrypt(struct aead_givcrypt_request *req)
+{
+ struct aead_request *areq = &req->areq;
+ struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
+ struct talitos_edesc *edesc;
+
+ /* allocate extended descriptor */
+ edesc = aead_edesc_alloc(areq, 0);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* set encrypt */
+ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
+
+ memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
+ /* avoid consecutive packets going out with same IV */
+ *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
+
+ return ipsec_esp(edesc, areq, req->giv, req->seq,
+ ipsec_esp_encrypt_done);
+}
+
+static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
+ const u8 *key, unsigned int keylen)
+{
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+
+ memcpy(&ctx->key, key, keylen);
+ ctx->keylen = keylen;
+
+ return 0;
+}
+
+static void common_nonsnoop_unmap(struct device *dev,
+ struct talitos_edesc *edesc,
+ struct ablkcipher_request *areq)
+{
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
+
+ talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
+
+ if (edesc->dma_len)
+ dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+}
+
+static void ablkcipher_done(struct device *dev,
+ struct talitos_desc *desc, void *context,
+ int err)
+{
+ struct ablkcipher_request *areq = context;
+ struct talitos_edesc *edesc;
+
+ edesc = container_of(desc, struct talitos_edesc, desc);
+
+ common_nonsnoop_unmap(dev, edesc, areq);
+
+ kfree(edesc);
+
+ areq->base.complete(&areq->base, err);
+}
+
+static int common_nonsnoop(struct talitos_edesc *edesc,
+ struct ablkcipher_request *areq,
+ void (*callback) (struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error))
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ struct device *dev = ctx->dev;
+ struct talitos_desc *desc = &edesc->desc;
+ unsigned int cryptlen = areq->nbytes;
+ unsigned int ivsize;
+ int sg_count, ret;
+
+ /* first DWORD empty */
+ desc->ptr[0].len = 0;
+ to_talitos_ptr(&desc->ptr[0], 0);
+ desc->ptr[0].j_extent = 0;
+
+ /* cipher iv */
+ ivsize = crypto_ablkcipher_ivsize(cipher);
+ map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, areq->info, 0,
+ DMA_TO_DEVICE);
+
+ /* cipher key */
+ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
+ (char *)&ctx->key, 0, DMA_TO_DEVICE);
+
+ /*
+ * cipher in
+ */
+ desc->ptr[3].len = cpu_to_be16(cryptlen);
+ desc->ptr[3].j_extent = 0;
+
+ sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
+ (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
+ : DMA_TO_DEVICE,
+ edesc->src_is_chained);
+
+ if (sg_count == 1) {
+ to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src));
+ } else {
+ sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
+ &edesc->link_tbl[0]);
+ if (sg_count > 1) {
+ to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
+ desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
+ dma_sync_single_for_device(dev, edesc->dma_link_tbl,
+ edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+ } else {
+ /* Only one segment now, so no link tbl needed */
+ to_talitos_ptr(&desc->ptr[3],
+ sg_dma_address(areq->src));
+ }
+ }
+
+ /* cipher out */
+ desc->ptr[4].len = cpu_to_be16(cryptlen);
+ desc->ptr[4].j_extent = 0;
+
+ if (areq->src != areq->dst)
+ sg_count = talitos_map_sg(dev, areq->dst,
+ edesc->dst_nents ? : 1,
+ DMA_FROM_DEVICE,
+ edesc->dst_is_chained);
+
+ if (sg_count == 1) {
+ to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst));
+ } else {
+ struct talitos_ptr *link_tbl_ptr =
+ &edesc->link_tbl[edesc->src_nents + 1];
+
+ to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl +
+ (edesc->src_nents + 1) *
+ sizeof(struct talitos_ptr));
+ desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
+ sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
+ link_tbl_ptr);
+ dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
+ edesc->dma_len, DMA_BIDIRECTIONAL);
+ }
+
+ /* iv out */
+ map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0,
+ DMA_FROM_DEVICE);
+
+ /* last DWORD empty */
+ desc->ptr[6].len = 0;
+ to_talitos_ptr(&desc->ptr[6], 0);
+ desc->ptr[6].j_extent = 0;
+
+ ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
+ if (ret != -EINPROGRESS) {
+ common_nonsnoop_unmap(dev, edesc, areq);
+ kfree(edesc);
+ }
+ return ret;
+}
+
+static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
+ areq)
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+
+ return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, 0,
+ areq->nbytes, 0, 0, areq->base.flags);
+}
+
+static int ablkcipher_encrypt(struct ablkcipher_request *areq)
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ struct talitos_edesc *edesc;
+
+ /* allocate extended descriptor */
+ edesc = ablkcipher_edesc_alloc(areq);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ /* set encrypt */
+ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
+
+ return common_nonsnoop(edesc, areq, ablkcipher_done);
+}
+
+static int ablkcipher_decrypt(struct ablkcipher_request *areq)
+{
+ struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
+ struct talitos_edesc *edesc;
+
+ /* allocate extended descriptor */
+ edesc = ablkcipher_edesc_alloc(areq);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
+
+ return common_nonsnoop(edesc, areq, ablkcipher_done);
+}
+
+static void common_nonsnoop_hash_unmap(struct device *dev,
+ struct talitos_edesc *edesc,
+ struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
+
+ /* When using hashctx-in, must unmap it. */
+ if (edesc->desc.ptr[1].len)
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
+ DMA_TO_DEVICE);
+
+ if (edesc->desc.ptr[2].len)
+ unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2],
+ DMA_TO_DEVICE);
+
+ talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL);
+
+ if (edesc->dma_len)
+ dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+
+}
+
+static void ahash_done(struct device *dev,
+ struct talitos_desc *desc, void *context,
+ int err)
+{
+ struct ahash_request *areq = context;
+ struct talitos_edesc *edesc =
+ container_of(desc, struct talitos_edesc, desc);
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ if (!req_ctx->last && req_ctx->to_hash_later) {
+ /* Position any partial block for next update/final/finup */
+ memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
+ req_ctx->nbuf = req_ctx->to_hash_later;
+ }
+ common_nonsnoop_hash_unmap(dev, edesc, areq);
+
+ kfree(edesc);
+
+ areq->base.complete(&areq->base, err);
+}
+
+static int common_nonsnoop_hash(struct talitos_edesc *edesc,
+ struct ahash_request *areq, unsigned int length,
+ void (*callback) (struct device *dev,
+ struct talitos_desc *desc,
+ void *context, int error))
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct device *dev = ctx->dev;
+ struct talitos_desc *desc = &edesc->desc;
+ int sg_count, ret;
+
+ /* first DWORD empty */
+ desc->ptr[0] = zero_entry;
+
+ /* hash context in */
+ if (!req_ctx->first || req_ctx->swinit) {
+ map_single_talitos_ptr(dev, &desc->ptr[1],
+ req_ctx->hw_context_size,
+ (char *)req_ctx->hw_context, 0,
+ DMA_TO_DEVICE);
+ req_ctx->swinit = 0;
+ } else {
+ desc->ptr[1] = zero_entry;
+ /* Indicate next op is not the first. */
+ req_ctx->first = 0;
+ }
+
+ /* HMAC key */
+ if (ctx->keylen)
+ map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
+ (char *)&ctx->key, 0, DMA_TO_DEVICE);
+ else
+ desc->ptr[2] = zero_entry;
+
+ /*
+ * data in
+ */
+ desc->ptr[3].len = cpu_to_be16(length);
+ desc->ptr[3].j_extent = 0;
+
+ sg_count = talitos_map_sg(dev, req_ctx->psrc,
+ edesc->src_nents ? : 1,
+ DMA_TO_DEVICE,
+ edesc->src_is_chained);
+
+ if (sg_count == 1) {
+ to_talitos_ptr(&desc->ptr[3], sg_dma_address(req_ctx->psrc));
+ } else {
+ sg_count = sg_to_link_tbl(req_ctx->psrc, sg_count, length,
+ &edesc->link_tbl[0]);
+ if (sg_count > 1) {
+ desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
+ to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl);
+ dma_sync_single_for_device(ctx->dev,
+ edesc->dma_link_tbl,
+ edesc->dma_len,
+ DMA_BIDIRECTIONAL);
+ } else {
+ /* Only one segment now, so no link tbl needed */
+ to_talitos_ptr(&desc->ptr[3],
+ sg_dma_address(req_ctx->psrc));
+ }
+ }
+
+ /* fifth DWORD empty */
+ desc->ptr[4] = zero_entry;
+
+ /* hash/HMAC out -or- hash context out */
+ if (req_ctx->last)
+ map_single_talitos_ptr(dev, &desc->ptr[5],
+ crypto_ahash_digestsize(tfm),
+ areq->result, 0, DMA_FROM_DEVICE);
+ else
+ map_single_talitos_ptr(dev, &desc->ptr[5],
+ req_ctx->hw_context_size,
+ req_ctx->hw_context, 0, DMA_FROM_DEVICE);
+
+ /* last DWORD empty */
+ desc->ptr[6] = zero_entry;
+
+ ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
+ if (ret != -EINPROGRESS) {
+ common_nonsnoop_hash_unmap(dev, edesc, areq);
+ kfree(edesc);
+ }
+ return ret;
+}
+
+static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
+ unsigned int nbytes)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, 1,
+ nbytes, 0, 0, areq->base.flags);
+}
+
+static int ahash_init(struct ahash_request *areq)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ /* Initialize the context */
+ req_ctx->nbuf = 0;
+ req_ctx->first = 1; /* first indicates h/w must init its context */
+ req_ctx->swinit = 0; /* assume h/w init of context */
+ req_ctx->hw_context_size =
+ (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
+ ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
+ : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
+
+ return 0;
+}
+
+/*
+ * on h/w without explicit sha224 support, we initialize h/w context
+ * manually with sha224 constants, and tell it to run sha256.
+ */
+static int ahash_init_sha224_swinit(struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ ahash_init(areq);
+ req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
+
+ req_ctx->hw_context[0] = SHA224_H0;
+ req_ctx->hw_context[1] = SHA224_H1;
+ req_ctx->hw_context[2] = SHA224_H2;
+ req_ctx->hw_context[3] = SHA224_H3;
+ req_ctx->hw_context[4] = SHA224_H4;
+ req_ctx->hw_context[5] = SHA224_H5;
+ req_ctx->hw_context[6] = SHA224_H6;
+ req_ctx->hw_context[7] = SHA224_H7;
+
+ /* init 64-bit count */
+ req_ctx->hw_context[8] = 0;
+ req_ctx->hw_context[9] = 0;
+
+ return 0;
+}
+
+static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+ struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct talitos_edesc *edesc;
+ unsigned int blocksize =
+ crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+ unsigned int nbytes_to_hash;
+ unsigned int to_hash_later;
+ unsigned int nsg;
+ int chained;
+
+ if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
+ /* Buffer up to one whole block */
+ sg_copy_to_buffer(areq->src,
+ sg_count(areq->src, nbytes, &chained),
+ req_ctx->buf + req_ctx->nbuf, nbytes);
+ req_ctx->nbuf += nbytes;
+ return 0;
+ }
+
+ /* At least (blocksize + 1) bytes are available to hash */
+ nbytes_to_hash = nbytes + req_ctx->nbuf;
+ to_hash_later = nbytes_to_hash & (blocksize - 1);
+
+ if (req_ctx->last)
+ to_hash_later = 0;
+ else if (to_hash_later)
+ /* There is a partial block. Hash the full block(s) now */
+ nbytes_to_hash -= to_hash_later;
+ else {
+ /* Keep one block buffered */
+ nbytes_to_hash -= blocksize;
+ to_hash_later = blocksize;
+ }
+
+ /* Chain in any previously buffered data */
+ if (req_ctx->nbuf) {
+ nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
+ sg_init_table(req_ctx->bufsl, nsg);
+ sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
+ if (nsg > 1)
+ scatterwalk_sg_chain(req_ctx->bufsl, 2, areq->src);
+ req_ctx->psrc = req_ctx->bufsl;
+ } else
+ req_ctx->psrc = areq->src;
+
+ if (to_hash_later) {
+ int nents = sg_count(areq->src, nbytes, &chained);
+ sg_copy_end_to_buffer(areq->src, nents,
+ req_ctx->bufnext,
+ to_hash_later,
+ nbytes - to_hash_later);
+ }
+ req_ctx->to_hash_later = to_hash_later;
+
+ /* Allocate extended descriptor */
+ edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
+ if (IS_ERR(edesc))
+ return PTR_ERR(edesc);
+
+ edesc->desc.hdr = ctx->desc_hdr_template;
+
+ /* On last one, request SEC to pad; otherwise continue */
+ if (req_ctx->last)
+ edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
+ else
+ edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
+
+ /* request SEC to INIT hash. */
+ if (req_ctx->first && !req_ctx->swinit)
+ edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
+
+ /* When the tfm context has a keylen, it's an HMAC.
+ * A first or last (ie. not middle) descriptor must request HMAC.
+ */
+ if (ctx->keylen && (req_ctx->first || req_ctx->last))
+ edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
+
+ return common_nonsnoop_hash(edesc, areq, nbytes_to_hash,
+ ahash_done);
+}
+
+static int ahash_update(struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ req_ctx->last = 0;
+
+ return ahash_process_req(areq, areq->nbytes);
+}
+
+static int ahash_final(struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ req_ctx->last = 1;
+
+ return ahash_process_req(areq, 0);
+}
+
+static int ahash_finup(struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+
+ req_ctx->last = 1;
+
+ return ahash_process_req(areq, areq->nbytes);
+}
+
+static int ahash_digest(struct ahash_request *areq)
+{
+ struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
+
+ ahash->init(areq);
+ req_ctx->last = 1;
+
+ return ahash_process_req(areq, areq->nbytes);
+}
+
+struct keyhash_result {
+ struct completion completion;
+ int err;
+};
+
+static void keyhash_complete(struct crypto_async_request *req, int err)
+{
+ struct keyhash_result *res = req->data;
+
+ if (err == -EINPROGRESS)
+ return;
+
+ res->err = err;
+ complete(&res->completion);
+}
+
+static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
+ u8 *hash)
+{
+ struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+
+ struct scatterlist sg[1];
+ struct ahash_request *req;
+ struct keyhash_result hresult;
+ int ret;
+
+ init_completion(&hresult.completion);
+
+ req = ahash_request_alloc(tfm, GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+
+ /* Keep tfm keylen == 0 during hash of the long key */
+ ctx->keylen = 0;
+ ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ keyhash_complete, &hresult);
+
+ sg_init_one(&sg[0], key, keylen);
+
+ ahash_request_set_crypt(req, sg, hash, keylen);
+ ret = crypto_ahash_digest(req);
+ switch (ret) {
+ case 0:
+ break;
+ case -EINPROGRESS:
+ case -EBUSY:
+ ret = wait_for_completion_interruptible(
+ &hresult.completion);
+ if (!ret)
+ ret = hresult.err;
+ break;
+ default:
+ break;
+ }
+ ahash_request_free(req);
+
+ return ret;
+}
+
+static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
+ unsigned int blocksize =
+ crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ unsigned int keysize = keylen;
+ u8 hash[SHA512_DIGEST_SIZE];
+ int ret;
+
+ if (keylen <= blocksize)
+ memcpy(ctx->key, key, keysize);
+ else {
+ /* Must get the hash of the long key */
+ ret = keyhash(tfm, key, keylen, hash);
+
+ if (ret) {
+ crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ keysize = digestsize;
+ memcpy(ctx->key, hash, digestsize);
+ }
+
+ ctx->keylen = keysize;
+
+ return 0;
+}
+
+
+struct talitos_alg_template {
+ u32 type;
+ union {
+ struct crypto_alg crypto;
+ struct ahash_alg hash;
+ } alg;
+ __be32 desc_hdr_template;
+};
+
+static struct talitos_alg_template driver_algs[] = {
+ /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(sha1),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_AESU |
+ DESC_HDR_MODE0_AESU_CBC |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_SHA1_HMAC,
+ },
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA1_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_DEU |
+ DESC_HDR_MODE0_DEU_CBC |
+ DESC_HDR_MODE0_DEU_3DES |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_SHA1_HMAC,
+ },
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(sha256),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_AESU |
+ DESC_HDR_MODE0_AESU_CBC |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_SHA256_HMAC,
+ },
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = SHA256_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_DEU |
+ DESC_HDR_MODE0_DEU_CBC |
+ DESC_HDR_MODE0_DEU_3DES |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_SHA256_HMAC,
+ },
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(md5),cbc(aes))",
+ .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_AESU |
+ DESC_HDR_MODE0_AESU_CBC |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_MD5_HMAC,
+ },
+ { .type = CRYPTO_ALG_TYPE_AEAD,
+ .alg.crypto = {
+ .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
+ .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_aead_type,
+ .cra_aead = {
+ .setkey = aead_setkey,
+ .setauthsize = aead_setauthsize,
+ .encrypt = aead_encrypt,
+ .decrypt = aead_decrypt,
+ .givencrypt = aead_givencrypt,
+ .geniv = "<built-in>",
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ .maxauthsize = MD5_DIGEST_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
+ DESC_HDR_SEL0_DEU |
+ DESC_HDR_MODE0_DEU_CBC |
+ DESC_HDR_MODE0_DEU_3DES |
+ DESC_HDR_SEL1_MDEUA |
+ DESC_HDR_MODE1_MDEU_INIT |
+ DESC_HDR_MODE1_MDEU_PAD |
+ DESC_HDR_MODE1_MDEU_MD5_HMAC,
+ },
+ /* ABLKCIPHER algorithms. */
+ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .alg.crypto = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-talitos",
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_ablkcipher = {
+ .setkey = ablkcipher_setkey,
+ .encrypt = ablkcipher_encrypt,
+ .decrypt = ablkcipher_decrypt,
+ .geniv = "eseqiv",
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_AESU |
+ DESC_HDR_MODE0_AESU_CBC,
+ },
+ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
+ .alg.crypto = {
+ .cra_name = "cbc(des3_ede)",
+ .cra_driver_name = "cbc-3des-talitos",
+ .cra_blocksize = DES3_EDE_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_ablkcipher = {
+ .setkey = ablkcipher_setkey,
+ .encrypt = ablkcipher_encrypt,
+ .decrypt = ablkcipher_decrypt,
+ .geniv = "eseqiv",
+ .min_keysize = DES3_EDE_KEY_SIZE,
+ .max_keysize = DES3_EDE_KEY_SIZE,
+ .ivsize = DES3_EDE_BLOCK_SIZE,
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_DEU |
+ DESC_HDR_MODE0_DEU_CBC |
+ DESC_HDR_MODE0_DEU_3DES,
+ },
+ /* AHASH algorithms. */
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = MD5_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "md5",
+ .cra_driver_name = "md5-talitos",
+ .cra_blocksize = MD5_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_MD5,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha1",
+ .cra_driver_name = "sha1-talitos",
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA1,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = SHA224_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "sha224-talitos",
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA224,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-talitos",
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA256,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "sha384-talitos",
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUB |
+ DESC_HDR_MODE0_MDEUB_SHA384,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-talitos",
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUB |
+ DESC_HDR_MODE0_MDEUB_SHA512,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = MD5_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(md5)",
+ .cra_driver_name = "hmac-md5-talitos",
+ .cra_blocksize = MD5_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_MD5,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = SHA1_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha1)",
+ .cra_driver_name = "hmac-sha1-talitos",
+ .cra_blocksize = SHA1_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA1,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = SHA224_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "hmac-sha224-talitos",
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA224,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = SHA256_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "hmac-sha256-talitos",
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA256,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = SHA384_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "hmac-sha384-talitos",
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUB |
+ DESC_HDR_MODE0_MDEUB_SHA384,
+ },
+ { .type = CRYPTO_ALG_TYPE_AHASH,
+ .alg.hash = {
+ .init = ahash_init,
+ .update = ahash_update,
+ .final = ahash_final,
+ .finup = ahash_finup,
+ .digest = ahash_digest,
+ .setkey = ahash_setkey,
+ .halg.digestsize = SHA512_DIGEST_SIZE,
+ .halg.base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "hmac-sha512-talitos",
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_flags = CRYPTO_ALG_TYPE_AHASH |
+ CRYPTO_ALG_ASYNC,
+ .cra_type = &crypto_ahash_type
+ }
+ },
+ .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUB |
+ DESC_HDR_MODE0_MDEUB_SHA512,
+ }
+};
+
+struct talitos_crypto_alg {
+ struct list_head entry;
+ struct device *dev;
+ struct talitos_alg_template algt;
+};
+
+static int talitos_cra_init(struct crypto_tfm *tfm)
+{
+ struct crypto_alg *alg = tfm->__crt_alg;
+ struct talitos_crypto_alg *talitos_alg;
+ struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct talitos_private *priv;
+
+ if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
+ talitos_alg = container_of(__crypto_ahash_alg(alg),
+ struct talitos_crypto_alg,
+ algt.alg.hash);
+ else
+ talitos_alg = container_of(alg, struct talitos_crypto_alg,
+ algt.alg.crypto);
+
+ /* update context with ptr to dev */
+ ctx->dev = talitos_alg->dev;
+
+ /* assign SEC channel to tfm in round-robin fashion */
+ priv = dev_get_drvdata(ctx->dev);
+ ctx->ch = atomic_inc_return(&priv->last_chan) &
+ (priv->num_channels - 1);
+
+ /* copy descriptor header template value */
+ ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
+
+ /* select done notification */
+ ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
+
+ return 0;
+}
+
+static int talitos_cra_init_aead(struct crypto_tfm *tfm)
+{
+ struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ talitos_cra_init(tfm);
+
+ /* random first IV */
+ get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
+
+ return 0;
+}
+
+static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
+{
+ struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ talitos_cra_init(tfm);
+
+ ctx->keylen = 0;
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct talitos_ahash_req_ctx));
+
+ return 0;
+}
+
+/*
+ * given the alg's descriptor header template, determine whether descriptor
+ * type and primary/secondary execution units required match the hw
+ * capabilities description provided in the device tree node.
+ */
+static int hw_supports(struct device *dev, __be32 desc_hdr_template)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ int ret;
+
+ ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
+ (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
+
+ if (SECONDARY_EU(desc_hdr_template))
+ ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
+ & priv->exec_units);
+
+ return ret;
+}
+
+static int talitos_remove(struct platform_device *ofdev)
+{
+ struct device *dev = &ofdev->dev;
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ struct talitos_crypto_alg *t_alg, *n;
+ int i;
+
+ list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
+ switch (t_alg->algt.type) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ case CRYPTO_ALG_TYPE_AEAD:
+ crypto_unregister_alg(&t_alg->algt.alg.crypto);
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ crypto_unregister_ahash(&t_alg->algt.alg.hash);
+ break;
+ }
+ list_del(&t_alg->entry);
+ kfree(t_alg);
+ }
+
+ if (hw_supports(dev, DESC_HDR_SEL0_RNG))
+ talitos_unregister_rng(dev);
+
+ for (i = 0; i < priv->num_channels; i++)
+ kfree(priv->chan[i].fifo);
+
+ kfree(priv->chan);
+
+ for (i = 0; i < 2; i++)
+ if (priv->irq[i]) {
+ free_irq(priv->irq[i], dev);
+ irq_dispose_mapping(priv->irq[i]);
+ }
+
+ tasklet_kill(&priv->done_task[0]);
+ if (priv->irq[1])
+ tasklet_kill(&priv->done_task[1]);
+
+ iounmap(priv->reg);
+
+ dev_set_drvdata(dev, NULL);
+
+ kfree(priv);
+
+ return 0;
+}
+
+static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
+ struct talitos_alg_template
+ *template)
+{
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ struct talitos_crypto_alg *t_alg;
+ struct crypto_alg *alg;
+
+ t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
+ if (!t_alg)
+ return ERR_PTR(-ENOMEM);
+
+ t_alg->algt = *template;
+
+ switch (t_alg->algt.type) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ alg = &t_alg->algt.alg.crypto;
+ alg->cra_init = talitos_cra_init;
+ break;
+ case CRYPTO_ALG_TYPE_AEAD:
+ alg = &t_alg->algt.alg.crypto;
+ alg->cra_init = talitos_cra_init_aead;
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ alg = &t_alg->algt.alg.hash.halg.base;
+ alg->cra_init = talitos_cra_init_ahash;
+ if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
+ !strncmp(alg->cra_name, "hmac", 4)) {
+ kfree(t_alg);
+ return ERR_PTR(-ENOTSUPP);
+ }
+ if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
+ (!strcmp(alg->cra_name, "sha224") ||
+ !strcmp(alg->cra_name, "hmac(sha224)"))) {
+ t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
+ t_alg->algt.desc_hdr_template =
+ DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
+ DESC_HDR_SEL0_MDEUA |
+ DESC_HDR_MODE0_MDEU_SHA256;
+ }
+ break;
+ default:
+ dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
+ kfree(t_alg);
+ return ERR_PTR(-EINVAL);
+ }
+
+ alg->cra_module = THIS_MODULE;
+ alg->cra_priority = TALITOS_CRA_PRIORITY;
+ alg->cra_alignmask = 0;
+ alg->cra_ctxsize = sizeof(struct talitos_ctx);
+ alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
+
+ t_alg->dev = dev;
+
+ return t_alg;
+}
+
+static int talitos_probe_irq(struct platform_device *ofdev)
+{
+ struct device *dev = &ofdev->dev;
+ struct device_node *np = ofdev->dev.of_node;
+ struct talitos_private *priv = dev_get_drvdata(dev);
+ int err;
+
+ priv->irq[0] = irq_of_parse_and_map(np, 0);
+ if (!priv->irq[0]) {
+ dev_err(dev, "failed to map irq\n");
+ return -EINVAL;
+ }
+
+ priv->irq[1] = irq_of_parse_and_map(np, 1);
+
+ /* get the primary irq line */
+ if (!priv->irq[1]) {
+ err = request_irq(priv->irq[0], talitos_interrupt_4ch, 0,
+ dev_driver_string(dev), dev);
+ goto primary_out;
+ }
+
+ err = request_irq(priv->irq[0], talitos_interrupt_ch0_2, 0,
+ dev_driver_string(dev), dev);
+ if (err)
+ goto primary_out;
+
+ /* get the secondary irq line */
+ err = request_irq(priv->irq[1], talitos_interrupt_ch1_3, 0,
+ dev_driver_string(dev), dev);
+ if (err) {
+ dev_err(dev, "failed to request secondary irq\n");
+ irq_dispose_mapping(priv->irq[1]);
+ priv->irq[1] = 0;
+ }
+
+ return err;
+
+primary_out:
+ if (err) {
+ dev_err(dev, "failed to request primary irq\n");
+ irq_dispose_mapping(priv->irq[0]);
+ priv->irq[0] = 0;
+ }
+
+ return err;
+}
+
+static int talitos_probe(struct platform_device *ofdev)
+{
+ struct device *dev = &ofdev->dev;
+ struct device_node *np = ofdev->dev.of_node;
+ struct talitos_private *priv;
+ const unsigned int *prop;
+ int i, err;
+
+ priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
+ if (!priv)
+ return -ENOMEM;
+
+ dev_set_drvdata(dev, priv);
+
+ priv->ofdev = ofdev;
+
+ spin_lock_init(&priv->reg_lock);
+
+ err = talitos_probe_irq(ofdev);
+ if (err)
+ goto err_out;
+
+ if (!priv->irq[1]) {
+ tasklet_init(&priv->done_task[0], talitos_done_4ch,
+ (unsigned long)dev);
+ } else {
+ tasklet_init(&priv->done_task[0], talitos_done_ch0_2,
+ (unsigned long)dev);
+ tasklet_init(&priv->done_task[1], talitos_done_ch1_3,
+ (unsigned long)dev);
+ }
+
+ INIT_LIST_HEAD(&priv->alg_list);
+
+ priv->reg = of_iomap(np, 0);
+ if (!priv->reg) {
+ dev_err(dev, "failed to of_iomap\n");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ /* get SEC version capabilities from device tree */
+ prop = of_get_property(np, "fsl,num-channels", NULL);
+ if (prop)
+ priv->num_channels = *prop;
+
+ prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
+ if (prop)
+ priv->chfifo_len = *prop;
+
+ prop = of_get_property(np, "fsl,exec-units-mask", NULL);
+ if (prop)
+ priv->exec_units = *prop;
+
+ prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
+ if (prop)
+ priv->desc_types = *prop;
+
+ if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
+ !priv->exec_units || !priv->desc_types) {
+ dev_err(dev, "invalid property data in device tree node\n");
+ err = -EINVAL;
+ goto err_out;
+ }
+
+ if (of_device_is_compatible(np, "fsl,sec3.0"))
+ priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
+
+ if (of_device_is_compatible(np, "fsl,sec2.1"))
+ priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
+ TALITOS_FTR_SHA224_HWINIT |
+ TALITOS_FTR_HMAC_OK;
+
+ priv->chan = kzalloc(sizeof(struct talitos_channel) *
+ priv->num_channels, GFP_KERNEL);
+ if (!priv->chan) {
+ dev_err(dev, "failed to allocate channel management space\n");
+ err = -ENOMEM;
+ goto err_out;
+ }
+
+ for (i = 0; i < priv->num_channels; i++) {
+ priv->chan[i].reg = priv->reg + TALITOS_CH_STRIDE * (i + 1);
+ if (!priv->irq[1] || !(i & 1))
+ priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
+ }
+
+ for (i = 0; i < priv->num_channels; i++) {
+ spin_lock_init(&priv->chan[i].head_lock);
+ spin_lock_init(&priv->chan[i].tail_lock);
+ }
+
+ priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
+
+ for (i = 0; i < priv->num_channels; i++) {
+ priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
+ priv->fifo_len, GFP_KERNEL);
+ if (!priv->chan[i].fifo) {
+ dev_err(dev, "failed to allocate request fifo %d\n", i);
+ err = -ENOMEM;
+ goto err_out;
+ }
+ }
+
+ for (i = 0; i < priv->num_channels; i++)
+ atomic_set(&priv->chan[i].submit_count,
+ -(priv->chfifo_len - 1));
+
+ dma_set_mask(dev, DMA_BIT_MASK(36));
+
+ /* reset and initialize the h/w */
+ err = init_device(dev);
+ if (err) {
+ dev_err(dev, "failed to initialize device\n");
+ goto err_out;
+ }
+
+ /* register the RNG, if available */
+ if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
+ err = talitos_register_rng(dev);
+ if (err) {
+ dev_err(dev, "failed to register hwrng: %d\n", err);
+ goto err_out;
+ } else
+ dev_info(dev, "hwrng\n");
+ }
+
+ /* register crypto algorithms the device supports */
+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
+ if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
+ struct talitos_crypto_alg *t_alg;
+ char *name = NULL;
+
+ t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
+ if (IS_ERR(t_alg)) {
+ err = PTR_ERR(t_alg);
+ if (err == -ENOTSUPP)
+ continue;
+ goto err_out;
+ }
+
+ switch (t_alg->algt.type) {
+ case CRYPTO_ALG_TYPE_ABLKCIPHER:
+ case CRYPTO_ALG_TYPE_AEAD:
+ err = crypto_register_alg(
+ &t_alg->algt.alg.crypto);
+ name = t_alg->algt.alg.crypto.cra_driver_name;
+ break;
+ case CRYPTO_ALG_TYPE_AHASH:
+ err = crypto_register_ahash(
+ &t_alg->algt.alg.hash);
+ name =
+ t_alg->algt.alg.hash.halg.base.cra_driver_name;
+ break;
+ }
+ if (err) {
+ dev_err(dev, "%s alg registration failed\n",
+ name);
+ kfree(t_alg);
+ } else
+ list_add_tail(&t_alg->entry, &priv->alg_list);
+ }
+ }
+ if (!list_empty(&priv->alg_list))
+ dev_info(dev, "%s algorithms registered in /proc/crypto\n",
+ (char *)of_get_property(np, "compatible", NULL));
+
+ return 0;
+
+err_out:
+ talitos_remove(ofdev);
+
+ return err;
+}
+
+static const struct of_device_id talitos_match[] = {
+ {
+ .compatible = "fsl,sec2.0",
+ },
+ {},
+};
+MODULE_DEVICE_TABLE(of, talitos_match);
+
+static struct platform_driver talitos_driver = {
+ .driver = {
+ .name = "talitos",
+ .owner = THIS_MODULE,
+ .of_match_table = talitos_match,
+ },
+ .probe = talitos_probe,
+ .remove = talitos_remove,
+};
+
+module_platform_driver(talitos_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
+MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.h b/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.h
new file mode 100644
index 0000000..3c17395
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/talitos.h
@@ -0,0 +1,228 @@
+/*
+ * Freescale SEC (talitos) device register and descriptor header defines
+ *
+ * Copyright (c) 2006-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/*
+ * TALITOS_xxx_LO addresses point to the low data bits (32-63) of the register
+ */
+
+/* global register offset addresses */
+#define TALITOS_MCR 0x1030 /* master control register */
+#define TALITOS_MCR_RCA0 (1 << 15) /* remap channel 0 */
+#define TALITOS_MCR_RCA1 (1 << 14) /* remap channel 1 */
+#define TALITOS_MCR_RCA2 (1 << 13) /* remap channel 2 */
+#define TALITOS_MCR_RCA3 (1 << 12) /* remap channel 3 */
+#define TALITOS_MCR_SWR 0x1 /* s/w reset */
+#define TALITOS_MCR_LO 0x1034
+#define TALITOS_IMR 0x1008 /* interrupt mask register */
+#define TALITOS_IMR_INIT 0x100ff /* enable channel IRQs */
+#define TALITOS_IMR_DONE 0x00055 /* done IRQs */
+#define TALITOS_IMR_LO 0x100C
+#define TALITOS_IMR_LO_INIT 0x20000 /* allow RNGU error IRQs */
+#define TALITOS_ISR 0x1010 /* interrupt status register */
+#define TALITOS_ISR_4CHERR 0xaa /* 4 channel errors mask */
+#define TALITOS_ISR_4CHDONE 0x55 /* 4 channel done mask */
+#define TALITOS_ISR_CH_0_2_ERR 0x22 /* channels 0, 2 errors mask */
+#define TALITOS_ISR_CH_0_2_DONE 0x11 /* channels 0, 2 done mask */
+#define TALITOS_ISR_CH_1_3_ERR 0x88 /* channels 1, 3 errors mask */
+#define TALITOS_ISR_CH_1_3_DONE 0x44 /* channels 1, 3 done mask */
+#define TALITOS_ISR_LO 0x1014
+#define TALITOS_ICR 0x1018 /* interrupt clear register */
+#define TALITOS_ICR_LO 0x101C
+
+/* channel register address stride */
+#define TALITOS_CH_BASE_OFFSET 0x1000 /* default channel map base */
+#define TALITOS_CH_STRIDE 0x100
+
+/* channel configuration register */
+#define TALITOS_CCCR 0x8
+#define TALITOS_CCCR_CONT 0x2 /* channel continue */
+#define TALITOS_CCCR_RESET 0x1 /* channel reset */
+#define TALITOS_CCCR_LO 0xc
+#define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */
+#define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */
+#define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */
+#define TALITOS_CCCR_LO_NT 0x4 /* notification type */
+#define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */
+
+/* CCPSR: channel pointer status register */
+#define TALITOS_CCPSR 0x10
+#define TALITOS_CCPSR_LO 0x14
+#define TALITOS_CCPSR_LO_DOF 0x8000 /* double FF write oflow error */
+#define TALITOS_CCPSR_LO_SOF 0x4000 /* single FF write oflow error */
+#define TALITOS_CCPSR_LO_MDTE 0x2000 /* master data transfer error */
+#define TALITOS_CCPSR_LO_SGDLZ 0x1000 /* s/g data len zero error */
+#define TALITOS_CCPSR_LO_FPZ 0x0800 /* fetch ptr zero error */
+#define TALITOS_CCPSR_LO_IDH 0x0400 /* illegal desc hdr error */
+#define TALITOS_CCPSR_LO_IEU 0x0200 /* invalid EU error */
+#define TALITOS_CCPSR_LO_EU 0x0100 /* EU error detected */
+#define TALITOS_CCPSR_LO_GB 0x0080 /* gather boundary error */
+#define TALITOS_CCPSR_LO_GRL 0x0040 /* gather return/length error */
+#define TALITOS_CCPSR_LO_SB 0x0020 /* scatter boundary error */
+#define TALITOS_CCPSR_LO_SRL 0x0010 /* scatter return/length error */
+
+/* channel fetch fifo register */
+#define TALITOS_FF 0x48
+#define TALITOS_FF_LO 0x4c
+
+/* current descriptor pointer register */
+#define TALITOS_CDPR 0x40
+#define TALITOS_CDPR_LO 0x44
+
+/* descriptor buffer register */
+#define TALITOS_DESCBUF 0x80
+#define TALITOS_DESCBUF_LO 0x84
+
+/* gather link table */
+#define TALITOS_GATHER 0xc0
+#define TALITOS_GATHER_LO 0xc4
+
+/* scatter link table */
+#define TALITOS_SCATTER 0xe0
+#define TALITOS_SCATTER_LO 0xe4
+
+/* execution unit interrupt status registers */
+#define TALITOS_DEUISR 0x2030 /* DES unit */
+#define TALITOS_DEUISR_LO 0x2034
+#define TALITOS_AESUISR 0x4030 /* AES unit */
+#define TALITOS_AESUISR_LO 0x4034
+#define TALITOS_MDEUISR 0x6030 /* message digest unit */
+#define TALITOS_MDEUISR_LO 0x6034
+#define TALITOS_MDEUICR 0x6038 /* interrupt control */
+#define TALITOS_MDEUICR_LO 0x603c
+#define TALITOS_MDEUICR_LO_ICE 0x4000 /* integrity check IRQ enable */
+#define TALITOS_AFEUISR 0x8030 /* arc4 unit */
+#define TALITOS_AFEUISR_LO 0x8034
+#define TALITOS_RNGUISR 0xa030 /* random number unit */
+#define TALITOS_RNGUISR_LO 0xa034
+#define TALITOS_RNGUSR 0xa028 /* rng status */
+#define TALITOS_RNGUSR_LO 0xa02c
+#define TALITOS_RNGUSR_LO_RD 0x1 /* reset done */
+#define TALITOS_RNGUSR_LO_OFL 0xff0000/* output FIFO length */
+#define TALITOS_RNGUDSR 0xa010 /* data size */
+#define TALITOS_RNGUDSR_LO 0xa014
+#define TALITOS_RNGU_FIFO 0xa800 /* output FIFO */
+#define TALITOS_RNGU_FIFO_LO 0xa804 /* output FIFO */
+#define TALITOS_RNGURCR 0xa018 /* reset control */
+#define TALITOS_RNGURCR_LO 0xa01c
+#define TALITOS_RNGURCR_LO_SR 0x1 /* software reset */
+#define TALITOS_PKEUISR 0xc030 /* public key unit */
+#define TALITOS_PKEUISR_LO 0xc034
+#define TALITOS_KEUISR 0xe030 /* kasumi unit */
+#define TALITOS_KEUISR_LO 0xe034
+#define TALITOS_CRCUISR 0xf030 /* cyclic redundancy check unit*/
+#define TALITOS_CRCUISR_LO 0xf034
+
+#define TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256 0x28
+#define TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512 0x48
+
+/*
+ * talitos descriptor header (hdr) bits
+ */
+
+/* written back when done */
+#define DESC_HDR_DONE cpu_to_be32(0xff000000)
+#define DESC_HDR_LO_ICCR1_MASK cpu_to_be32(0x00180000)
+#define DESC_HDR_LO_ICCR1_PASS cpu_to_be32(0x00080000)
+#define DESC_HDR_LO_ICCR1_FAIL cpu_to_be32(0x00100000)
+
+/* primary execution unit select */
+#define DESC_HDR_SEL0_MASK cpu_to_be32(0xf0000000)
+#define DESC_HDR_SEL0_AFEU cpu_to_be32(0x10000000)
+#define DESC_HDR_SEL0_DEU cpu_to_be32(0x20000000)
+#define DESC_HDR_SEL0_MDEUA cpu_to_be32(0x30000000)
+#define DESC_HDR_SEL0_MDEUB cpu_to_be32(0xb0000000)
+#define DESC_HDR_SEL0_RNG cpu_to_be32(0x40000000)
+#define DESC_HDR_SEL0_PKEU cpu_to_be32(0x50000000)
+#define DESC_HDR_SEL0_AESU cpu_to_be32(0x60000000)
+#define DESC_HDR_SEL0_KEU cpu_to_be32(0x70000000)
+#define DESC_HDR_SEL0_CRCU cpu_to_be32(0x80000000)
+
+/* primary execution unit mode (MODE0) and derivatives */
+#define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000)
+#define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000)
+#define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000)
+#define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000)
+#define DESC_HDR_MODE0_MDEU_CONT cpu_to_be32(0x08000000)
+#define DESC_HDR_MODE0_MDEU_INIT cpu_to_be32(0x01000000)
+#define DESC_HDR_MODE0_MDEU_HMAC cpu_to_be32(0x00800000)
+#define DESC_HDR_MODE0_MDEU_PAD cpu_to_be32(0x00400000)
+#define DESC_HDR_MODE0_MDEU_SHA224 cpu_to_be32(0x00300000)
+#define DESC_HDR_MODE0_MDEU_MD5 cpu_to_be32(0x00200000)
+#define DESC_HDR_MODE0_MDEU_SHA256 cpu_to_be32(0x00100000)
+#define DESC_HDR_MODE0_MDEU_SHA1 cpu_to_be32(0x00000000)
+#define DESC_HDR_MODE0_MDEUB_SHA384 cpu_to_be32(0x00000000)
+#define DESC_HDR_MODE0_MDEUB_SHA512 cpu_to_be32(0x00200000)
+#define DESC_HDR_MODE0_MDEU_MD5_HMAC (DESC_HDR_MODE0_MDEU_MD5 | \
+ DESC_HDR_MODE0_MDEU_HMAC)
+#define DESC_HDR_MODE0_MDEU_SHA256_HMAC (DESC_HDR_MODE0_MDEU_SHA256 | \
+ DESC_HDR_MODE0_MDEU_HMAC)
+#define DESC_HDR_MODE0_MDEU_SHA1_HMAC (DESC_HDR_MODE0_MDEU_SHA1 | \
+ DESC_HDR_MODE0_MDEU_HMAC)
+
+/* secondary execution unit select (SEL1) */
+#define DESC_HDR_SEL1_MASK cpu_to_be32(0x000f0000)
+#define DESC_HDR_SEL1_MDEUA cpu_to_be32(0x00030000)
+#define DESC_HDR_SEL1_MDEUB cpu_to_be32(0x000b0000)
+#define DESC_HDR_SEL1_CRCU cpu_to_be32(0x00080000)
+
+/* secondary execution unit mode (MODE1) and derivatives */
+#define DESC_HDR_MODE1_MDEU_CICV cpu_to_be32(0x00004000)
+#define DESC_HDR_MODE1_MDEU_INIT cpu_to_be32(0x00001000)
+#define DESC_HDR_MODE1_MDEU_HMAC cpu_to_be32(0x00000800)
+#define DESC_HDR_MODE1_MDEU_PAD cpu_to_be32(0x00000400)
+#define DESC_HDR_MODE1_MDEU_SHA224 cpu_to_be32(0x00000300)
+#define DESC_HDR_MODE1_MDEU_MD5 cpu_to_be32(0x00000200)
+#define DESC_HDR_MODE1_MDEU_SHA256 cpu_to_be32(0x00000100)
+#define DESC_HDR_MODE1_MDEU_SHA1 cpu_to_be32(0x00000000)
+#define DESC_HDR_MODE1_MDEUB_SHA384 cpu_to_be32(0x00000000)
+#define DESC_HDR_MODE1_MDEUB_SHA512 cpu_to_be32(0x00000200)
+#define DESC_HDR_MODE1_MDEU_MD5_HMAC (DESC_HDR_MODE1_MDEU_MD5 | \
+ DESC_HDR_MODE1_MDEU_HMAC)
+#define DESC_HDR_MODE1_MDEU_SHA256_HMAC (DESC_HDR_MODE1_MDEU_SHA256 | \
+ DESC_HDR_MODE1_MDEU_HMAC)
+#define DESC_HDR_MODE1_MDEU_SHA1_HMAC (DESC_HDR_MODE1_MDEU_SHA1 | \
+ DESC_HDR_MODE1_MDEU_HMAC)
+
+/* direction of overall data flow (DIR) */
+#define DESC_HDR_DIR_INBOUND cpu_to_be32(0x00000002)
+
+/* request done notification (DN) */
+#define DESC_HDR_DONE_NOTIFY cpu_to_be32(0x00000001)
+
+/* descriptor types */
+#define DESC_HDR_TYPE_AESU_CTR_NONSNOOP cpu_to_be32(0 << 3)
+#define DESC_HDR_TYPE_IPSEC_ESP cpu_to_be32(1 << 3)
+#define DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU cpu_to_be32(2 << 3)
+#define DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU cpu_to_be32(4 << 3)
+
+/* link table extent field bits */
+#define DESC_PTR_LNKTBL_JUMP 0x80
+#define DESC_PTR_LNKTBL_RETURN 0x02
+#define DESC_PTR_LNKTBL_NEXT 0x01
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.c b/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.c
new file mode 100644
index 0000000..422a976
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.c
@@ -0,0 +1,1096 @@
+/*
+ * drivers/crypto/tegra-aes.c
+ *
+ * Driver for NVIDIA Tegra AES hardware engine residing inside the
+ * Bit Stream Engine for Video (BSEV) hardware block.
+ *
+ * The programming sequence for this engine is with the help
+ * of commands which travel via a command queue residing between the
+ * CPU and the BSEV block. The BSEV engine has an internal RAM (VRAM)
+ * where the final input plaintext, keys and the IV have to be copied
+ * before starting the encrypt/decrypt operation.
+ *
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/clk.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/mutex.h>
+#include <linux/interrupt.h>
+#include <linux/completion.h>
+#include <linux/workqueue.h>
+
+#include <mach/clk.h>
+
+#include <crypto/scatterwalk.h>
+#include <crypto/aes.h>
+#include <crypto/internal/rng.h>
+
+#include "tegra-aes.h"
+
+#define FLAGS_MODE_MASK 0x00FF
+#define FLAGS_ENCRYPT BIT(0)
+#define FLAGS_CBC BIT(1)
+#define FLAGS_GIV BIT(2)
+#define FLAGS_RNG BIT(3)
+#define FLAGS_OFB BIT(4)
+#define FLAGS_NEW_KEY BIT(5)
+#define FLAGS_NEW_IV BIT(6)
+#define FLAGS_INIT BIT(7)
+#define FLAGS_FAST BIT(8)
+#define FLAGS_BUSY 9
+
+/*
+ * Defines AES engine Max process bytes size in one go, which takes 1 msec.
+ * AES engine spends about 176 cycles/16-bytes or 11 cycles/byte
+ * The duration CPU can use the BSE to 1 msec, then the number of available
+ * cycles of AVP/BSE is 216K. In this duration, AES can process 216/11 ~= 19KB
+ * Based on this AES_HW_DMA_BUFFER_SIZE_BYTES is configured to 16KB.
+ */
+#define AES_HW_DMA_BUFFER_SIZE_BYTES 0x4000
+
+/*
+ * The key table length is 64 bytes
+ * (This includes first upto 32 bytes key + 16 bytes original initial vector
+ * and 16 bytes updated initial vector)
+ */
+#define AES_HW_KEY_TABLE_LENGTH_BYTES 64
+
+/*
+ * The memory being used is divides as follows:
+ * 1. Key - 32 bytes
+ * 2. Original IV - 16 bytes
+ * 3. Updated IV - 16 bytes
+ * 4. Key schedule - 256 bytes
+ *
+ * 1+2+3 constitute the hw key table.
+ */
+#define AES_HW_IV_SIZE 16
+#define AES_HW_KEYSCHEDULE_LEN 256
+#define AES_IVKEY_SIZE (AES_HW_KEY_TABLE_LENGTH_BYTES + AES_HW_KEYSCHEDULE_LEN)
+
+/* Define commands required for AES operation */
+enum {
+ CMD_BLKSTARTENGINE = 0x0E,
+ CMD_DMASETUP = 0x10,
+ CMD_DMACOMPLETE = 0x11,
+ CMD_SETTABLE = 0x15,
+ CMD_MEMDMAVD = 0x22,
+};
+
+/* Define sub-commands */
+enum {
+ SUBCMD_VRAM_SEL = 0x1,
+ SUBCMD_CRYPTO_TABLE_SEL = 0x3,
+ SUBCMD_KEY_TABLE_SEL = 0x8,
+};
+
+/* memdma_vd command */
+#define MEMDMA_DIR_DTOVRAM 0 /* sdram -> vram */
+#define MEMDMA_DIR_VTODRAM 1 /* vram -> sdram */
+#define MEMDMA_DIR_SHIFT 25
+#define MEMDMA_NUM_WORDS_SHIFT 12
+
+/* command queue bit shifts */
+enum {
+ CMDQ_KEYTABLEADDR_SHIFT = 0,
+ CMDQ_KEYTABLEID_SHIFT = 17,
+ CMDQ_VRAMSEL_SHIFT = 23,
+ CMDQ_TABLESEL_SHIFT = 24,
+ CMDQ_OPCODE_SHIFT = 26,
+};
+
+/*
+ * The secure key slot contains a unique secure key generated
+ * and loaded by the bootloader. This slot is marked as non-accessible
+ * to the kernel.
+ */
+#define SSK_SLOT_NUM 4
+
+#define AES_NR_KEYSLOTS 8
+#define TEGRA_AES_QUEUE_LENGTH 50
+#define DEFAULT_RNG_BLK_SZ 16
+
+/* The command queue depth */
+#define AES_HW_MAX_ICQ_LENGTH 5
+
+struct tegra_aes_slot {
+ struct list_head node;
+ int slot_num;
+};
+
+static struct tegra_aes_slot ssk = {
+ .slot_num = SSK_SLOT_NUM,
+};
+
+struct tegra_aes_reqctx {
+ unsigned long mode;
+};
+
+struct tegra_aes_dev {
+ struct device *dev;
+ void __iomem *io_base;
+ dma_addr_t ivkey_phys_base;
+ void __iomem *ivkey_base;
+ struct clk *aes_clk;
+ struct tegra_aes_ctx *ctx;
+ int irq;
+ unsigned long flags;
+ struct completion op_complete;
+ u32 *buf_in;
+ dma_addr_t dma_buf_in;
+ u32 *buf_out;
+ dma_addr_t dma_buf_out;
+ u8 *iv;
+ u8 dt[DEFAULT_RNG_BLK_SZ];
+ int ivlen;
+ u64 ctr;
+ spinlock_t lock;
+ struct crypto_queue queue;
+ struct tegra_aes_slot *slots;
+ struct ablkcipher_request *req;
+ size_t total;
+ struct scatterlist *in_sg;
+ size_t in_offset;
+ struct scatterlist *out_sg;
+ size_t out_offset;
+};
+
+static struct tegra_aes_dev *aes_dev;
+
+struct tegra_aes_ctx {
+ struct tegra_aes_dev *dd;
+ unsigned long flags;
+ struct tegra_aes_slot *slot;
+ u8 key[AES_MAX_KEY_SIZE];
+ size_t keylen;
+};
+
+static struct tegra_aes_ctx rng_ctx = {
+ .flags = FLAGS_NEW_KEY,
+ .keylen = AES_KEYSIZE_128,
+};
+
+/* keep registered devices data here */
+static struct list_head dev_list;
+static DEFINE_SPINLOCK(list_lock);
+static DEFINE_MUTEX(aes_lock);
+
+static void aes_workqueue_handler(struct work_struct *work);
+static DECLARE_WORK(aes_work, aes_workqueue_handler);
+static struct workqueue_struct *aes_wq;
+
+extern unsigned long long tegra_chip_uid(void);
+
+static inline u32 aes_readl(struct tegra_aes_dev *dd, u32 offset)
+{
+ return readl(dd->io_base + offset);
+}
+
+static inline void aes_writel(struct tegra_aes_dev *dd, u32 val, u32 offset)
+{
+ writel(val, dd->io_base + offset);
+}
+
+static int aes_start_crypt(struct tegra_aes_dev *dd, u32 in_addr, u32 out_addr,
+ int nblocks, int mode, bool upd_iv)
+{
+ u32 cmdq[AES_HW_MAX_ICQ_LENGTH];
+ int i, eng_busy, icq_empty, ret;
+ u32 value;
+
+ /* reset all the interrupt bits */
+ aes_writel(dd, 0xFFFFFFFF, TEGRA_AES_INTR_STATUS);
+
+ /* enable error, dma xfer complete interrupts */
+ aes_writel(dd, 0x33, TEGRA_AES_INT_ENB);
+
+ cmdq[0] = CMD_DMASETUP << CMDQ_OPCODE_SHIFT;
+ cmdq[1] = in_addr;
+ cmdq[2] = CMD_BLKSTARTENGINE << CMDQ_OPCODE_SHIFT | (nblocks-1);
+ cmdq[3] = CMD_DMACOMPLETE << CMDQ_OPCODE_SHIFT;
+
+ value = aes_readl(dd, TEGRA_AES_CMDQUE_CONTROL);
+ /* access SDRAM through AHB */
+ value &= ~TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD;
+ value &= ~TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD;
+ value |= TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD |
+ TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD |
+ TEGRA_AES_CMDQ_CTRL_ICMDQEN_FIELD;
+ aes_writel(dd, value, TEGRA_AES_CMDQUE_CONTROL);
+ dev_dbg(dd->dev, "cmd_q_ctrl=0x%x", value);
+
+ value = (0x1 << TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT) |
+ ((dd->ctx->keylen * 8) <<
+ TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT) |
+ ((u32)upd_iv << TEGRA_AES_SECURE_IV_SELECT_SHIFT);
+
+ if (mode & FLAGS_CBC) {
+ value |= ((((mode & FLAGS_ENCRYPT) ? 2 : 3)
+ << TEGRA_AES_SECURE_XOR_POS_SHIFT) |
+ (((mode & FLAGS_ENCRYPT) ? 2 : 3)
+ << TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT) |
+ ((mode & FLAGS_ENCRYPT) ? 1 : 0)
+ << TEGRA_AES_SECURE_CORE_SEL_SHIFT);
+ } else if (mode & FLAGS_OFB) {
+ value |= ((TEGRA_AES_SECURE_XOR_POS_FIELD) |
+ (2 << TEGRA_AES_SECURE_INPUT_SEL_SHIFT) |
+ (TEGRA_AES_SECURE_CORE_SEL_FIELD));
+ } else if (mode & FLAGS_RNG) {
+ value |= (((mode & FLAGS_ENCRYPT) ? 1 : 0)
+ << TEGRA_AES_SECURE_CORE_SEL_SHIFT |
+ TEGRA_AES_SECURE_RNG_ENB_FIELD);
+ } else {
+ value |= (((mode & FLAGS_ENCRYPT) ? 1 : 0)
+ << TEGRA_AES_SECURE_CORE_SEL_SHIFT);
+ }
+
+ dev_dbg(dd->dev, "secure_in_sel=0x%x", value);
+ aes_writel(dd, value, TEGRA_AES_SECURE_INPUT_SELECT);
+
+ aes_writel(dd, out_addr, TEGRA_AES_SECURE_DEST_ADDR);
+ INIT_COMPLETION(dd->op_complete);
+
+ for (i = 0; i < AES_HW_MAX_ICQ_LENGTH - 1; i++) {
+ do {
+ value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
+ eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
+ icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
+ } while (eng_busy & (!icq_empty));
+ aes_writel(dd, cmdq[i], TEGRA_AES_ICMDQUE_WR);
+ }
+
+ ret = wait_for_completion_timeout(&dd->op_complete,
+ msecs_to_jiffies(150));
+ if (ret == 0) {
+ dev_err(dd->dev, "timed out (0x%x)\n",
+ aes_readl(dd, TEGRA_AES_INTR_STATUS));
+ return -ETIMEDOUT;
+ }
+
+ aes_writel(dd, cmdq[AES_HW_MAX_ICQ_LENGTH - 1], TEGRA_AES_ICMDQUE_WR);
+ return 0;
+}
+
+static void aes_release_key_slot(struct tegra_aes_slot *slot)
+{
+ if (slot->slot_num == SSK_SLOT_NUM)
+ return;
+
+ spin_lock(&list_lock);
+ list_add_tail(&slot->node, &dev_list);
+ slot = NULL;
+ spin_unlock(&list_lock);
+}
+
+static struct tegra_aes_slot *aes_find_key_slot(void)
+{
+ struct tegra_aes_slot *slot = NULL;
+ struct list_head *new_head;
+ int empty;
+
+ spin_lock(&list_lock);
+ empty = list_empty(&dev_list);
+ if (!empty) {
+ slot = list_entry(&dev_list, struct tegra_aes_slot, node);
+ new_head = dev_list.next;
+ list_del(&dev_list);
+ dev_list.next = new_head->next;
+ dev_list.prev = NULL;
+ }
+ spin_unlock(&list_lock);
+
+ return slot;
+}
+
+static int aes_set_key(struct tegra_aes_dev *dd)
+{
+ u32 value, cmdq[2];
+ struct tegra_aes_ctx *ctx = dd->ctx;
+ int eng_busy, icq_empty, dma_busy;
+ bool use_ssk = false;
+
+ /* use ssk? */
+ if (!dd->ctx->slot) {
+ dev_dbg(dd->dev, "using ssk");
+ dd->ctx->slot = &ssk;
+ use_ssk = true;
+ }
+
+ /* enable key schedule generation in hardware */
+ value = aes_readl(dd, TEGRA_AES_SECURE_CONFIG_EXT);
+ value &= ~TEGRA_AES_SECURE_KEY_SCH_DIS_FIELD;
+ aes_writel(dd, value, TEGRA_AES_SECURE_CONFIG_EXT);
+
+ /* select the key slot */
+ value = aes_readl(dd, TEGRA_AES_SECURE_CONFIG);
+ value &= ~TEGRA_AES_SECURE_KEY_INDEX_FIELD;
+ value |= (ctx->slot->slot_num << TEGRA_AES_SECURE_KEY_INDEX_SHIFT);
+ aes_writel(dd, value, TEGRA_AES_SECURE_CONFIG);
+
+ if (use_ssk)
+ return 0;
+
+ /* copy the key table from sdram to vram */
+ cmdq[0] = CMD_MEMDMAVD << CMDQ_OPCODE_SHIFT |
+ MEMDMA_DIR_DTOVRAM << MEMDMA_DIR_SHIFT |
+ AES_HW_KEY_TABLE_LENGTH_BYTES / sizeof(u32) <<
+ MEMDMA_NUM_WORDS_SHIFT;
+ cmdq[1] = (u32)dd->ivkey_phys_base;
+
+ aes_writel(dd, cmdq[0], TEGRA_AES_ICMDQUE_WR);
+ aes_writel(dd, cmdq[1], TEGRA_AES_ICMDQUE_WR);
+
+ do {
+ value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
+ eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
+ icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
+ dma_busy = value & TEGRA_AES_DMA_BUSY_FIELD;
+ } while (eng_busy & (!icq_empty) & dma_busy);
+
+ /* settable command to get key into internal registers */
+ value = CMD_SETTABLE << CMDQ_OPCODE_SHIFT |
+ SUBCMD_CRYPTO_TABLE_SEL << CMDQ_TABLESEL_SHIFT |
+ SUBCMD_VRAM_SEL << CMDQ_VRAMSEL_SHIFT |
+ (SUBCMD_KEY_TABLE_SEL | ctx->slot->slot_num) <<
+ CMDQ_KEYTABLEID_SHIFT;
+ aes_writel(dd, value, TEGRA_AES_ICMDQUE_WR);
+
+ do {
+ value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
+ eng_busy = value & TEGRA_AES_ENGINE_BUSY_FIELD;
+ icq_empty = value & TEGRA_AES_ICQ_EMPTY_FIELD;
+ } while (eng_busy & (!icq_empty));
+
+ return 0;
+}
+
+static int tegra_aes_handle_req(struct tegra_aes_dev *dd)
+{
+ struct crypto_async_request *async_req, *backlog;
+ struct crypto_ablkcipher *tfm;
+ struct tegra_aes_ctx *ctx;
+ struct tegra_aes_reqctx *rctx;
+ struct ablkcipher_request *req;
+ unsigned long flags;
+ int dma_max = AES_HW_DMA_BUFFER_SIZE_BYTES;
+ int ret = 0, nblocks, total;
+ int count = 0;
+ dma_addr_t addr_in, addr_out;
+ struct scatterlist *in_sg, *out_sg;
+
+ if (!dd)
+ return -EINVAL;
+
+ spin_lock_irqsave(&dd->lock, flags);
+ backlog = crypto_get_backlog(&dd->queue);
+ async_req = crypto_dequeue_request(&dd->queue);
+ if (!async_req)
+ clear_bit(FLAGS_BUSY, &dd->flags);
+ spin_unlock_irqrestore(&dd->lock, flags);
+
+ if (!async_req)
+ return -ENODATA;
+
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
+ req = ablkcipher_request_cast(async_req);
+
+ dev_dbg(dd->dev, "%s: get new req\n", __func__);
+
+ if (!req->src || !req->dst)
+ return -EINVAL;
+
+ /* take mutex to access the aes hw */
+ mutex_lock(&aes_lock);
+
+ /* assign new request to device */
+ dd->req = req;
+ dd->total = req->nbytes;
+ dd->in_offset = 0;
+ dd->in_sg = req->src;
+ dd->out_offset = 0;
+ dd->out_sg = req->dst;
+
+ in_sg = dd->in_sg;
+ out_sg = dd->out_sg;
+
+ total = dd->total;
+
+ tfm = crypto_ablkcipher_reqtfm(req);
+ rctx = ablkcipher_request_ctx(req);
+ ctx = crypto_ablkcipher_ctx(tfm);
+ rctx->mode &= FLAGS_MODE_MASK;
+ dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
+
+ dd->iv = (u8 *)req->info;
+ dd->ivlen = crypto_ablkcipher_ivsize(tfm);
+
+ /* assign new context to device */
+ ctx->dd = dd;
+ dd->ctx = ctx;
+
+ if (ctx->flags & FLAGS_NEW_KEY) {
+ /* copy the key */
+ memcpy(dd->ivkey_base, ctx->key, ctx->keylen);
+ memset(dd->ivkey_base + ctx->keylen, 0, AES_HW_KEY_TABLE_LENGTH_BYTES - ctx->keylen);
+ aes_set_key(dd);
+ ctx->flags &= ~FLAGS_NEW_KEY;
+ }
+
+ if (((dd->flags & FLAGS_CBC) || (dd->flags & FLAGS_OFB)) && dd->iv) {
+ /* set iv to the aes hw slot
+ * Hw generates updated iv only after iv is set in slot.
+ * So key and iv is passed asynchronously.
+ */
+ memcpy(dd->buf_in, dd->iv, dd->ivlen);
+
+ ret = aes_start_crypt(dd, (u32)dd->dma_buf_in,
+ dd->dma_buf_out, 1, FLAGS_CBC, false);
+ if (ret < 0) {
+ dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
+ goto out;
+ }
+ }
+
+ while (total) {
+ dev_dbg(dd->dev, "remain: %d\n", total);
+ ret = dma_map_sg(dd->dev, in_sg, 1, DMA_TO_DEVICE);
+ if (!ret) {
+ dev_err(dd->dev, "dma_map_sg() error\n");
+ goto out;
+ }
+
+ ret = dma_map_sg(dd->dev, out_sg, 1, DMA_FROM_DEVICE);
+ if (!ret) {
+ dev_err(dd->dev, "dma_map_sg() error\n");
+ dma_unmap_sg(dd->dev, dd->in_sg,
+ 1, DMA_TO_DEVICE);
+ goto out;
+ }
+
+ addr_in = sg_dma_address(in_sg);
+ addr_out = sg_dma_address(out_sg);
+ dd->flags |= FLAGS_FAST;
+ count = min_t(int, sg_dma_len(in_sg), dma_max);
+ WARN_ON(sg_dma_len(in_sg) != sg_dma_len(out_sg));
+ nblocks = DIV_ROUND_UP(count, AES_BLOCK_SIZE);
+
+ ret = aes_start_crypt(dd, addr_in, addr_out, nblocks,
+ dd->flags, true);
+
+ dma_unmap_sg(dd->dev, out_sg, 1, DMA_FROM_DEVICE);
+ dma_unmap_sg(dd->dev, in_sg, 1, DMA_TO_DEVICE);
+
+ if (ret < 0) {
+ dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
+ goto out;
+ }
+ dd->flags &= ~FLAGS_FAST;
+
+ dev_dbg(dd->dev, "out: copied %d\n", count);
+ total -= count;
+ in_sg = sg_next(in_sg);
+ out_sg = sg_next(out_sg);
+ WARN_ON(((total != 0) && (!in_sg || !out_sg)));
+ }
+
+out:
+ mutex_unlock(&aes_lock);
+
+ dd->total = total;
+
+ if (dd->req->base.complete)
+ dd->req->base.complete(&dd->req->base, ret);
+
+ dev_dbg(dd->dev, "%s: exit\n", __func__);
+ return ret;
+}
+
+static int tegra_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct tegra_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct tegra_aes_dev *dd = aes_dev;
+ struct tegra_aes_slot *key_slot;
+
+ if ((keylen != AES_KEYSIZE_128) && (keylen != AES_KEYSIZE_192) &&
+ (keylen != AES_KEYSIZE_256)) {
+ dev_err(dd->dev, "unsupported key size\n");
+ crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
+ return -EINVAL;
+ }
+
+ dev_dbg(dd->dev, "keylen: %d\n", keylen);
+
+ ctx->dd = dd;
+
+ if (key) {
+ if (!ctx->slot) {
+ key_slot = aes_find_key_slot();
+ if (!key_slot) {
+ dev_err(dd->dev, "no empty slot\n");
+ return -ENOMEM;
+ }
+
+ ctx->slot = key_slot;
+ }
+
+ memcpy(ctx->key, key, keylen);
+ ctx->keylen = keylen;
+ }
+
+ ctx->flags |= FLAGS_NEW_KEY;
+ dev_dbg(dd->dev, "done\n");
+ return 0;
+}
+
+static void aes_workqueue_handler(struct work_struct *work)
+{
+ struct tegra_aes_dev *dd = aes_dev;
+ int ret;
+
+ ret = clk_enable(dd->aes_clk);
+ if (ret)
+ BUG_ON("clock enable failed");
+
+ /* empty the crypto queue and then return */
+ do {
+ ret = tegra_aes_handle_req(dd);
+ } while (!ret);
+
+ clk_disable(dd->aes_clk);
+}
+
+static irqreturn_t aes_irq(int irq, void *dev_id)
+{
+ struct tegra_aes_dev *dd = (struct tegra_aes_dev *)dev_id;
+ u32 value = aes_readl(dd, TEGRA_AES_INTR_STATUS);
+ int busy = test_bit(FLAGS_BUSY, &dd->flags);
+
+ if (!busy) {
+ dev_dbg(dd->dev, "spurious interrupt\n");
+ return IRQ_NONE;
+ }
+
+ dev_dbg(dd->dev, "irq_stat: 0x%x\n", value);
+ if (value & TEGRA_AES_INT_ERROR_MASK)
+ aes_writel(dd, TEGRA_AES_INT_ERROR_MASK, TEGRA_AES_INTR_STATUS);
+
+ if (!(value & TEGRA_AES_ENGINE_BUSY_FIELD))
+ complete(&dd->op_complete);
+ else
+ return IRQ_NONE;
+
+ return IRQ_HANDLED;
+}
+
+static int tegra_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
+{
+ struct tegra_aes_reqctx *rctx = ablkcipher_request_ctx(req);
+ struct tegra_aes_dev *dd = aes_dev;
+ unsigned long flags;
+ int err = 0;
+ int busy;
+
+ dev_dbg(dd->dev, "nbytes: %d, enc: %d, cbc: %d, ofb: %d\n",
+ req->nbytes, !!(mode & FLAGS_ENCRYPT),
+ !!(mode & FLAGS_CBC), !!(mode & FLAGS_OFB));
+
+ rctx->mode = mode;
+
+ spin_lock_irqsave(&dd->lock, flags);
+ err = ablkcipher_enqueue_request(&dd->queue, req);
+ busy = test_and_set_bit(FLAGS_BUSY, &dd->flags);
+ spin_unlock_irqrestore(&dd->lock, flags);
+
+ if (!busy)
+ queue_work(aes_wq, &aes_work);
+
+ return err;
+}
+
+static int tegra_aes_ecb_encrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, FLAGS_ENCRYPT);
+}
+
+static int tegra_aes_ecb_decrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, 0);
+}
+
+static int tegra_aes_cbc_encrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
+}
+
+static int tegra_aes_cbc_decrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, FLAGS_CBC);
+}
+
+static int tegra_aes_ofb_encrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_OFB);
+}
+
+static int tegra_aes_ofb_decrypt(struct ablkcipher_request *req)
+{
+ return tegra_aes_crypt(req, FLAGS_OFB);
+}
+
+static int tegra_aes_get_random(struct crypto_rng *tfm, u8 *rdata,
+ unsigned int dlen)
+{
+ struct tegra_aes_dev *dd = aes_dev;
+ struct tegra_aes_ctx *ctx = &rng_ctx;
+ int ret, i;
+ u8 *dest = rdata, *dt = dd->dt;
+
+ /* take mutex to access the aes hw */
+ mutex_lock(&aes_lock);
+
+ ret = clk_enable(dd->aes_clk);
+ if (ret)
+ return ret;
+
+ ctx->dd = dd;
+ dd->ctx = ctx;
+ dd->flags = FLAGS_ENCRYPT | FLAGS_RNG;
+
+ memcpy(dd->buf_in, dt, DEFAULT_RNG_BLK_SZ);
+
+ ret = aes_start_crypt(dd, (u32)dd->dma_buf_in,
+ (u32)dd->dma_buf_out, 1, dd->flags, true);
+ if (ret < 0) {
+ dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
+ dlen = ret;
+ goto out;
+ }
+ memcpy(dest, dd->buf_out, dlen);
+
+ /* update the DT */
+ for (i = DEFAULT_RNG_BLK_SZ - 1; i >= 0; i--) {
+ dt[i] += 1;
+ if (dt[i] != 0)
+ break;
+ }
+
+out:
+ clk_disable(dd->aes_clk);
+ mutex_unlock(&aes_lock);
+
+ dev_dbg(dd->dev, "%s: done\n", __func__);
+ return dlen;
+}
+
+static int tegra_aes_rng_reset(struct crypto_rng *tfm, u8 *seed,
+ unsigned int slen)
+{
+ struct tegra_aes_dev *dd = aes_dev;
+ struct tegra_aes_ctx *ctx = &rng_ctx;
+ struct tegra_aes_slot *key_slot;
+ struct timespec ts;
+ int ret = 0;
+ u64 nsec, tmp[2];
+ u8 *dt;
+
+ if (!ctx || !dd) {
+ dev_err(dd->dev, "ctx=0x%x, dd=0x%x\n",
+ (unsigned int)ctx, (unsigned int)dd);
+ return -EINVAL;
+ }
+
+ if (slen < (DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128)) {
+ dev_err(dd->dev, "seed size invalid");
+ return -ENOMEM;
+ }
+
+ /* take mutex to access the aes hw */
+ mutex_lock(&aes_lock);
+
+ if (!ctx->slot) {
+ key_slot = aes_find_key_slot();
+ if (!key_slot) {
+ dev_err(dd->dev, "no empty slot\n");
+ mutex_unlock(&aes_lock);
+ return -ENOMEM;
+ }
+ ctx->slot = key_slot;
+ }
+
+ ctx->dd = dd;
+ dd->ctx = ctx;
+ dd->ctr = 0;
+
+ ctx->keylen = AES_KEYSIZE_128;
+ ctx->flags |= FLAGS_NEW_KEY;
+
+ /* copy the key to the key slot */
+ memcpy(dd->ivkey_base, seed + DEFAULT_RNG_BLK_SZ, AES_KEYSIZE_128);
+ memset(dd->ivkey_base + AES_KEYSIZE_128, 0, AES_HW_KEY_TABLE_LENGTH_BYTES - AES_KEYSIZE_128);
+
+ dd->iv = seed;
+ dd->ivlen = slen;
+
+ dd->flags = FLAGS_ENCRYPT | FLAGS_RNG;
+
+ ret = clk_enable(dd->aes_clk);
+ if (ret)
+ return ret;
+
+ aes_set_key(dd);
+
+ /* set seed to the aes hw slot */
+ memcpy(dd->buf_in, dd->iv, DEFAULT_RNG_BLK_SZ);
+ ret = aes_start_crypt(dd, (u32)dd->dma_buf_in,
+ dd->dma_buf_out, 1, FLAGS_CBC, false);
+ if (ret < 0) {
+ dev_err(dd->dev, "aes_start_crypt fail(%d)\n", ret);
+ goto out;
+ }
+
+ if (dd->ivlen >= (2 * DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128)) {
+ dt = dd->iv + DEFAULT_RNG_BLK_SZ + AES_KEYSIZE_128;
+ } else {
+ getnstimeofday(&ts);
+ nsec = timespec_to_ns(&ts);
+ do_div(nsec, 1000);
+ nsec ^= dd->ctr << 56;
+ dd->ctr++;
+ tmp[0] = nsec;
+ tmp[1] = tegra_chip_uid();
+ dt = (u8 *)tmp;
+ }
+ memcpy(dd->dt, dt, DEFAULT_RNG_BLK_SZ);
+
+out:
+ clk_disable(dd->aes_clk);
+ mutex_unlock(&aes_lock);
+
+ dev_dbg(dd->dev, "%s: done\n", __func__);
+ return ret;
+}
+
+static int tegra_aes_cra_init(struct crypto_tfm *tfm)
+{
+ tfm->crt_ablkcipher.reqsize = sizeof(struct tegra_aes_reqctx);
+
+ return 0;
+}
+
+void tegra_aes_cra_exit(struct crypto_tfm *tfm)
+{
+ struct tegra_aes_ctx *ctx =
+ crypto_ablkcipher_ctx((struct crypto_ablkcipher *)tfm);
+
+ if (ctx && ctx->slot)
+ aes_release_key_slot(ctx->slot);
+}
+
+static struct crypto_alg algs[] = {
+ {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-tegra",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_alignmask = 3,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = tegra_aes_setkey,
+ .encrypt = tegra_aes_ecb_encrypt,
+ .decrypt = tegra_aes_ecb_decrypt,
+ },
+ }, {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-tegra",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_alignmask = 3,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_MIN_KEY_SIZE,
+ .setkey = tegra_aes_setkey,
+ .encrypt = tegra_aes_cbc_encrypt,
+ .decrypt = tegra_aes_cbc_decrypt,
+ }
+ }, {
+ .cra_name = "ofb(aes)",
+ .cra_driver_name = "ofb-aes-tegra",
+ .cra_priority = 300,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_alignmask = 3,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_u.ablkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_MIN_KEY_SIZE,
+ .setkey = tegra_aes_setkey,
+ .encrypt = tegra_aes_ofb_encrypt,
+ .decrypt = tegra_aes_ofb_decrypt,
+ }
+ }, {
+ .cra_name = "ansi_cprng",
+ .cra_driver_name = "rng-aes-tegra",
+ .cra_flags = CRYPTO_ALG_TYPE_RNG,
+ .cra_ctxsize = sizeof(struct tegra_aes_ctx),
+ .cra_type = &crypto_rng_type,
+ .cra_u.rng = {
+ .rng_make_random = tegra_aes_get_random,
+ .rng_reset = tegra_aes_rng_reset,
+ .seedsize = AES_KEYSIZE_128 + (2 * DEFAULT_RNG_BLK_SZ),
+ }
+ }
+};
+
+static int tegra_aes_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct tegra_aes_dev *dd;
+ struct resource *res;
+ int err = -ENOMEM, i = 0, j;
+
+ dd = devm_kzalloc(dev, sizeof(struct tegra_aes_dev), GFP_KERNEL);
+ if (dd == NULL) {
+ dev_err(dev, "unable to alloc data struct.\n");
+ return err;
+ }
+
+ dd->dev = dev;
+ platform_set_drvdata(pdev, dd);
+
+ dd->slots = devm_kzalloc(dev, sizeof(struct tegra_aes_slot) *
+ AES_NR_KEYSLOTS, GFP_KERNEL);
+ if (dd->slots == NULL) {
+ dev_err(dev, "unable to alloc slot struct.\n");
+ goto out;
+ }
+
+ spin_lock_init(&dd->lock);
+ crypto_init_queue(&dd->queue, TEGRA_AES_QUEUE_LENGTH);
+
+ /* Get the module base address */
+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!res) {
+ dev_err(dev, "invalid resource type: base\n");
+ err = -ENODEV;
+ goto out;
+ }
+
+ if (!devm_request_mem_region(&pdev->dev, res->start,
+ resource_size(res),
+ dev_name(&pdev->dev))) {
+ dev_err(&pdev->dev, "Couldn't request MEM resource\n");
+ return -ENODEV;
+ }
+
+ dd->io_base = devm_ioremap(dev, res->start, resource_size(res));
+ if (!dd->io_base) {
+ dev_err(dev, "can't ioremap register space\n");
+ err = -ENOMEM;
+ goto out;
+ }
+
+ /* Initialize the vde clock */
+ dd->aes_clk = clk_get(dev, "vde");
+ if (IS_ERR(dd->aes_clk)) {
+ dev_err(dev, "iclock intialization failed.\n");
+ err = -ENODEV;
+ goto out;
+ }
+
+ err = clk_set_rate(dd->aes_clk, ULONG_MAX);
+ if (err) {
+ dev_err(dd->dev, "iclk set_rate fail(%d)\n", err);
+ goto out;
+ }
+
+ /*
+ * the foll contiguous memory is allocated as follows -
+ * - hardware key table
+ * - key schedule
+ */
+ dd->ivkey_base = dma_alloc_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES,
+ &dd->ivkey_phys_base,
+ GFP_KERNEL);
+ if (!dd->ivkey_base) {
+ dev_err(dev, "can not allocate iv/key buffer\n");
+ err = -ENOMEM;
+ goto out;
+ }
+
+ dd->buf_in = dma_alloc_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ &dd->dma_buf_in, GFP_KERNEL);
+ if (!dd->buf_in) {
+ dev_err(dev, "can not allocate dma-in buffer\n");
+ err = -ENOMEM;
+ goto out;
+ }
+
+ dd->buf_out = dma_alloc_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ &dd->dma_buf_out, GFP_KERNEL);
+ if (!dd->buf_out) {
+ dev_err(dev, "can not allocate dma-out buffer\n");
+ err = -ENOMEM;
+ goto out;
+ }
+
+ init_completion(&dd->op_complete);
+ aes_wq = alloc_workqueue("tegra_aes_wq", WQ_HIGHPRI | WQ_UNBOUND, 1);
+ if (!aes_wq) {
+ dev_err(dev, "alloc_workqueue failed\n");
+ goto out;
+ }
+
+ /* get the irq */
+ res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
+ if (!res) {
+ dev_err(dev, "invalid resource type: base\n");
+ err = -ENODEV;
+ goto out;
+ }
+ dd->irq = res->start;
+
+ err = devm_request_irq(dev, dd->irq, aes_irq, IRQF_TRIGGER_HIGH |
+ IRQF_SHARED, "tegra-aes", dd);
+ if (err) {
+ dev_err(dev, "request_irq failed\n");
+ goto out;
+ }
+
+ mutex_init(&aes_lock);
+ INIT_LIST_HEAD(&dev_list);
+
+ spin_lock_init(&list_lock);
+ spin_lock(&list_lock);
+ for (i = 0; i < AES_NR_KEYSLOTS; i++) {
+ if (i == SSK_SLOT_NUM)
+ continue;
+ dd->slots[i].slot_num = i;
+ INIT_LIST_HEAD(&dd->slots[i].node);
+ list_add_tail(&dd->slots[i].node, &dev_list);
+ }
+ spin_unlock(&list_lock);
+
+ aes_dev = dd;
+ for (i = 0; i < ARRAY_SIZE(algs); i++) {
+ INIT_LIST_HEAD(&algs[i].cra_list);
+
+ algs[i].cra_priority = 300;
+ algs[i].cra_ctxsize = sizeof(struct tegra_aes_ctx);
+ algs[i].cra_module = THIS_MODULE;
+ algs[i].cra_init = tegra_aes_cra_init;
+ algs[i].cra_exit = tegra_aes_cra_exit;
+
+ err = crypto_register_alg(&algs[i]);
+ if (err)
+ goto out;
+ }
+
+ dev_info(dev, "registered");
+ return 0;
+
+out:
+ for (j = 0; j < i; j++)
+ crypto_unregister_alg(&algs[j]);
+ if (dd->ivkey_base)
+ dma_free_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES,
+ dd->ivkey_base, dd->ivkey_phys_base);
+ if (dd->buf_in)
+ dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ dd->buf_in, dd->dma_buf_in);
+ if (dd->buf_out)
+ dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ dd->buf_out, dd->dma_buf_out);
+ if (IS_ERR(dd->aes_clk))
+ clk_put(dd->aes_clk);
+ if (aes_wq)
+ destroy_workqueue(aes_wq);
+ spin_lock(&list_lock);
+ list_del(&dev_list);
+ spin_unlock(&list_lock);
+
+ aes_dev = NULL;
+
+ dev_err(dev, "%s: initialization failed.\n", __func__);
+ return err;
+}
+
+static int __devexit tegra_aes_remove(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct tegra_aes_dev *dd = platform_get_drvdata(pdev);
+ int i;
+
+ for (i = 0; i < ARRAY_SIZE(algs); i++)
+ crypto_unregister_alg(&algs[i]);
+
+ cancel_work_sync(&aes_work);
+ destroy_workqueue(aes_wq);
+ spin_lock(&list_lock);
+ list_del(&dev_list);
+ spin_unlock(&list_lock);
+
+ dma_free_coherent(dev, AES_HW_KEY_TABLE_LENGTH_BYTES,
+ dd->ivkey_base, dd->ivkey_phys_base);
+ dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ dd->buf_in, dd->dma_buf_in);
+ dma_free_coherent(dev, AES_HW_DMA_BUFFER_SIZE_BYTES,
+ dd->buf_out, dd->dma_buf_out);
+ clk_put(dd->aes_clk);
+ aes_dev = NULL;
+
+ return 0;
+}
+
+static struct of_device_id tegra_aes_of_match[] __devinitdata = {
+ { .compatible = "nvidia,tegra20-aes", },
+ { .compatible = "nvidia,tegra30-aes", },
+ { },
+};
+
+static struct platform_driver tegra_aes_driver = {
+ .probe = tegra_aes_probe,
+ .remove = __devexit_p(tegra_aes_remove),
+ .driver = {
+ .name = "tegra-aes",
+ .owner = THIS_MODULE,
+ .of_match_table = tegra_aes_of_match,
+ },
+};
+
+module_platform_driver(tegra_aes_driver);
+
+MODULE_DESCRIPTION("Tegra AES/OFB/CPRNG hw acceleration support.");
+MODULE_AUTHOR("NVIDIA Corporation");
+MODULE_LICENSE("GPL v2");
diff --git a/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.h b/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.h
new file mode 100644
index 0000000..6006333
--- /dev/null
+++ b/ap/os/linux/linux-3.4.x/drivers/crypto/tegra-aes.h
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2010, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __CRYPTODEV_TEGRA_AES_H
+#define __CRYPTODEV_TEGRA_AES_H
+
+#define TEGRA_AES_ICMDQUE_WR 0x1000
+#define TEGRA_AES_CMDQUE_CONTROL 0x1008
+#define TEGRA_AES_INTR_STATUS 0x1018
+#define TEGRA_AES_INT_ENB 0x1040
+#define TEGRA_AES_CONFIG 0x1044
+#define TEGRA_AES_IRAM_ACCESS_CFG 0x10A0
+#define TEGRA_AES_SECURE_DEST_ADDR 0x1100
+#define TEGRA_AES_SECURE_INPUT_SELECT 0x1104
+#define TEGRA_AES_SECURE_CONFIG 0x1108
+#define TEGRA_AES_SECURE_CONFIG_EXT 0x110C
+#define TEGRA_AES_SECURE_SECURITY 0x1110
+#define TEGRA_AES_SECURE_HASH_RESULT0 0x1120
+#define TEGRA_AES_SECURE_HASH_RESULT1 0x1124
+#define TEGRA_AES_SECURE_HASH_RESULT2 0x1128
+#define TEGRA_AES_SECURE_HASH_RESULT3 0x112C
+#define TEGRA_AES_SECURE_SEC_SEL0 0x1140
+#define TEGRA_AES_SECURE_SEC_SEL1 0x1144
+#define TEGRA_AES_SECURE_SEC_SEL2 0x1148
+#define TEGRA_AES_SECURE_SEC_SEL3 0x114C
+#define TEGRA_AES_SECURE_SEC_SEL4 0x1150
+#define TEGRA_AES_SECURE_SEC_SEL5 0x1154
+#define TEGRA_AES_SECURE_SEC_SEL6 0x1158
+#define TEGRA_AES_SECURE_SEC_SEL7 0x115C
+
+/* interrupt status reg masks and shifts */
+#define TEGRA_AES_ENGINE_BUSY_FIELD BIT(0)
+#define TEGRA_AES_ICQ_EMPTY_FIELD BIT(3)
+#define TEGRA_AES_DMA_BUSY_FIELD BIT(23)
+
+/* secure select reg masks and shifts */
+#define TEGRA_AES_SECURE_SEL0_KEYREAD_ENB0_FIELD BIT(0)
+
+/* secure config ext masks and shifts */
+#define TEGRA_AES_SECURE_KEY_SCH_DIS_FIELD BIT(15)
+
+/* secure config masks and shifts */
+#define TEGRA_AES_SECURE_KEY_INDEX_SHIFT 20
+#define TEGRA_AES_SECURE_KEY_INDEX_FIELD (0x1F << TEGRA_AES_SECURE_KEY_INDEX_SHIFT)
+#define TEGRA_AES_SECURE_BLOCK_CNT_SHIFT 0
+#define TEGRA_AES_SECURE_BLOCK_CNT_FIELD (0xFFFFF << TEGRA_AES_SECURE_BLOCK_CNT_SHIFT)
+
+/* stream interface select masks and shifts */
+#define TEGRA_AES_CMDQ_CTRL_UCMDQEN_FIELD BIT(0)
+#define TEGRA_AES_CMDQ_CTRL_ICMDQEN_FIELD BIT(1)
+#define TEGRA_AES_CMDQ_CTRL_SRC_STM_SEL_FIELD BIT(4)
+#define TEGRA_AES_CMDQ_CTRL_DST_STM_SEL_FIELD BIT(5)
+
+/* config register masks and shifts */
+#define TEGRA_AES_CONFIG_ENDIAN_ENB_FIELD BIT(10)
+#define TEGRA_AES_CONFIG_MODE_SEL_SHIFT 0
+#define TEGRA_AES_CONFIG_MODE_SEL_FIELD (0x1F << TEGRA_AES_CONFIG_MODE_SEL_SHIFT)
+
+/* extended config */
+#define TEGRA_AES_SECURE_OFFSET_CNT_SHIFT 24
+#define TEGRA_AES_SECURE_OFFSET_CNT_FIELD (0xFF << TEGRA_AES_SECURE_OFFSET_CNT_SHIFT)
+#define TEGRA_AES_SECURE_KEYSCHED_GEN_FIELD BIT(15)
+
+/* init vector select */
+#define TEGRA_AES_SECURE_IV_SELECT_SHIFT 10
+#define TEGRA_AES_SECURE_IV_SELECT_FIELD BIT(10)
+
+/* secure engine input */
+#define TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT 28
+#define TEGRA_AES_SECURE_INPUT_ALG_SEL_FIELD (0xF << TEGRA_AES_SECURE_INPUT_ALG_SEL_SHIFT)
+#define TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT 16
+#define TEGRA_AES_SECURE_INPUT_KEY_LEN_FIELD (0xFFF << TEGRA_AES_SECURE_INPUT_KEY_LEN_SHIFT)
+#define TEGRA_AES_SECURE_RNG_ENB_FIELD BIT(11)
+#define TEGRA_AES_SECURE_CORE_SEL_SHIFT 9
+#define TEGRA_AES_SECURE_CORE_SEL_FIELD BIT(9)
+#define TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT 7
+#define TEGRA_AES_SECURE_VCTRAM_SEL_FIELD (0x3 << TEGRA_AES_SECURE_VCTRAM_SEL_SHIFT)
+#define TEGRA_AES_SECURE_INPUT_SEL_SHIFT 5
+#define TEGRA_AES_SECURE_INPUT_SEL_FIELD (0x3 << TEGRA_AES_SECURE_INPUT_SEL_SHIFT)
+#define TEGRA_AES_SECURE_XOR_POS_SHIFT 3
+#define TEGRA_AES_SECURE_XOR_POS_FIELD (0x3 << TEGRA_AES_SECURE_XOR_POS_SHIFT)
+#define TEGRA_AES_SECURE_HASH_ENB_FIELD BIT(2)
+#define TEGRA_AES_SECURE_ON_THE_FLY_FIELD BIT(0)
+
+/* interrupt error mask */
+#define TEGRA_AES_INT_ERROR_MASK 0xFFF000
+
+#endif