M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>
S: Maintained
F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-aes.yaml
-F: drivers/crypto/keembay/Kconfig
-F: drivers/crypto/keembay/Makefile
-F: drivers/crypto/keembay/keembay-ocs-aes-core.c
-F: drivers/crypto/keembay/ocs-aes.c
-F: drivers/crypto/keembay/ocs-aes.h
+F: drivers/crypto/intel/keembay/Kconfig
+F: drivers/crypto/intel/keembay/Makefile
+F: drivers/crypto/intel/keembay/keembay-ocs-aes-core.c
+F: drivers/crypto/intel/keembay/ocs-aes.c
+F: drivers/crypto/intel/keembay/ocs-aes.h
INTEL KEEM BAY OCS ECC CRYPTO DRIVER
M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>
M: Mark Gross <mgross@linux.intel.com>
S: Maintained
F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml
-F: drivers/crypto/keembay/Kconfig
-F: drivers/crypto/keembay/Makefile
-F: drivers/crypto/keembay/keembay-ocs-ecc.c
+F: drivers/crypto/intel/keembay/Kconfig
+F: drivers/crypto/intel/keembay/Makefile
+F: drivers/crypto/intel/keembay/keembay-ocs-ecc.c
INTEL KEEM BAY OCS HCU CRYPTO DRIVER
M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>
M: Declan Murphy <declan.murphy@intel.com>
S: Maintained
F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-hcu.yaml
-F: drivers/crypto/keembay/Kconfig
-F: drivers/crypto/keembay/Makefile
-F: drivers/crypto/keembay/keembay-ocs-hcu-core.c
-F: drivers/crypto/keembay/ocs-hcu.c
-F: drivers/crypto/keembay/ocs-hcu.h
+F: drivers/crypto/intel/keembay/Kconfig
+F: drivers/crypto/intel/keembay/Makefile
+F: drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c
+F: drivers/crypto/intel/keembay/ocs-hcu.c
+F: drivers/crypto/intel/keembay/ocs-hcu.h
INTEL THUNDER BAY EMMC PHY DRIVER
M: Nandhini Srikandan <nandhini.srikandan@intel.com>
source "drivers/crypto/cavium/cpt/Kconfig"
source "drivers/crypto/cavium/nitrox/Kconfig"
source "drivers/crypto/marvell/Kconfig"
+source "drivers/crypto/intel/Kconfig"
config CRYPTO_DEV_CAVIUM_ZIP
tristate "Cavium ZIP driver"
used for crypto offload. Select this if you want to use hardware
acceleration for cryptographic algorithms on these devices.
-source "drivers/crypto/keembay/Kconfig"
source "drivers/crypto/aspeed/Kconfig"
endif # CRYPTO_HW
obj-y += xilinx/
obj-y += hisilicon/
obj-$(CONFIG_CRYPTO_DEV_AMLOGIC_GXL) += amlogic/
-obj-y += keembay/
+obj-y += intel/
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+source "drivers/crypto/intel/keembay/Kconfig"
--- /dev/null
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y += keembay/
--- /dev/null
+config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
+ tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration"
+ depends on HAS_IOMEM
+ depends on ARCH_KEEMBAY || COMPILE_TEST
+ select CRYPTO_SKCIPHER
+ select CRYPTO_AEAD
+ select CRYPTO_ENGINE
+ help
+ Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and
+ SM4 cipher hardware acceleration for use with Crypto API.
+
+ Provides HW acceleration for the following transformations:
+ cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4)
+ and gcm(sm4).
+
+ Optionally, support for the following transformations can also be
+ enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)).
+
+config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+ bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration"
+ depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
+ help
+ Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
+ AES/SM4 ECB mode hardware acceleration for use with Crypto API.
+
+ Provides OCS version of ecb(aes) and ecb(sm4)
+
+ Intel does not recommend use of ECB mode with AES/SM4.
+
+config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+ bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration"
+ depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
+ help
+ Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
+ AES/SM4 CBC with CTS mode hardware acceleration for use with
+ Crypto API.
+
+ Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)).
+
+ Intel does not recommend use of CTS mode with AES/SM4.
+
+config CRYPTO_DEV_KEEMBAY_OCS_ECC
+ tristate "Support for Intel Keem Bay OCS ECC HW acceleration"
+ depends on ARCH_KEEMBAY || COMPILE_TEST
+ depends on OF
+ depends on HAS_IOMEM
+ select CRYPTO_ECDH
+ select CRYPTO_ENGINE
+ help
+ Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
+ Elliptic Curve Cryptography (ECC) hardware acceleration for use with
+ Crypto API.
+
+ Provides OCS acceleration for ECDH-256 and ECDH-384.
+
+ Say Y or M if you are compiling for the Intel Keem Bay SoC. The
+ module will be called keembay-ocs-ecc.
+
+ If unsure, say N.
+
+config CRYPTO_DEV_KEEMBAY_OCS_HCU
+ tristate "Support for Intel Keem Bay OCS HCU HW acceleration"
+ select CRYPTO_HASH
+ select CRYPTO_ENGINE
+ depends on HAS_IOMEM
+ depends on ARCH_KEEMBAY || COMPILE_TEST
+ depends on OF
+ help
+ Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash
+ Control Unit (HCU) hardware acceleration for use with Crypto API.
+
+ Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and
+ sm3, as well as the HMAC variant of these algorithms.
+
+ Say Y or M if you're building for the Intel Keem Bay SoC. If compiled
+ as a module, the module will be called keembay-ocs-hcu.
+
+ If unsure, say N.
+
+config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
+ bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU"
+ depends on CRYPTO_DEV_KEEMBAY_OCS_HCU
+ help
+ Enables support for sha224 and hmac(sha224) algorithms in the Intel
+ Keem Bay OCS HCU driver. Intel recommends not to use these
+ algorithms.
+
+ Provides OCS HCU hardware acceleration of sha224 and hmac(224).
+
+ If unsure, say N.
--- /dev/null
+#
+# Makefile for Intel Keem Bay OCS Crypto API Linux drivers
+#
+obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o
+keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o
+
+obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o
+
+obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o
+keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Keem Bay OCS AES Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#include <crypto/aes.h>
+#include <crypto/engine.h>
+#include <crypto/gcm.h>
+#include <crypto/scatterwalk.h>
+
+#include <crypto/internal/aead.h>
+#include <crypto/internal/skcipher.h>
+
+#include "ocs-aes.h"
+
+#define KMB_OCS_PRIORITY 350
+#define DRV_NAME "keembay-ocs-aes"
+
+#define OCS_AES_MIN_KEY_SIZE 16
+#define OCS_AES_MAX_KEY_SIZE 32
+#define OCS_AES_KEYSIZE_128 16
+#define OCS_AES_KEYSIZE_192 24
+#define OCS_AES_KEYSIZE_256 32
+#define OCS_SM4_KEY_SIZE 16
+
+/**
+ * struct ocs_aes_tctx - OCS AES Transform context
+ * @engine_ctx: Engine context.
+ * @aes_dev: The OCS AES device.
+ * @key: AES/SM4 key.
+ * @key_len: The length (in bytes) of @key.
+ * @cipher: OCS cipher to use (either AES or SM4).
+ * @sw_cipher: The cipher to use as fallback.
+ * @use_fallback: Whether or not fallback cipher should be used.
+ */
+struct ocs_aes_tctx {
+ struct crypto_engine_ctx engine_ctx;
+ struct ocs_aes_dev *aes_dev;
+ u8 key[OCS_AES_KEYSIZE_256];
+ unsigned int key_len;
+ enum ocs_cipher cipher;
+ union {
+ struct crypto_sync_skcipher *sk;
+ struct crypto_aead *aead;
+ } sw_cipher;
+ bool use_fallback;
+};
+
+/**
+ * struct ocs_aes_rctx - OCS AES Request context.
+ * @instruction: Instruction to be executed (encrypt / decrypt).
+ * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS)
+ * @src_nents: Number of source SG entries.
+ * @dst_nents: Number of destination SG entries.
+ * @src_dma_count: The number of DMA-mapped entries of the source SG.
+ * @dst_dma_count: The number of DMA-mapped entries of the destination SG.
+ * @in_place: Whether or not this is an in place request, i.e.,
+ * src_sg == dst_sg.
+ * @src_dll: OCS DMA linked list for input data.
+ * @dst_dll: OCS DMA linked list for output data.
+ * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC
+ * mode).
+ * @cts_swap: Whether or not CTS swap must be performed.
+ * @aad_src_dll: OCS DMA linked list for input AAD data.
+ * @aad_dst_dll: OCS DMA linked list for output AAD data.
+ * @in_tag: Buffer to hold input encrypted tag (only used for
+ * CCM/GCM decrypt).
+ * @out_tag: Buffer to hold output encrypted / decrypted tag (only
+ * used for GCM encrypt / decrypt).
+ */
+struct ocs_aes_rctx {
+ /* Fields common across all modes. */
+ enum ocs_instruction instruction;
+ enum ocs_mode mode;
+ int src_nents;
+ int dst_nents;
+ int src_dma_count;
+ int dst_dma_count;
+ bool in_place;
+ struct ocs_dll_desc src_dll;
+ struct ocs_dll_desc dst_dll;
+
+ /* CBC specific */
+ u8 last_ct_blk[AES_BLOCK_SIZE];
+
+ /* CTS specific */
+ int cts_swap;
+
+ /* CCM/GCM specific */
+ struct ocs_dll_desc aad_src_dll;
+ struct ocs_dll_desc aad_dst_dll;
+ u8 in_tag[AES_BLOCK_SIZE];
+
+ /* GCM specific */
+ u8 out_tag[AES_BLOCK_SIZE];
+};
+
+/* Driver data. */
+struct ocs_aes_drv {
+ struct list_head dev_list;
+ spinlock_t lock; /* Protects dev_list. */
+};
+
+static struct ocs_aes_drv ocs_aes = {
+ .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list),
+ .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock),
+};
+
+static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx)
+{
+ struct ocs_aes_dev *aes_dev;
+
+ spin_lock(&ocs_aes.lock);
+
+ if (tctx->aes_dev) {
+ aes_dev = tctx->aes_dev;
+ goto exit;
+ }
+
+ /* Only a single OCS device available */
+ aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list);
+ tctx->aes_dev = aes_dev;
+
+exit:
+ spin_unlock(&ocs_aes.lock);
+
+ return aes_dev;
+}
+
+/*
+ * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual
+ * key is being passed in.
+ *
+ * Return: 0 if key is valid, -EINVAL otherwise.
+ */
+static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher)
+{
+ if (!in_key)
+ return -EINVAL;
+
+ /* For AES, only 128-byte or 256-byte keys are supported. */
+ if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 ||
+ key_len == OCS_AES_KEYSIZE_256))
+ return 0;
+
+ /* For SM4, only 128-byte keys are supported. */
+ if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128)
+ return 0;
+
+ /* Everything else is unsupported. */
+ return -EINVAL;
+}
+
+/* Save key into transformation context. */
+static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len,
+ enum ocs_cipher cipher)
+{
+ int ret;
+
+ ret = check_key(in_key, key_len, cipher);
+ if (ret)
+ return ret;
+
+ memcpy(tctx->key, in_key, key_len);
+ tctx->key_len = key_len;
+ tctx->cipher = cipher;
+
+ return 0;
+}
+
+/* Set key for symmetric cypher. */
+static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+ size_t key_len, enum ocs_cipher cipher)
+{
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+
+ /* Fallback is used for AES with 192-bit key. */
+ tctx->use_fallback = (cipher == OCS_AES &&
+ key_len == OCS_AES_KEYSIZE_192);
+
+ if (!tctx->use_fallback)
+ return save_key(tctx, in_key, key_len, cipher);
+
+ crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk,
+ CRYPTO_TFM_REQ_MASK);
+ crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk,
+ tfm->base.crt_flags &
+ CRYPTO_TFM_REQ_MASK);
+
+ return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len);
+}
+
+/* Set key for AEAD cipher. */
+static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
+ size_t key_len, enum ocs_cipher cipher)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
+
+ /* Fallback is used for AES with 192-bit key. */
+ tctx->use_fallback = (cipher == OCS_AES &&
+ key_len == OCS_AES_KEYSIZE_192);
+
+ if (!tctx->use_fallback)
+ return save_key(tctx, in_key, key_len, cipher);
+
+ crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
+ crypto_aead_set_flags(tctx->sw_cipher.aead,
+ crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
+
+ return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len);
+}
+
+/* Swap two AES blocks in SG lists. */
+static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents,
+ off_t blk1_offset, off_t blk2_offset)
+{
+ u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE];
+
+ /*
+ * No easy way to copy within sg list, so copy both blocks to temporary
+ * buffers first.
+ */
+ sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset);
+ sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset);
+ sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset);
+ sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset);
+}
+
+/* Initialize request context to default values. */
+static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx)
+{
+ /* Zero everything. */
+ memset(rctx, 0, sizeof(*rctx));
+
+ /* Set initial value for DMA addresses. */
+ rctx->src_dll.dma_addr = DMA_MAPPING_ERROR;
+ rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR;
+ rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR;
+ rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR;
+}
+
+static int kmb_ocs_sk_validate_input(struct skcipher_request *req,
+ enum ocs_mode mode)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ int iv_size = crypto_skcipher_ivsize(tfm);
+
+ switch (mode) {
+ case OCS_MODE_ECB:
+ /* Ensure input length is multiple of block size */
+ if (req->cryptlen % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CBC:
+ /* Ensure input length is multiple of block size */
+ if (req->cryptlen % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ /* Ensure IV is present and block size in length */
+ if (!req->iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+ /*
+ * NOTE: Since req->cryptlen == 0 case was already handled in
+ * kmb_ocs_sk_common(), the above two conditions also guarantee
+ * that: cryptlen >= iv_size
+ */
+ return 0;
+
+ case OCS_MODE_CTR:
+ /* Ensure IV is present and block size in length */
+ if (!req->iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+ return 0;
+
+ case OCS_MODE_CTS:
+ /* Ensure input length >= block size */
+ if (req->cryptlen < AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ /* Ensure IV is present and block size in length */
+ if (!req->iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+/*
+ * Called by encrypt() / decrypt() skcipher functions.
+ *
+ * Use fallback if needed, otherwise initialize context and enqueue request
+ * into engine.
+ */
+static int kmb_ocs_sk_common(struct skcipher_request *req,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ enum ocs_mode mode)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ struct ocs_aes_dev *aes_dev;
+ int rc;
+
+ if (tctx->use_fallback) {
+ SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk);
+
+ skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk);
+ skcipher_request_set_callback(subreq, req->base.flags, NULL,
+ NULL);
+ skcipher_request_set_crypt(subreq, req->src, req->dst,
+ req->cryptlen, req->iv);
+
+ if (instruction == OCS_ENCRYPT)
+ rc = crypto_skcipher_encrypt(subreq);
+ else
+ rc = crypto_skcipher_decrypt(subreq);
+
+ skcipher_request_zero(subreq);
+
+ return rc;
+ }
+
+ /*
+ * If cryptlen == 0, no processing needed for ECB, CBC and CTR.
+ *
+ * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL.
+ */
+ if (!req->cryptlen && mode != OCS_MODE_CTS)
+ return 0;
+
+ rc = kmb_ocs_sk_validate_input(req, mode);
+ if (rc)
+ return rc;
+
+ aes_dev = kmb_ocs_aes_find_dev(tctx);
+ if (!aes_dev)
+ return -ENODEV;
+
+ if (cipher != tctx->cipher)
+ return -EINVAL;
+
+ ocs_aes_init_rctx(rctx);
+ rctx->instruction = instruction;
+ rctx->mode = mode;
+
+ return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req);
+}
+
+static void cleanup_ocs_dma_linked_list(struct device *dev,
+ struct ocs_dll_desc *dll)
+{
+ if (dll->vaddr)
+ dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr);
+ dll->vaddr = NULL;
+ dll->size = 0;
+ dll->dma_addr = DMA_MAPPING_ERROR;
+}
+
+static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ struct device *dev = tctx->aes_dev->dev;
+
+ if (rctx->src_dma_count) {
+ dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
+ rctx->src_dma_count = 0;
+ }
+
+ if (rctx->dst_dma_count) {
+ dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
+ DMA_BIDIRECTIONAL :
+ DMA_FROM_DEVICE);
+ rctx->dst_dma_count = 0;
+ }
+
+ /* Clean up OCS DMA linked lists */
+ cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
+ cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
+}
+
+static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ int iv_size = crypto_skcipher_ivsize(tfm);
+ int rc;
+
+ /*
+ * For CBC decrypt, save last block (iv) to last_ct_blk buffer.
+ *
+ * Note: if we are here, we already checked that cryptlen >= iv_size
+ * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see
+ * kmb_ocs_sk_validate_input().
+ */
+ if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT)
+ scatterwalk_map_and_copy(rctx->last_ct_blk, req->src,
+ req->cryptlen - iv_size, iv_size, 0);
+
+ /* For CTS decrypt, swap last two blocks, if needed. */
+ if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT)
+ sg_swap_blocks(req->dst, rctx->dst_nents,
+ req->cryptlen - AES_BLOCK_SIZE,
+ req->cryptlen - (2 * AES_BLOCK_SIZE));
+
+ /* src and dst buffers are the same, use bidirectional DMA mapping. */
+ rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
+ rctx->dst_nents, DMA_BIDIRECTIONAL);
+ if (rctx->dst_dma_count == 0) {
+ dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
+ return -ENOMEM;
+ }
+
+ /* Create DST linked list */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count, &rctx->dst_dll,
+ req->cryptlen, 0);
+ if (rc)
+ return rc;
+ /*
+ * If descriptor creation was successful, set the src_dll.dma_addr to
+ * the value of dst_dll.dma_addr, as we do in-place AES operation on
+ * the src.
+ */
+ rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
+
+ return 0;
+}
+
+static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ int rc;
+
+ rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen);
+ if (rctx->src_nents < 0)
+ return -EBADMSG;
+
+ /* Map SRC SG. */
+ rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
+ rctx->src_nents, DMA_TO_DEVICE);
+ if (rctx->src_dma_count == 0) {
+ dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
+ return -ENOMEM;
+ }
+
+ /* Create SRC linked list */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
+ rctx->src_dma_count, &rctx->src_dll,
+ req->cryptlen, 0);
+ if (rc)
+ return rc;
+
+ /* Map DST SG. */
+ rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
+ rctx->dst_nents, DMA_FROM_DEVICE);
+ if (rctx->dst_dma_count == 0) {
+ dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
+ return -ENOMEM;
+ }
+
+ /* Create DST linked list */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count, &rctx->dst_dll,
+ req->cryptlen, 0);
+ if (rc)
+ return rc;
+
+ /* If this is not a CTS decrypt operation with swapping, we are done. */
+ if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT))
+ return 0;
+
+ /*
+ * Otherwise, we have to copy src to dst (as we cannot modify src).
+ * Use OCS AES bypass mode to copy src to dst via DMA.
+ *
+ * NOTE: for anything other than small data sizes this is rather
+ * inefficient.
+ */
+ rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr,
+ rctx->src_dll.dma_addr, req->cryptlen);
+ if (rc)
+ return rc;
+
+ /*
+ * Now dst == src, so clean up what we did so far and use in_place
+ * logic.
+ */
+ kmb_ocs_sk_dma_cleanup(req);
+ rctx->in_place = true;
+
+ return kmb_ocs_sk_prepare_inplace(req);
+}
+
+static int kmb_ocs_sk_run(struct skcipher_request *req)
+{
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ struct ocs_aes_dev *aes_dev = tctx->aes_dev;
+ int iv_size = crypto_skcipher_ivsize(tfm);
+ int rc;
+
+ rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
+ if (rctx->dst_nents < 0)
+ return -EBADMSG;
+
+ /*
+ * If 2 blocks or greater, and multiple of block size swap last two
+ * blocks to be compatible with other crypto API CTS implementations:
+ * OCS mode uses CBC-CS2, whereas other crypto API implementations use
+ * CBC-CS3.
+ * CBC-CS2 and CBC-CS3 defined by:
+ * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf
+ */
+ rctx->cts_swap = (rctx->mode == OCS_MODE_CTS &&
+ req->cryptlen > AES_BLOCK_SIZE &&
+ req->cryptlen % AES_BLOCK_SIZE == 0);
+
+ rctx->in_place = (req->src == req->dst);
+
+ if (rctx->in_place)
+ rc = kmb_ocs_sk_prepare_inplace(req);
+ else
+ rc = kmb_ocs_sk_prepare_notinplace(req);
+
+ if (rc)
+ goto error;
+
+ rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction,
+ rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr,
+ req->cryptlen, req->iv, iv_size);
+ if (rc)
+ goto error;
+
+ /* Clean-up DMA before further processing output. */
+ kmb_ocs_sk_dma_cleanup(req);
+
+ /* For CTS Encrypt, swap last 2 blocks, if needed. */
+ if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) {
+ sg_swap_blocks(req->dst, rctx->dst_nents,
+ req->cryptlen - AES_BLOCK_SIZE,
+ req->cryptlen - (2 * AES_BLOCK_SIZE));
+ return 0;
+ }
+
+ /* For CBC copy IV to req->IV. */
+ if (rctx->mode == OCS_MODE_CBC) {
+ /* CBC encrypt case. */
+ if (rctx->instruction == OCS_ENCRYPT) {
+ scatterwalk_map_and_copy(req->iv, req->dst,
+ req->cryptlen - iv_size,
+ iv_size, 0);
+ return 0;
+ }
+ /* CBC decrypt case. */
+ if (rctx->in_place)
+ memcpy(req->iv, rctx->last_ct_blk, iv_size);
+ else
+ scatterwalk_map_and_copy(req->iv, req->src,
+ req->cryptlen - iv_size,
+ iv_size, 0);
+ return 0;
+ }
+ /* For all other modes there's nothing to do. */
+
+ return 0;
+
+error:
+ kmb_ocs_sk_dma_cleanup(req);
+
+ return rc;
+}
+
+static int kmb_ocs_aead_validate_input(struct aead_request *req,
+ enum ocs_instruction instruction,
+ enum ocs_mode mode)
+{
+ struct crypto_aead *tfm = crypto_aead_reqtfm(req);
+ int tag_size = crypto_aead_authsize(tfm);
+ int iv_size = crypto_aead_ivsize(tfm);
+
+ /* For decrypt crytplen == len(PT) + len(tag). */
+ if (instruction == OCS_DECRYPT && req->cryptlen < tag_size)
+ return -EINVAL;
+
+ /* IV is mandatory. */
+ if (!req->iv)
+ return -EINVAL;
+
+ switch (mode) {
+ case OCS_MODE_GCM:
+ if (iv_size != GCM_AES_IV_SIZE)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CCM:
+ /* Ensure IV is present and block size in length */
+ if (iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ return 0;
+
+ default:
+ return -EINVAL;
+ }
+}
+
+/*
+ * Called by encrypt() / decrypt() aead functions.
+ *
+ * Use fallback if needed, otherwise initialize context and enqueue request
+ * into engine.
+ */
+static int kmb_ocs_aead_common(struct aead_request *req,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ enum ocs_mode mode)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct ocs_aes_rctx *rctx = aead_request_ctx(req);
+ struct ocs_aes_dev *dd;
+ int rc;
+
+ if (tctx->use_fallback) {
+ struct aead_request *subreq = aead_request_ctx(req);
+
+ aead_request_set_tfm(subreq, tctx->sw_cipher.aead);
+ aead_request_set_callback(subreq, req->base.flags,
+ req->base.complete, req->base.data);
+ aead_request_set_crypt(subreq, req->src, req->dst,
+ req->cryptlen, req->iv);
+ aead_request_set_ad(subreq, req->assoclen);
+ rc = crypto_aead_setauthsize(tctx->sw_cipher.aead,
+ crypto_aead_authsize(crypto_aead_reqtfm(req)));
+ if (rc)
+ return rc;
+
+ return (instruction == OCS_ENCRYPT) ?
+ crypto_aead_encrypt(subreq) :
+ crypto_aead_decrypt(subreq);
+ }
+
+ rc = kmb_ocs_aead_validate_input(req, instruction, mode);
+ if (rc)
+ return rc;
+
+ dd = kmb_ocs_aes_find_dev(tctx);
+ if (!dd)
+ return -ENODEV;
+
+ if (cipher != tctx->cipher)
+ return -EINVAL;
+
+ ocs_aes_init_rctx(rctx);
+ rctx->instruction = instruction;
+ rctx->mode = mode;
+
+ return crypto_transfer_aead_request_to_engine(dd->engine, req);
+}
+
+static void kmb_ocs_aead_dma_cleanup(struct aead_request *req)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ struct ocs_aes_rctx *rctx = aead_request_ctx(req);
+ struct device *dev = tctx->aes_dev->dev;
+
+ if (rctx->src_dma_count) {
+ dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
+ rctx->src_dma_count = 0;
+ }
+
+ if (rctx->dst_dma_count) {
+ dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
+ DMA_BIDIRECTIONAL :
+ DMA_FROM_DEVICE);
+ rctx->dst_dma_count = 0;
+ }
+ /* Clean up OCS DMA linked lists */
+ cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
+ cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
+ cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll);
+ cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll);
+}
+
+/**
+ * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing.
+ * @req: The AEAD request being processed.
+ * @src_dll_size: Where to store the length of the data mapped into the
+ * src_dll OCS DMA list.
+ *
+ * Do the following:
+ * - DMA map req->src and req->dst
+ * - Initialize the following OCS DMA linked lists: rctx->src_dll,
+ * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ struct ocs_aes_rctx *rctx = aead_request_ctx(req);
+ u32 in_size; /* The length of the data to be mapped by src_dll. */
+ u32 out_size; /* The length of the data to be mapped by dst_dll. */
+ u32 dst_size; /* The length of the data in dst_sg. */
+ int rc;
+
+ /* Get number of entries in input data SG list. */
+ rctx->src_nents = sg_nents_for_len(req->src,
+ req->assoclen + req->cryptlen);
+ if (rctx->src_nents < 0)
+ return -EBADMSG;
+
+ if (rctx->instruction == OCS_DECRYPT) {
+ /*
+ * For decrypt:
+ * - src sg list is: AAD|CT|tag
+ * - dst sg list expects: AAD|PT
+ *
+ * in_size == len(CT); out_size == len(PT)
+ */
+
+ /* req->cryptlen includes both CT and tag. */
+ in_size = req->cryptlen - tag_size;
+
+ /* out_size = PT size == CT size */
+ out_size = in_size;
+
+ /* len(dst_sg) == len(AAD) + len(PT) */
+ dst_size = req->assoclen + out_size;
+
+ /*
+ * Copy tag from source SG list to 'in_tag' buffer.
+ *
+ * Note: this needs to be done here, before DMA mapping src_sg.
+ */
+ sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag,
+ tag_size, req->assoclen + in_size);
+
+ } else { /* OCS_ENCRYPT */
+ /*
+ * For encrypt:
+ * src sg list is: AAD|PT
+ * dst sg list expects: AAD|CT|tag
+ */
+ /* in_size == len(PT) */
+ in_size = req->cryptlen;
+
+ /*
+ * In CCM mode the OCS engine appends the tag to the ciphertext,
+ * but in GCM mode the tag must be read from the tag registers
+ * and appended manually below
+ */
+ out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size :
+ in_size;
+ /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */
+ dst_size = req->assoclen + in_size + tag_size;
+ }
+ *src_dll_size = in_size;
+
+ /* Get number of entries in output data SG list. */
+ rctx->dst_nents = sg_nents_for_len(req->dst, dst_size);
+ if (rctx->dst_nents < 0)
+ return -EBADMSG;
+
+ rctx->in_place = (req->src == req->dst) ? 1 : 0;
+
+ /* Map destination; use bidirectional mapping for in-place case. */
+ rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
+ rctx->dst_nents,
+ rctx->in_place ? DMA_BIDIRECTIONAL :
+ DMA_FROM_DEVICE);
+ if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) {
+ dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
+ return -ENOMEM;
+ }
+
+ /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count,
+ &rctx->aad_dst_dll, req->assoclen,
+ 0);
+ if (rc)
+ return rc;
+
+ /* Create DST list: maps dst[AAD_SIZE:out_size] */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count, &rctx->dst_dll,
+ out_size, req->assoclen);
+ if (rc)
+ return rc;
+
+ if (rctx->in_place) {
+ /* If this is not CCM encrypt, we are done. */
+ if (!(rctx->mode == OCS_MODE_CCM &&
+ rctx->instruction == OCS_ENCRYPT)) {
+ /*
+ * SRC and DST are the same, so re-use the same DMA
+ * addresses (to avoid allocating new DMA lists
+ * identical to the dst ones).
+ */
+ rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
+ rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr;
+
+ return 0;
+ }
+ /*
+ * For CCM encrypt the input and output linked lists contain
+ * different amounts of data, so, we need to create different
+ * SRC and AAD SRC lists, even for the in-place case.
+ */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count,
+ &rctx->aad_src_dll,
+ req->assoclen, 0);
+ if (rc)
+ return rc;
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
+ rctx->dst_dma_count,
+ &rctx->src_dll, in_size,
+ req->assoclen);
+ if (rc)
+ return rc;
+
+ return 0;
+ }
+ /* Not in-place case. */
+
+ /* Map source SG. */
+ rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
+ rctx->src_nents, DMA_TO_DEVICE);
+ if (rctx->src_dma_count == 0 && rctx->src_nents != 0) {
+ dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
+ return -ENOMEM;
+ }
+
+ /* Create AAD SRC list. */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
+ rctx->src_dma_count,
+ &rctx->aad_src_dll,
+ req->assoclen, 0);
+ if (rc)
+ return rc;
+
+ /* Create SRC list. */
+ rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
+ rctx->src_dma_count,
+ &rctx->src_dll, in_size,
+ req->assoclen);
+ if (rc)
+ return rc;
+
+ if (req->assoclen == 0)
+ return 0;
+
+ /* Copy AAD from src sg to dst sg using OCS DMA. */
+ rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr,
+ rctx->aad_src_dll.dma_addr, req->cryptlen);
+ if (rc)
+ dev_err(tctx->aes_dev->dev,
+ "Failed to copy source AAD to destination AAD\n");
+
+ return rc;
+}
+
+static int kmb_ocs_aead_run(struct aead_request *req)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
+ struct ocs_aes_rctx *rctx = aead_request_ctx(req);
+ u32 in_size; /* The length of the data mapped by src_dll. */
+ int rc;
+
+ rc = kmb_ocs_aead_dma_prepare(req, &in_size);
+ if (rc)
+ goto exit;
+
+ /* For CCM, we just call the OCS processing and we are done. */
+ if (rctx->mode == OCS_MODE_CCM) {
+ rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher,
+ rctx->instruction, rctx->dst_dll.dma_addr,
+ rctx->src_dll.dma_addr, in_size,
+ req->iv,
+ rctx->aad_src_dll.dma_addr, req->assoclen,
+ rctx->in_tag, tag_size);
+ goto exit;
+ }
+ /* GCM case; invoke OCS processing. */
+ rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher,
+ rctx->instruction,
+ rctx->dst_dll.dma_addr,
+ rctx->src_dll.dma_addr, in_size,
+ req->iv,
+ rctx->aad_src_dll.dma_addr, req->assoclen,
+ rctx->out_tag, tag_size);
+ if (rc)
+ goto exit;
+
+ /* For GCM decrypt, we have to compare in_tag with out_tag. */
+ if (rctx->instruction == OCS_DECRYPT) {
+ rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ?
+ -EBADMSG : 0;
+ goto exit;
+ }
+
+ /* For GCM encrypt, we must manually copy out_tag to DST sg. */
+
+ /* Clean-up must be called before the sg_pcopy_from_buffer() below. */
+ kmb_ocs_aead_dma_cleanup(req);
+
+ /* Copy tag to destination sg after AAD and CT. */
+ sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag,
+ tag_size, req->assoclen + req->cryptlen);
+
+ /* Return directly as DMA cleanup already done. */
+ return 0;
+
+exit:
+ kmb_ocs_aead_dma_cleanup(req);
+
+ return rc;
+}
+
+static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine,
+ void *areq)
+{
+ struct skcipher_request *req =
+ container_of(areq, struct skcipher_request, base);
+ struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ int err;
+
+ if (!tctx->aes_dev) {
+ err = -ENODEV;
+ goto exit;
+ }
+
+ err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
+ tctx->cipher);
+ if (err)
+ goto exit;
+
+ err = kmb_ocs_sk_run(req);
+
+exit:
+ crypto_finalize_skcipher_request(engine, req, err);
+
+ return 0;
+}
+
+static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine,
+ void *areq)
+{
+ struct aead_request *req = container_of(areq,
+ struct aead_request, base);
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
+ int err;
+
+ if (!tctx->aes_dev)
+ return -ENODEV;
+
+ err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
+ tctx->cipher);
+ if (err)
+ goto exit;
+
+ err = kmb_ocs_aead_run(req);
+
+exit:
+ crypto_finalize_aead_request(tctx->aes_dev->engine, req, err);
+
+ return 0;
+}
+
+static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES);
+}
+
+static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES);
+}
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB);
+}
+
+static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB);
+}
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
+
+static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC);
+}
+
+static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC);
+}
+
+static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR);
+}
+
+static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR);
+}
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS);
+}
+
+static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS);
+}
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
+
+static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM);
+}
+
+static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM);
+}
+
+static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM);
+}
+
+static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM);
+}
+
+static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4);
+}
+
+static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
+ unsigned int key_len)
+{
+ return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4);
+}
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB);
+}
+
+static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB);
+}
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
+
+static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC);
+}
+
+static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC);
+}
+
+static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR);
+}
+
+static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR);
+}
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS);
+}
+
+static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req)
+{
+ return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS);
+}
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
+
+static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM);
+}
+
+static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM);
+}
+
+static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM);
+}
+
+static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req)
+{
+ return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM);
+}
+
+static inline int ocs_common_init(struct ocs_aes_tctx *tctx)
+{
+ tctx->engine_ctx.op.prepare_request = NULL;
+ tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request;
+ tctx->engine_ctx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static int ocs_aes_init_tfm(struct crypto_skcipher *tfm)
+{
+ const char *alg_name = crypto_tfm_alg_name(&tfm->base);
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+ struct crypto_sync_skcipher *blk;
+
+ /* set fallback cipher in case it will be needed */
+ blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(blk))
+ return PTR_ERR(blk);
+
+ tctx->sw_cipher.sk = blk;
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
+
+ return ocs_common_init(tctx);
+}
+
+static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm)
+{
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+
+ crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
+
+ return ocs_common_init(tctx);
+}
+
+static inline void clear_key(struct ocs_aes_tctx *tctx)
+{
+ memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256);
+
+ /* Zero key registers if set */
+ if (tctx->aes_dev)
+ ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256,
+ tctx->key, OCS_AES);
+}
+
+static void ocs_exit_tfm(struct crypto_skcipher *tfm)
+{
+ struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
+
+ clear_key(tctx);
+
+ if (tctx->sw_cipher.sk) {
+ crypto_free_sync_skcipher(tctx->sw_cipher.sk);
+ tctx->sw_cipher.sk = NULL;
+ }
+}
+
+static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx)
+{
+ tctx->engine_ctx.op.prepare_request = NULL;
+ tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request;
+ tctx->engine_ctx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static int ocs_aes_aead_cra_init(struct crypto_aead *tfm)
+{
+ const char *alg_name = crypto_tfm_alg_name(&tfm->base);
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
+ struct crypto_aead *blk;
+
+ /* Set fallback cipher in case it will be needed */
+ blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
+ if (IS_ERR(blk))
+ return PTR_ERR(blk);
+
+ tctx->sw_cipher.aead = blk;
+
+ crypto_aead_set_reqsize(tfm,
+ max(sizeof(struct ocs_aes_rctx),
+ (sizeof(struct aead_request) +
+ crypto_aead_reqsize(tctx->sw_cipher.aead))));
+
+ return ocs_common_aead_init(tctx);
+}
+
+static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ switch (authsize) {
+ case 4:
+ case 6:
+ case 8:
+ case 10:
+ case 12:
+ case 14:
+ case 16:
+ return 0;
+ default:
+ return -EINVAL;
+ }
+}
+
+static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm,
+ unsigned int authsize)
+{
+ return crypto_gcm_check_authsize(authsize);
+}
+
+static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
+
+ crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
+
+ return ocs_common_aead_init(tctx);
+}
+
+static void ocs_aead_cra_exit(struct crypto_aead *tfm)
+{
+ struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
+
+ clear_key(tctx);
+
+ if (tctx->sw_cipher.aead) {
+ crypto_free_aead(tctx->sw_cipher.aead);
+ tctx->sw_cipher.aead = NULL;
+ }
+}
+
+static struct skcipher_alg algs[] = {
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+ {
+ .base.cra_name = "ecb(aes)",
+ .base.cra_driver_name = "ecb-aes-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_AES_MIN_KEY_SIZE,
+ .max_keysize = OCS_AES_MAX_KEY_SIZE,
+ .setkey = kmb_ocs_aes_set_key,
+ .encrypt = kmb_ocs_aes_ecb_encrypt,
+ .decrypt = kmb_ocs_aes_ecb_decrypt,
+ .init = ocs_aes_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
+ {
+ .base.cra_name = "cbc(aes)",
+ .base.cra_driver_name = "cbc-aes-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_AES_MIN_KEY_SIZE,
+ .max_keysize = OCS_AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_aes_set_key,
+ .encrypt = kmb_ocs_aes_cbc_encrypt,
+ .decrypt = kmb_ocs_aes_cbc_decrypt,
+ .init = ocs_aes_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+ {
+ .base.cra_name = "ctr(aes)",
+ .base.cra_driver_name = "ctr-aes-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_AES_MIN_KEY_SIZE,
+ .max_keysize = OCS_AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_aes_set_key,
+ .encrypt = kmb_ocs_aes_ctr_encrypt,
+ .decrypt = kmb_ocs_aes_ctr_decrypt,
+ .init = ocs_aes_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+ {
+ .base.cra_name = "cts(cbc(aes))",
+ .base.cra_driver_name = "cts-aes-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_AES_MIN_KEY_SIZE,
+ .max_keysize = OCS_AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_aes_set_key,
+ .encrypt = kmb_ocs_aes_cts_encrypt,
+ .decrypt = kmb_ocs_aes_cts_decrypt,
+ .init = ocs_aes_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+ {
+ .base.cra_name = "ecb(sm4)",
+ .base.cra_driver_name = "ecb-sm4-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_SM4_KEY_SIZE,
+ .max_keysize = OCS_SM4_KEY_SIZE,
+ .setkey = kmb_ocs_sm4_set_key,
+ .encrypt = kmb_ocs_sm4_ecb_encrypt,
+ .decrypt = kmb_ocs_sm4_ecb_decrypt,
+ .init = ocs_sm4_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
+ {
+ .base.cra_name = "cbc(sm4)",
+ .base.cra_driver_name = "cbc-sm4-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_SM4_KEY_SIZE,
+ .max_keysize = OCS_SM4_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_sm4_set_key,
+ .encrypt = kmb_ocs_sm4_cbc_encrypt,
+ .decrypt = kmb_ocs_sm4_cbc_decrypt,
+ .init = ocs_sm4_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+ {
+ .base.cra_name = "ctr(sm4)",
+ .base.cra_driver_name = "ctr-sm4-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .base.cra_blocksize = 1,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_SM4_KEY_SIZE,
+ .max_keysize = OCS_SM4_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_sm4_set_key,
+ .encrypt = kmb_ocs_sm4_ctr_encrypt,
+ .decrypt = kmb_ocs_sm4_ctr_decrypt,
+ .init = ocs_sm4_init_tfm,
+ .exit = ocs_exit_tfm,
+ },
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+ {
+ .base.cra_name = "cts(cbc(sm4))",
+ .base.cra_driver_name = "cts-sm4-keembay-ocs",
+ .base.cra_priority = KMB_OCS_PRIORITY,
+ .base.cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .base.cra_blocksize = AES_BLOCK_SIZE,
+ .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .base.cra_module = THIS_MODULE,
+ .base.cra_alignmask = 0,
+
+ .min_keysize = OCS_SM4_KEY_SIZE,
+ .max_keysize = OCS_SM4_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = kmb_ocs_sm4_set_key,
+ .encrypt = kmb_ocs_sm4_cts_encrypt,
+ .decrypt = kmb_ocs_sm4_cts_decrypt,
+ .init = ocs_sm4_init_tfm,
+ .exit = ocs_exit_tfm,
+ }
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
+};
+
+static struct aead_alg algs_aead[] = {
+ {
+ .base = {
+ .cra_name = "gcm(aes)",
+ .cra_driver_name = "gcm-aes-keembay-ocs",
+ .cra_priority = KMB_OCS_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = ocs_aes_aead_cra_init,
+ .exit = ocs_aead_cra_exit,
+ .ivsize = GCM_AES_IV_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setauthsize = kmb_ocs_aead_gcm_setauthsize,
+ .setkey = kmb_ocs_aes_aead_set_key,
+ .encrypt = kmb_ocs_aes_gcm_encrypt,
+ .decrypt = kmb_ocs_aes_gcm_decrypt,
+ },
+ {
+ .base = {
+ .cra_name = "ccm(aes)",
+ .cra_driver_name = "ccm-aes-keembay-ocs",
+ .cra_priority = KMB_OCS_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY |
+ CRYPTO_ALG_NEED_FALLBACK,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = ocs_aes_aead_cra_init,
+ .exit = ocs_aead_cra_exit,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setauthsize = kmb_ocs_aead_ccm_setauthsize,
+ .setkey = kmb_ocs_aes_aead_set_key,
+ .encrypt = kmb_ocs_aes_ccm_encrypt,
+ .decrypt = kmb_ocs_aes_ccm_decrypt,
+ },
+ {
+ .base = {
+ .cra_name = "gcm(sm4)",
+ .cra_driver_name = "gcm-sm4-keembay-ocs",
+ .cra_priority = KMB_OCS_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = ocs_sm4_aead_cra_init,
+ .exit = ocs_aead_cra_exit,
+ .ivsize = GCM_AES_IV_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setauthsize = kmb_ocs_aead_gcm_setauthsize,
+ .setkey = kmb_ocs_sm4_aead_set_key,
+ .encrypt = kmb_ocs_sm4_gcm_encrypt,
+ .decrypt = kmb_ocs_sm4_gcm_decrypt,
+ },
+ {
+ .base = {
+ .cra_name = "ccm(sm4)",
+ .cra_driver_name = "ccm-sm4-keembay-ocs",
+ .cra_priority = KMB_OCS_PRIORITY,
+ .cra_flags = CRYPTO_ALG_ASYNC |
+ CRYPTO_ALG_KERN_DRIVER_ONLY,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct ocs_aes_tctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ },
+ .init = ocs_sm4_aead_cra_init,
+ .exit = ocs_aead_cra_exit,
+ .ivsize = AES_BLOCK_SIZE,
+ .maxauthsize = AES_BLOCK_SIZE,
+ .setauthsize = kmb_ocs_aead_ccm_setauthsize,
+ .setkey = kmb_ocs_sm4_aead_set_key,
+ .encrypt = kmb_ocs_sm4_ccm_encrypt,
+ .decrypt = kmb_ocs_sm4_ccm_decrypt,
+ }
+};
+
+static void unregister_aes_algs(struct ocs_aes_dev *aes_dev)
+{
+ crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead));
+ crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
+}
+
+static int register_aes_algs(struct ocs_aes_dev *aes_dev)
+{
+ int ret;
+
+ /*
+ * If any algorithm fails to register, all preceding algorithms that
+ * were successfully registered will be automatically unregistered.
+ */
+ ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead));
+ if (ret)
+ return ret;
+
+ ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs));
+ if (ret)
+ crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs));
+
+ return ret;
+}
+
+/* Device tree driver match. */
+static const struct of_device_id kmb_ocs_aes_of_match[] = {
+ {
+ .compatible = "intel,keembay-ocs-aes",
+ },
+ {}
+};
+
+static int kmb_ocs_aes_remove(struct platform_device *pdev)
+{
+ struct ocs_aes_dev *aes_dev;
+
+ aes_dev = platform_get_drvdata(pdev);
+
+ unregister_aes_algs(aes_dev);
+
+ spin_lock(&ocs_aes.lock);
+ list_del(&aes_dev->list);
+ spin_unlock(&ocs_aes.lock);
+
+ crypto_engine_exit(aes_dev->engine);
+
+ return 0;
+}
+
+static int kmb_ocs_aes_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct ocs_aes_dev *aes_dev;
+ int rc;
+
+ aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL);
+ if (!aes_dev)
+ return -ENOMEM;
+
+ aes_dev->dev = dev;
+
+ platform_set_drvdata(pdev, aes_dev);
+
+ rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+ if (rc) {
+ dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc);
+ return rc;
+ }
+
+ /* Get base register address. */
+ aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(aes_dev->base_reg))
+ return PTR_ERR(aes_dev->base_reg);
+
+ /* Get and request IRQ */
+ aes_dev->irq = platform_get_irq(pdev, 0);
+ if (aes_dev->irq < 0)
+ return aes_dev->irq;
+
+ rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler,
+ NULL, 0, "keembay-ocs-aes", aes_dev);
+ if (rc < 0) {
+ dev_err(dev, "Could not request IRQ\n");
+ return rc;
+ }
+
+ INIT_LIST_HEAD(&aes_dev->list);
+ spin_lock(&ocs_aes.lock);
+ list_add_tail(&aes_dev->list, &ocs_aes.dev_list);
+ spin_unlock(&ocs_aes.lock);
+
+ init_completion(&aes_dev->irq_completion);
+
+ /* Initialize crypto engine */
+ aes_dev->engine = crypto_engine_alloc_init(dev, true);
+ if (!aes_dev->engine) {
+ rc = -ENOMEM;
+ goto list_del;
+ }
+
+ rc = crypto_engine_start(aes_dev->engine);
+ if (rc) {
+ dev_err(dev, "Could not start crypto engine\n");
+ goto cleanup;
+ }
+
+ rc = register_aes_algs(aes_dev);
+ if (rc) {
+ dev_err(dev,
+ "Could not register OCS algorithms with Crypto API\n");
+ goto cleanup;
+ }
+
+ return 0;
+
+cleanup:
+ crypto_engine_exit(aes_dev->engine);
+list_del:
+ spin_lock(&ocs_aes.lock);
+ list_del(&aes_dev->list);
+ spin_unlock(&ocs_aes.lock);
+
+ return rc;
+}
+
+/* The OCS driver is a platform device. */
+static struct platform_driver kmb_ocs_aes_driver = {
+ .probe = kmb_ocs_aes_probe,
+ .remove = kmb_ocs_aes_remove,
+ .driver = {
+ .name = DRV_NAME,
+ .of_match_table = kmb_ocs_aes_of_match,
+ },
+};
+
+module_platform_driver(kmb_ocs_aes_driver);
+
+MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver");
+MODULE_LICENSE("GPL");
+
+MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs");
+MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs");
+
+MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs");
+MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs");
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
+MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs");
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
+
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
+MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs");
+MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs");
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Keem Bay OCS ECC Crypto Driver.
+ *
+ * Copyright (C) 2019-2021 Intel Corporation
+ */
+
+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+
+#include <linux/clk.h>
+#include <linux/completion.h>
+#include <linux/crypto.h>
+#include <linux/delay.h>
+#include <linux/fips.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#include <crypto/ecc_curve.h>
+#include <crypto/ecdh.h>
+#include <crypto/engine.h>
+#include <crypto/kpp.h>
+#include <crypto/rng.h>
+
+#include <crypto/internal/ecc.h>
+#include <crypto/internal/kpp.h>
+
+#define DRV_NAME "keembay-ocs-ecc"
+
+#define KMB_OCS_ECC_PRIORITY 350
+
+#define HW_OFFS_OCS_ECC_COMMAND 0x00000000
+#define HW_OFFS_OCS_ECC_STATUS 0x00000004
+#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080
+#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100
+#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180
+#define HW_OFFS_OCS_ECC_ISR 0x00000400
+#define HW_OFFS_OCS_ECC_IER 0x00000404
+
+#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0)
+#define HW_OCS_ECC_COMMAND_INS_BP BIT(0)
+
+#define HW_OCS_ECC_COMMAND_START_VAL BIT(0)
+
+#define OCS_ECC_OP_SIZE_384 BIT(8)
+#define OCS_ECC_OP_SIZE_256 0
+
+/* ECC Instruction : for ECC_COMMAND */
+#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP)
+#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP)
+
+#define ECC_ENABLE_INTR 1
+
+#define POLL_USEC 100
+#define TIMEOUT_USEC 10000
+
+#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS
+#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \
+ << ECC_DIGITS_TO_BYTES_SHIFT)
+
+#define POW_CUBE 3
+
+/**
+ * struct ocs_ecc_dev - ECC device context
+ * @list: List of device contexts
+ * @dev: OCS ECC device
+ * @base_reg: IO base address of OCS ECC
+ * @engine: Crypto engine for the device
+ * @irq_done: IRQ done completion.
+ * @irq: IRQ number
+ */
+struct ocs_ecc_dev {
+ struct list_head list;
+ struct device *dev;
+ void __iomem *base_reg;
+ struct crypto_engine *engine;
+ struct completion irq_done;
+ int irq;
+};
+
+/**
+ * struct ocs_ecc_ctx - Transformation context.
+ * @engine_ctx: Crypto engine ctx.
+ * @ecc_dev: The ECC driver associated with this context.
+ * @curve: The elliptic curve used by this transformation.
+ * @private_key: The private key.
+ */
+struct ocs_ecc_ctx {
+ struct crypto_engine_ctx engine_ctx;
+ struct ocs_ecc_dev *ecc_dev;
+ const struct ecc_curve *curve;
+ u64 private_key[KMB_ECC_VLI_MAX_DIGITS];
+};
+
+/* Driver data. */
+struct ocs_ecc_drv {
+ struct list_head dev_list;
+ spinlock_t lock; /* Protects dev_list. */
+};
+
+/* Global variable holding the list of OCS ECC devices (only one expected). */
+static struct ocs_ecc_drv ocs_ecc = {
+ .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list),
+ .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock),
+};
+
+/* Get OCS ECC tfm context from kpp_request. */
+static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req)
+{
+ return kpp_tfm_ctx(crypto_kpp_reqtfm(req));
+}
+
+/* Converts number of digits to number of bytes. */
+static inline unsigned int digits_to_bytes(unsigned int n)
+{
+ return n << ECC_DIGITS_TO_BYTES_SHIFT;
+}
+
+/*
+ * Wait for ECC idle i.e when an operation (other than write operations)
+ * is done.
+ */
+static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev)
+{
+ u32 value;
+
+ return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS),
+ value,
+ !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE),
+ POLL_USEC, TIMEOUT_USEC);
+}
+
+static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size)
+{
+ iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL,
+ ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
+}
+
+/* Direct write of u32 buffer to ECC engine with associated instruction. */
+static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev,
+ u32 op_size,
+ u32 inst,
+ const void *data_in,
+ size_t data_size)
+{
+ iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
+
+ /* MMIO Write src uint32 to dst. */
+ memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in,
+ data_size);
+}
+
+/* Start OCS ECC operation and wait for its completion. */
+static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size,
+ u32 inst)
+{
+ reinit_completion(&ecc_dev->irq_done);
+
+ iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER);
+ iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
+
+ return wait_for_completion_interruptible(&ecc_dev->irq_done);
+}
+
+/**
+ * ocs_ecc_read_cx_out() - Read the CX data output buffer.
+ * @dev: The OCS ECC device to read from.
+ * @cx_out: The buffer where to store the CX value. Must be at least
+ * @byte_count byte long.
+ * @byte_count: The amount of data to read.
+ */
+static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out,
+ size_t byte_count)
+{
+ memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT,
+ byte_count);
+}
+
+/**
+ * ocs_ecc_read_cy_out() - Read the CX data output buffer.
+ * @dev: The OCS ECC device to read from.
+ * @cy_out: The buffer where to store the CY value. Must be at least
+ * @byte_count byte long.
+ * @byte_count: The amount of data to read.
+ */
+static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out,
+ size_t byte_count)
+{
+ memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT,
+ byte_count);
+}
+
+static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx)
+{
+ if (tctx->ecc_dev)
+ return tctx->ecc_dev;
+
+ spin_lock(&ocs_ecc.lock);
+
+ /* Only a single OCS device available. */
+ tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev,
+ list);
+
+ spin_unlock(&ocs_ecc.lock);
+
+ return tctx->ecc_dev;
+}
+
+/* Do point multiplication using OCS ECC HW. */
+static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev,
+ struct ecc_point *result,
+ const struct ecc_point *point,
+ u64 *scalar,
+ const struct ecc_curve *curve)
+{
+ u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */
+ u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ?
+ OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;
+ size_t nbytes = digits_to_bytes(curve->g.ndigits);
+ int rc = 0;
+
+ /* Generate random nbytes for Simple and Differential SCA protection. */
+ rc = crypto_get_default_rng();
+ if (rc)
+ return rc;
+
+ rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes);
+ crypto_put_default_rng();
+ if (rc)
+ return rc;
+
+ /* Wait engine to be idle before starting new operation. */
+ rc = ocs_ecc_wait_idle(ecc_dev);
+ if (rc)
+ return rc;
+
+ /* Send ecc_start pulse as well as indicating operation size. */
+ ocs_ecc_cmd_start(ecc_dev, op_size);
+
+ /* Write ax param; Base point (Gx). */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,
+ point->x, nbytes);
+
+ /* Write ay param; Base point (Gy). */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,
+ point->y, nbytes);
+
+ /*
+ * Write the private key into DATA_IN reg.
+ *
+ * Since DATA_IN register is used to write different values during the
+ * computation private Key value is overwritten with
+ * side-channel-resistance value.
+ */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D,
+ scalar, nbytes);
+
+ /* Write operand by/l. */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L,
+ sca, nbytes);
+ memzero_explicit(sca, sizeof(sca));
+
+ /* Write p = curve prime(GF modulus). */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,
+ curve->p, nbytes);
+
+ /* Write a = curve coefficient. */
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A,
+ curve->a, nbytes);
+
+ /* Make hardware perform the multiplication. */
+ rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A);
+ if (rc)
+ return rc;
+
+ /* Read result. */
+ ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes);
+ ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes);
+
+ return 0;
+}
+
+/**
+ * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW.
+ * @ecc_dev: The OCS ECC device to use.
+ * @scalar_out: Where to store the output scalar.
+ * @scalar_a: Input scalar operand 'a'.
+ * @scalar_b: Input scalar operand 'b'
+ * @curve: The curve on which the operation is performed.
+ * @ndigits: The size of the operands (in digits).
+ * @inst: The operation to perform (as an OCS ECC instruction).
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out,
+ const u64 *scalar_a, const u64 *scalar_b,
+ const struct ecc_curve *curve,
+ unsigned int ndigits, const u32 inst)
+{
+ u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ?
+ OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;
+ size_t nbytes = digits_to_bytes(ndigits);
+ int rc;
+
+ /* Wait engine to be idle before starting new operation. */
+ rc = ocs_ecc_wait_idle(ecc_dev);
+ if (rc)
+ return rc;
+
+ /* Send ecc_start pulse as well as indicating operation size. */
+ ocs_ecc_cmd_start(ecc_dev, op_size);
+
+ /* Write ax param (Base point (Gx).*/
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,
+ scalar_a, nbytes);
+
+ /* Write ay param Base point (Gy).*/
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,
+ scalar_b, nbytes);
+
+ /* Write p = curve prime(GF modulus).*/
+ ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,
+ curve->p, nbytes);
+
+ /* Give instruction A.B or A+B to ECC engine. */
+ rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst);
+ if (rc)
+ return rc;
+
+ ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes);
+
+ if (vli_is_zero(scalar_out, ndigits))
+ return -EINVAL;
+
+ return 0;
+}
+
+/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */
+static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev,
+ const struct ecc_curve *curve,
+ struct ecc_point *pk)
+{
+ u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
+ u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
+ u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
+ int rc;
+
+ if (WARN_ON(pk->ndigits != curve->g.ndigits))
+ return -EINVAL;
+
+ /* Check 1: Verify key is not the zero point. */
+ if (ecc_point_is_zero(pk))
+ return -EINVAL;
+
+ /* Check 2: Verify key is in the range [0, p-1]. */
+ if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1)
+ return -EINVAL;
+
+ if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1)
+ return -EINVAL;
+
+ /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */
+
+ /* y^2 */
+ /* Compute y^2 -> store in yy */
+ rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits,
+ OCS_ECC_INST_CALC_A_MUL_B_MODP);
+ if (rc)
+ goto exit;
+
+ /* x^3 */
+ /* Assigning w = 3, used for calculating x^3. */
+ w[0] = POW_CUBE;
+ /* Load the next stage.*/
+ rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits,
+ OCS_ECC_INST_CALC_A_POW_B_MODP);
+ if (rc)
+ goto exit;
+
+ /* Do a*x -> store in w. */
+ rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve,
+ pk->ndigits,
+ OCS_ECC_INST_CALC_A_MUL_B_MODP);
+ if (rc)
+ goto exit;
+
+ /* Do ax + b == w + b; store in w. */
+ rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve,
+ pk->ndigits,
+ OCS_ECC_INST_CALC_A_ADD_B_MODP);
+ if (rc)
+ goto exit;
+
+ /* x^3 + ax + b == x^3 + w -> store in w. */
+ rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits,
+ OCS_ECC_INST_CALC_A_ADD_B_MODP);
+ if (rc)
+ goto exit;
+
+ /* Compare y^2 == x^3 + a·x + b. */
+ rc = vli_cmp(yy, w, pk->ndigits);
+ if (rc)
+ rc = -EINVAL;
+
+exit:
+ memzero_explicit(xxx, sizeof(xxx));
+ memzero_explicit(yy, sizeof(yy));
+ memzero_explicit(w, sizeof(w));
+
+ return rc;
+}
+
+/* SP800-56A section 5.6.2.3.3 full verification */
+static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev,
+ const struct ecc_curve *curve,
+ struct ecc_point *pk)
+{
+ struct ecc_point *nQ;
+ int rc;
+
+ /* Checks 1 through 3 */
+ rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);
+ if (rc)
+ return rc;
+
+ /* Check 4: Verify that nQ is the zero point. */
+ nQ = ecc_alloc_point(pk->ndigits);
+ if (!nQ)
+ return -ENOMEM;
+
+ rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve);
+ if (rc)
+ goto exit;
+
+ if (!ecc_point_is_zero(nQ))
+ rc = -EINVAL;
+
+exit:
+ ecc_free_point(nQ);
+
+ return rc;
+}
+
+static int kmb_ecc_is_key_valid(const struct ecc_curve *curve,
+ const u64 *private_key, size_t private_key_len)
+{
+ size_t ndigits = curve->g.ndigits;
+ u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1};
+ u64 res[KMB_ECC_VLI_MAX_DIGITS];
+
+ if (private_key_len != digits_to_bytes(ndigits))
+ return -EINVAL;
+
+ if (!private_key)
+ return -EINVAL;
+
+ /* Make sure the private key is in the range [2, n-3]. */
+ if (vli_cmp(one, private_key, ndigits) != -1)
+ return -EINVAL;
+
+ vli_sub(res, curve->n, one, ndigits);
+ vli_sub(res, res, one, ndigits);
+ if (vli_cmp(res, private_key, ndigits) != 1)
+ return -EINVAL;
+
+ return 0;
+}
+
+/*
+ * ECC private keys are generated using the method of extra random bits,
+ * equivalent to that described in FIPS 186-4, Appendix B.4.1.
+ *
+ * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer
+ * than requested
+ * 0 <= c mod(n-1) <= n-2 and implies that
+ * 1 <= d <= n-1
+ *
+ * This method generates a private key uniformly distributed in the range
+ * [1, n-1].
+ */
+static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey)
+{
+ size_t nbytes = digits_to_bytes(curve->g.ndigits);
+ u64 priv[KMB_ECC_VLI_MAX_DIGITS];
+ size_t nbits;
+ int rc;
+
+ nbits = vli_num_bits(curve->n, curve->g.ndigits);
+
+ /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */
+ if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv))
+ return -EINVAL;
+
+ /*
+ * FIPS 186-4 recommends that the private key should be obtained from a
+ * RBG with a security strength equal to or greater than the security
+ * strength associated with N.
+ *
+ * The maximum security strength identified by NIST SP800-57pt1r4 for
+ * ECC is 256 (N >= 512).
+ *
+ * This condition is met by the default RNG because it selects a favored
+ * DRBG with a security strength of 256.
+ */
+ if (crypto_get_default_rng())
+ return -EFAULT;
+
+ rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes);
+ crypto_put_default_rng();
+ if (rc)
+ goto cleanup;
+
+ rc = kmb_ecc_is_key_valid(curve, priv, nbytes);
+ if (rc)
+ goto cleanup;
+
+ ecc_swap_digits(priv, privkey, curve->g.ndigits);
+
+cleanup:
+ memzero_explicit(&priv, sizeof(priv));
+
+ return rc;
+}
+
+static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
+ unsigned int len)
+{
+ struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
+ struct ecdh params;
+ int rc = 0;
+
+ rc = crypto_ecdh_decode_key(buf, len, ¶ms);
+ if (rc)
+ goto cleanup;
+
+ /* Ensure key size is not bigger then expected. */
+ if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) {
+ rc = -EINVAL;
+ goto cleanup;
+ }
+
+ /* Auto-generate private key is not provided. */
+ if (!params.key || !params.key_size) {
+ rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key);
+ goto cleanup;
+ }
+
+ rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key,
+ params.key_size);
+ if (rc)
+ goto cleanup;
+
+ ecc_swap_digits((const u64 *)params.key, tctx->private_key,
+ tctx->curve->g.ndigits);
+cleanup:
+ memzero_explicit(¶ms, sizeof(params));
+
+ if (rc)
+ tctx->curve = NULL;
+
+ return rc;
+}
+
+/* Compute shared secret. */
+static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx,
+ struct kpp_request *req)
+{
+ struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;
+ const struct ecc_curve *curve = tctx->curve;
+ u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS];
+ u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];
+ size_t copied, nbytes, pubk_len;
+ struct ecc_point *pk, *result;
+ int rc;
+
+ nbytes = digits_to_bytes(curve->g.ndigits);
+
+ /* Public key is a point, thus it has two coordinates */
+ pubk_len = 2 * nbytes;
+
+ /* Copy public key from SG list to pubk_buf. */
+ copied = sg_copy_to_buffer(req->src,
+ sg_nents_for_len(req->src, pubk_len),
+ pubk_buf, pubk_len);
+ if (copied != pubk_len)
+ return -EINVAL;
+
+ /* Allocate and initialize public key point. */
+ pk = ecc_alloc_point(curve->g.ndigits);
+ if (!pk)
+ return -ENOMEM;
+
+ ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits);
+ ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits);
+
+ /*
+ * Check the public key for following
+ * Check 1: Verify key is not the zero point.
+ * Check 2: Verify key is in the range [1, p-1].
+ * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p
+ */
+ rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);
+ if (rc)
+ goto exit_free_pk;
+
+ /* Allocate point for storing computed shared secret. */
+ result = ecc_alloc_point(pk->ndigits);
+ if (!result) {
+ rc = -ENOMEM;
+ goto exit_free_pk;
+ }
+
+ /* Calculate the shared secret.*/
+ rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve);
+ if (rc)
+ goto exit_free_result;
+
+ if (ecc_point_is_zero(result)) {
+ rc = -EFAULT;
+ goto exit_free_result;
+ }
+
+ /* Copy shared secret from point to buffer. */
+ ecc_swap_digits(result->x, shared_secret, result->ndigits);
+
+ /* Request might ask for less bytes than what we have. */
+ nbytes = min_t(size_t, nbytes, req->dst_len);
+
+ copied = sg_copy_from_buffer(req->dst,
+ sg_nents_for_len(req->dst, nbytes),
+ shared_secret, nbytes);
+
+ if (copied != nbytes)
+ rc = -EINVAL;
+
+ memzero_explicit(shared_secret, sizeof(shared_secret));
+
+exit_free_result:
+ ecc_free_point(result);
+
+exit_free_pk:
+ ecc_free_point(pk);
+
+ return rc;
+}
+
+/* Compute public key. */
+static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx,
+ struct kpp_request *req)
+{
+ const struct ecc_curve *curve = tctx->curve;
+ u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];
+ struct ecc_point *pk;
+ size_t pubk_len;
+ size_t copied;
+ int rc;
+
+ /* Public key is a point, so it has double the digits. */
+ pubk_len = 2 * digits_to_bytes(curve->g.ndigits);
+
+ pk = ecc_alloc_point(curve->g.ndigits);
+ if (!pk)
+ return -ENOMEM;
+
+ /* Public Key(pk) = priv * G. */
+ rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key,
+ curve);
+ if (rc)
+ goto exit;
+
+ /* SP800-56A rev 3 5.6.2.1.3 key check */
+ if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) {
+ rc = -EAGAIN;
+ goto exit;
+ }
+
+ /* Copy public key from point to buffer. */
+ ecc_swap_digits(pk->x, pubk_buf, pk->ndigits);
+ ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits);
+
+ /* Copy public key to req->dst. */
+ copied = sg_copy_from_buffer(req->dst,
+ sg_nents_for_len(req->dst, pubk_len),
+ pubk_buf, pubk_len);
+
+ if (copied != pubk_len)
+ rc = -EINVAL;
+
+exit:
+ ecc_free_point(pk);
+
+ return rc;
+}
+
+static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine,
+ void *areq)
+{
+ struct kpp_request *req = container_of(areq, struct kpp_request, base);
+ struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
+ struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;
+ int rc;
+
+ if (req->src)
+ rc = kmb_ecc_do_shared_secret(tctx, req);
+ else
+ rc = kmb_ecc_do_public_key(tctx, req);
+
+ crypto_finalize_kpp_request(ecc_dev->engine, req, rc);
+
+ return 0;
+}
+
+static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req)
+{
+ struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
+ const struct ecc_curve *curve = tctx->curve;
+
+ /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */
+ if (!tctx->curve)
+ return -EINVAL;
+
+ /* Ensure dst is present. */
+ if (!req->dst)
+ return -EINVAL;
+
+ /* Check the request dst is big enough to hold the public key. */
+ if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits)))
+ return -EINVAL;
+
+ /* 'src' is not supposed to be present when generate pubk is called. */
+ if (req->src)
+ return -EINVAL;
+
+ return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,
+ req);
+}
+
+static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req)
+{
+ struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
+ const struct ecc_curve *curve = tctx->curve;
+
+ /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */
+ if (!tctx->curve)
+ return -EINVAL;
+
+ /* Ensure dst is present. */
+ if (!req->dst)
+ return -EINVAL;
+
+ /* Ensure src is present. */
+ if (!req->src)
+ return -EINVAL;
+
+ /*
+ * req->src is expected to the (other-side) public key, so its length
+ * must be 2 * coordinate size (in bytes).
+ */
+ if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits))
+ return -EINVAL;
+
+ return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,
+ req);
+}
+
+static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id)
+{
+ memset(tctx, 0, sizeof(*tctx));
+
+ tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx);
+
+ if (IS_ERR(tctx->ecc_dev)) {
+ pr_err("Failed to find the device : %ld\n",
+ PTR_ERR(tctx->ecc_dev));
+ return PTR_ERR(tctx->ecc_dev);
+ }
+
+ tctx->curve = ecc_get_curve(curve_id);
+ if (!tctx->curve)
+ return -EOPNOTSUPP;
+
+ tctx->engine_ctx.op.prepare_request = NULL;
+ tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request;
+ tctx->engine_ctx.op.unprepare_request = NULL;
+
+ return 0;
+}
+
+static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
+{
+ struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
+
+ return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256);
+}
+
+static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
+{
+ struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
+
+ return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384);
+}
+
+static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm)
+{
+ struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
+
+ memzero_explicit(tctx->private_key, sizeof(*tctx->private_key));
+}
+
+static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm)
+{
+ struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
+
+ /* Public key is made of two coordinates, so double the digits. */
+ return digits_to_bytes(tctx->curve->g.ndigits) * 2;
+}
+
+static struct kpp_alg ocs_ecdh_p256 = {
+ .set_secret = kmb_ocs_ecdh_set_secret,
+ .generate_public_key = kmb_ocs_ecdh_generate_public_key,
+ .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,
+ .init = kmb_ocs_ecdh_nist_p256_init_tfm,
+ .exit = kmb_ocs_ecdh_exit_tfm,
+ .max_size = kmb_ocs_ecdh_max_size,
+ .base = {
+ .cra_name = "ecdh-nist-p256",
+ .cra_driver_name = "ecdh-nist-p256-keembay-ocs",
+ .cra_priority = KMB_OCS_ECC_PRIORITY,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct ocs_ecc_ctx),
+ },
+};
+
+static struct kpp_alg ocs_ecdh_p384 = {
+ .set_secret = kmb_ocs_ecdh_set_secret,
+ .generate_public_key = kmb_ocs_ecdh_generate_public_key,
+ .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,
+ .init = kmb_ocs_ecdh_nist_p384_init_tfm,
+ .exit = kmb_ocs_ecdh_exit_tfm,
+ .max_size = kmb_ocs_ecdh_max_size,
+ .base = {
+ .cra_name = "ecdh-nist-p384",
+ .cra_driver_name = "ecdh-nist-p384-keembay-ocs",
+ .cra_priority = KMB_OCS_ECC_PRIORITY,
+ .cra_module = THIS_MODULE,
+ .cra_ctxsize = sizeof(struct ocs_ecc_ctx),
+ },
+};
+
+static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id)
+{
+ struct ocs_ecc_dev *ecc_dev = dev_id;
+ u32 status;
+
+ /*
+ * Read the status register and write it back to clear the
+ * DONE_INT_STATUS bit.
+ */
+ status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);
+ iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);
+
+ if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE))
+ return IRQ_NONE;
+
+ complete(&ecc_dev->irq_done);
+
+ return IRQ_HANDLED;
+}
+
+static int kmb_ocs_ecc_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct ocs_ecc_dev *ecc_dev;
+ int rc;
+
+ ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL);
+ if (!ecc_dev)
+ return -ENOMEM;
+
+ ecc_dev->dev = dev;
+
+ platform_set_drvdata(pdev, ecc_dev);
+
+ INIT_LIST_HEAD(&ecc_dev->list);
+ init_completion(&ecc_dev->irq_done);
+
+ /* Get base register address. */
+ ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);
+ if (IS_ERR(ecc_dev->base_reg)) {
+ dev_err(dev, "Failed to get base address\n");
+ rc = PTR_ERR(ecc_dev->base_reg);
+ goto list_del;
+ }
+
+ /* Get and request IRQ */
+ ecc_dev->irq = platform_get_irq(pdev, 0);
+ if (ecc_dev->irq < 0) {
+ rc = ecc_dev->irq;
+ goto list_del;
+ }
+
+ rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler,
+ NULL, 0, "keembay-ocs-ecc", ecc_dev);
+ if (rc < 0) {
+ dev_err(dev, "Could not request IRQ\n");
+ goto list_del;
+ }
+
+ /* Add device to the list of OCS ECC devices. */
+ spin_lock(&ocs_ecc.lock);
+ list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list);
+ spin_unlock(&ocs_ecc.lock);
+
+ /* Initialize crypto engine. */
+ ecc_dev->engine = crypto_engine_alloc_init(dev, 1);
+ if (!ecc_dev->engine) {
+ dev_err(dev, "Could not allocate crypto engine\n");
+ rc = -ENOMEM;
+ goto list_del;
+ }
+
+ rc = crypto_engine_start(ecc_dev->engine);
+ if (rc) {
+ dev_err(dev, "Could not start crypto engine\n");
+ goto cleanup;
+ }
+
+ /* Register the KPP algo. */
+ rc = crypto_register_kpp(&ocs_ecdh_p256);
+ if (rc) {
+ dev_err(dev,
+ "Could not register OCS algorithms with Crypto API\n");
+ goto cleanup;
+ }
+
+ rc = crypto_register_kpp(&ocs_ecdh_p384);
+ if (rc) {
+ dev_err(dev,
+ "Could not register OCS algorithms with Crypto API\n");
+ goto ocs_ecdh_p384_error;
+ }
+
+ return 0;
+
+ocs_ecdh_p384_error:
+ crypto_unregister_kpp(&ocs_ecdh_p256);
+
+cleanup:
+ crypto_engine_exit(ecc_dev->engine);
+
+list_del:
+ spin_lock(&ocs_ecc.lock);
+ list_del(&ecc_dev->list);
+ spin_unlock(&ocs_ecc.lock);
+
+ return rc;
+}
+
+static int kmb_ocs_ecc_remove(struct platform_device *pdev)
+{
+ struct ocs_ecc_dev *ecc_dev;
+
+ ecc_dev = platform_get_drvdata(pdev);
+
+ crypto_unregister_kpp(&ocs_ecdh_p384);
+ crypto_unregister_kpp(&ocs_ecdh_p256);
+
+ spin_lock(&ocs_ecc.lock);
+ list_del(&ecc_dev->list);
+ spin_unlock(&ocs_ecc.lock);
+
+ crypto_engine_exit(ecc_dev->engine);
+
+ return 0;
+}
+
+/* Device tree driver match. */
+static const struct of_device_id kmb_ocs_ecc_of_match[] = {
+ {
+ .compatible = "intel,keembay-ocs-ecc",
+ },
+ {}
+};
+
+/* The OCS driver is a platform device. */
+static struct platform_driver kmb_ocs_ecc_driver = {
+ .probe = kmb_ocs_ecc_probe,
+ .remove = kmb_ocs_ecc_remove,
+ .driver = {
+ .name = DRV_NAME,
+ .of_match_table = kmb_ocs_ecc_of_match,
+ },
+};
+module_platform_driver(kmb_ocs_ecc_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver");
+MODULE_ALIAS_CRYPTO("ecdh-nist-p256");
+MODULE_ALIAS_CRYPTO("ecdh-nist-p384");
+MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs");
+MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Keem Bay OCS HCU Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+
+#include <crypto/engine.h>
+#include <crypto/scatterwalk.h>
+#include <crypto/sha2.h>
+#include <crypto/sm3.h>
+#include <crypto/hmac.h>
+#include <crypto/internal/hash.h>
+
+#include "ocs-hcu.h"
+
+#define DRV_NAME "keembay-ocs-hcu"
+
+/* Flag marking a final request. */
+#define REQ_FINAL BIT(0)
+/* Flag marking a HMAC request. */
+#define REQ_FLAGS_HMAC BIT(1)
+/* Flag set when HW HMAC is being used. */
+#define REQ_FLAGS_HMAC_HW BIT(2)
+/* Flag set when SW HMAC is being used. */
+#define REQ_FLAGS_HMAC_SW BIT(3)
+
+/**
+ * struct ocs_hcu_ctx: OCS HCU Transform context.
+ * @engine_ctx: Crypto Engine context.
+ * @hcu_dev: The OCS HCU device used by the transformation.
+ * @key: The key (used only for HMAC transformations).
+ * @key_len: The length of the key.
+ * @is_sm3_tfm: Whether or not this is an SM3 transformation.
+ * @is_hmac_tfm: Whether or not this is a HMAC transformation.
+ */
+struct ocs_hcu_ctx {
+ struct crypto_engine_ctx engine_ctx;
+ struct ocs_hcu_dev *hcu_dev;
+ u8 key[SHA512_BLOCK_SIZE];
+ size_t key_len;
+ bool is_sm3_tfm;
+ bool is_hmac_tfm;
+};
+
+/**
+ * struct ocs_hcu_rctx - Context for the request.
+ * @hcu_dev: OCS HCU device to be used to service the request.
+ * @flags: Flags tracking request status.
+ * @algo: Algorithm to use for the request.
+ * @blk_sz: Block size of the transformation / request.
+ * @dig_sz: Digest size of the transformation / request.
+ * @dma_list: OCS DMA linked list.
+ * @hash_ctx: OCS HCU hashing context.
+ * @buffer: Buffer to store: partial block of data and SW HMAC
+ * artifacts (ipad, opad, etc.).
+ * @buf_cnt: Number of bytes currently stored in the buffer.
+ * @buf_dma_addr: The DMA address of @buffer (when mapped).
+ * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped.
+ * @sg: Head of the scatterlist entries containing data.
+ * @sg_data_total: Total data in the SG list at any time.
+ * @sg_data_offset: Offset into the data of the current individual SG node.
+ * @sg_dma_nents: Number of sg entries mapped in dma_list.
+ */
+struct ocs_hcu_rctx {
+ struct ocs_hcu_dev *hcu_dev;
+ u32 flags;
+ enum ocs_hcu_algo algo;
+ size_t blk_sz;
+ size_t dig_sz;
+ struct ocs_hcu_dma_list *dma_list;
+ struct ocs_hcu_hash_ctx hash_ctx;
+ /*
+ * Buffer is double the block size because we need space for SW HMAC
+ * artifacts, i.e:
+ * - ipad (1 block) + a possible partial block of data.
+ * - opad (1 block) + digest of H(k ^ ipad || m)
+ */
+ u8 buffer[2 * SHA512_BLOCK_SIZE];
+ size_t buf_cnt;
+ dma_addr_t buf_dma_addr;
+ size_t buf_dma_count;
+ struct scatterlist *sg;
+ unsigned int sg_data_total;
+ unsigned int sg_data_offset;
+ unsigned int sg_dma_nents;
+};
+
+/**
+ * struct ocs_hcu_drv - Driver data
+ * @dev_list: The list of HCU devices.
+ * @lock: The lock protecting dev_list.
+ */
+struct ocs_hcu_drv {
+ struct list_head dev_list;
+ spinlock_t lock; /* Protects dev_list. */
+};
+
+static struct ocs_hcu_drv ocs_hcu = {
+ .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list),
+ .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock),
+};
+
+/*
+ * Return the total amount of data in the request; that is: the data in the
+ * request buffer + the data in the sg list.
+ */
+static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx)
+{
+ return rctx->sg_data_total + rctx->buf_cnt;
+}
+
+/* Move remaining content of scatter-gather list to context buffer. */
+static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx)
+{
+ size_t count;
+
+ if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) {
+ WARN(1, "%s: sg data does not fit in buffer\n", __func__);
+ return -EINVAL;
+ }
+
+ while (rctx->sg_data_total) {
+ if (!rctx->sg) {
+ WARN(1, "%s: unexpected NULL sg\n", __func__);
+ return -EINVAL;
+ }
+ /*
+ * If current sg has been fully processed, skip to the next
+ * one.
+ */
+ if (rctx->sg_data_offset == rctx->sg->length) {
+ rctx->sg = sg_next(rctx->sg);
+ rctx->sg_data_offset = 0;
+ continue;
+ }
+ /*
+ * Determine the maximum data available to copy from the node.
+ * Minimum of the length left in the sg node, or the total data
+ * in the request.
+ */
+ count = min(rctx->sg->length - rctx->sg_data_offset,
+ rctx->sg_data_total);
+ /* Copy from scatter-list entry to context buffer. */
+ scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt],
+ rctx->sg, rctx->sg_data_offset,
+ count, 0);
+
+ rctx->sg_data_offset += count;
+ rctx->sg_data_total -= count;
+ rctx->buf_cnt += count;
+ }
+
+ return 0;
+}
+
+static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
+
+ /* If the HCU device for the request was previously set, return it. */
+ if (tctx->hcu_dev)
+ return tctx->hcu_dev;
+
+ /*
+ * Otherwise, get the first HCU device available (there should be one
+ * and only one device).
+ */
+ spin_lock_bh(&ocs_hcu.lock);
+ tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list,
+ struct ocs_hcu_dev,
+ list);
+ spin_unlock_bh(&ocs_hcu.lock);
+
+ return tctx->hcu_dev;
+}
+
+/* Free OCS DMA linked list and DMA-able context buffer. */
+static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req,
+ struct ocs_hcu_rctx *rctx)
+{
+ struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev;
+ struct device *dev = hcu_dev->dev;
+
+ /* Unmap rctx->buffer (if mapped). */
+ if (rctx->buf_dma_count) {
+ dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count,
+ DMA_TO_DEVICE);
+ rctx->buf_dma_count = 0;
+ }
+
+ /* Unmap req->src (if mapped). */
+ if (rctx->sg_dma_nents) {
+ dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE);
+ rctx->sg_dma_nents = 0;
+ }
+
+ /* Free dma_list (if allocated). */
+ if (rctx->dma_list) {
+ ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list);
+ rctx->dma_list = NULL;
+ }
+}
+
+/*
+ * Prepare for DMA operation:
+ * - DMA-map request context buffer (if needed)
+ * - DMA-map SG list (only the entries to be processed, see note below)
+ * - Allocate OCS HCU DMA linked list (number of elements = SG entries to
+ * process + context buffer (if not empty)).
+ * - Add DMA-mapped request context buffer to OCS HCU DMA list.
+ * - Add SG entries to DMA list.
+ *
+ * Note: if this is a final request, we process all the data in the SG list,
+ * otherwise we can only process up to the maximum amount of block-aligned data
+ * (the remainder will be put into the context buffer and processed in the next
+ * request).
+ */
+static int kmb_ocs_dma_prepare(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ struct device *dev = rctx->hcu_dev->dev;
+ unsigned int remainder = 0;
+ unsigned int total;
+ size_t nents;
+ size_t count;
+ int rc;
+ int i;
+
+ /* This function should be called only when there is data to process. */
+ total = kmb_get_total_data(rctx);
+ if (!total)
+ return -EINVAL;
+
+ /*
+ * If this is not a final DMA (terminated DMA), the data passed to the
+ * HCU must be aligned to the block size; compute the remainder data to
+ * be processed in the next request.
+ */
+ if (!(rctx->flags & REQ_FINAL))
+ remainder = total % rctx->blk_sz;
+
+ /* Determine the number of scatter gather list entries to process. */
+ nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder);
+
+ /* If there are entries to process, map them. */
+ if (nents) {
+ rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents,
+ DMA_TO_DEVICE);
+ if (!rctx->sg_dma_nents) {
+ dev_err(dev, "Failed to MAP SG\n");
+ rc = -ENOMEM;
+ goto cleanup;
+ }
+ /*
+ * The value returned by dma_map_sg() can be < nents; so update
+ * nents accordingly.
+ */
+ nents = rctx->sg_dma_nents;
+ }
+
+ /*
+ * If context buffer is not empty, map it and add extra DMA entry for
+ * it.
+ */
+ if (rctx->buf_cnt) {
+ rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer,
+ rctx->buf_cnt,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, rctx->buf_dma_addr)) {
+ dev_err(dev, "Failed to map request context buffer\n");
+ rc = -ENOMEM;
+ goto cleanup;
+ }
+ rctx->buf_dma_count = rctx->buf_cnt;
+ /* Increase number of dma entries. */
+ nents++;
+ }
+
+ /* Allocate OCS HCU DMA list. */
+ rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents);
+ if (!rctx->dma_list) {
+ rc = -ENOMEM;
+ goto cleanup;
+ }
+
+ /* Add request context buffer (if previously DMA-mapped) */
+ if (rctx->buf_dma_count) {
+ rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list,
+ rctx->buf_dma_addr,
+ rctx->buf_dma_count);
+ if (rc)
+ goto cleanup;
+ }
+
+ /* Add the SG nodes to be processed to the DMA linked list. */
+ for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) {
+ /*
+ * The number of bytes to add to the list entry is the minimum
+ * between:
+ * - The DMA length of the SG entry.
+ * - The data left to be processed.
+ */
+ count = min(rctx->sg_data_total - remainder,
+ sg_dma_len(rctx->sg) - rctx->sg_data_offset);
+ /*
+ * Do not create a zero length DMA descriptor. Check in case of
+ * zero length SG node.
+ */
+ if (count == 0)
+ continue;
+ /* Add sg to HCU DMA list. */
+ rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev,
+ rctx->dma_list,
+ rctx->sg->dma_address,
+ count);
+ if (rc)
+ goto cleanup;
+
+ /* Update amount of data remaining in SG list. */
+ rctx->sg_data_total -= count;
+
+ /*
+ * If remaining data is equal to remainder (note: 'less than'
+ * case should never happen in practice), we are done: update
+ * offset and exit the loop.
+ */
+ if (rctx->sg_data_total <= remainder) {
+ WARN_ON(rctx->sg_data_total < remainder);
+ rctx->sg_data_offset += count;
+ break;
+ }
+
+ /*
+ * If we get here is because we need to process the next sg in
+ * the list; set offset within the sg to 0.
+ */
+ rctx->sg_data_offset = 0;
+ }
+
+ return 0;
+cleanup:
+ dev_err(dev, "Failed to prepare DMA.\n");
+ kmb_ocs_hcu_dma_cleanup(req, rctx);
+
+ return rc;
+}
+
+static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+
+ /* Clear buffer of any data. */
+ memzero_explicit(rctx->buffer, sizeof(rctx->buffer));
+}
+
+static int kmb_ocs_hcu_handle_queue(struct ahash_request *req)
+{
+ struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
+
+ if (!hcu_dev)
+ return -ENOENT;
+
+ return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req);
+}
+
+static int prepare_ipad(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
+ int i;
+
+ WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__);
+ WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW),
+ "%s: HMAC_SW flag is not set\n", __func__);
+ /*
+ * Key length must be equal to block size. If key is shorter,
+ * we pad it with zero (note: key cannot be longer, since
+ * longer keys are hashed by kmb_ocs_hcu_setkey()).
+ */
+ if (ctx->key_len > rctx->blk_sz) {
+ WARN(1, "%s: Invalid key length in tfm context\n", __func__);
+ return -EINVAL;
+ }
+ memzero_explicit(&ctx->key[ctx->key_len],
+ rctx->blk_sz - ctx->key_len);
+ ctx->key_len = rctx->blk_sz;
+ /*
+ * Prepare IPAD for HMAC. Only done for first block.
+ * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m))
+ * k ^ ipad will be first hashed block.
+ * k ^ opad will be calculated in the final request.
+ * Only needed if not using HW HMAC.
+ */
+ for (i = 0; i < rctx->blk_sz; i++)
+ rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE;
+ rctx->buf_cnt = rctx->blk_sz;
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)
+{
+ struct ahash_request *req = container_of(areq, struct ahash_request,
+ base);
+ struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
+ int rc;
+ int i;
+
+ if (!hcu_dev) {
+ rc = -ENOENT;
+ goto error;
+ }
+
+ /*
+ * If hardware HMAC flag is set, perform HMAC in hardware.
+ *
+ * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx)
+ */
+ if (rctx->flags & REQ_FLAGS_HMAC_HW) {
+ /* Map input data into the HCU DMA linked list. */
+ rc = kmb_ocs_dma_prepare(req);
+ if (rc)
+ goto error;
+
+ rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len,
+ rctx->dma_list, req->result, rctx->dig_sz);
+
+ /* Unmap data and free DMA list regardless of return code. */
+ kmb_ocs_hcu_dma_cleanup(req, rctx);
+
+ /* Process previous return code. */
+ if (rc)
+ goto error;
+
+ goto done;
+ }
+
+ /* Handle update request case. */
+ if (!(rctx->flags & REQ_FINAL)) {
+ /* Update should always have input data. */
+ if (!kmb_get_total_data(rctx))
+ return -EINVAL;
+
+ /* Map input data into the HCU DMA linked list. */
+ rc = kmb_ocs_dma_prepare(req);
+ if (rc)
+ goto error;
+
+ /* Do hashing step. */
+ rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx,
+ rctx->dma_list);
+
+ /* Unmap data and free DMA list regardless of return code. */
+ kmb_ocs_hcu_dma_cleanup(req, rctx);
+
+ /* Process previous return code. */
+ if (rc)
+ goto error;
+
+ /*
+ * Reset request buffer count (data in the buffer was just
+ * processed).
+ */
+ rctx->buf_cnt = 0;
+ /*
+ * Move remaining sg data into the request buffer, so that it
+ * will be processed during the next request.
+ *
+ * NOTE: we have remaining data if kmb_get_total_data() was not
+ * a multiple of block size.
+ */
+ rc = flush_sg_to_ocs_buffer(rctx);
+ if (rc)
+ goto error;
+
+ goto done;
+ }
+
+ /* If we get here, this is a final request. */
+
+ /* If there is data to process, use finup. */
+ if (kmb_get_total_data(rctx)) {
+ /* Map input data into the HCU DMA linked list. */
+ rc = kmb_ocs_dma_prepare(req);
+ if (rc)
+ goto error;
+
+ /* Do hashing step. */
+ rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx,
+ rctx->dma_list,
+ req->result, rctx->dig_sz);
+ /* Free DMA list regardless of return code. */
+ kmb_ocs_hcu_dma_cleanup(req, rctx);
+
+ /* Process previous return code. */
+ if (rc)
+ goto error;
+
+ } else { /* Otherwise (if we have no data), use final. */
+ rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result,
+ rctx->dig_sz);
+ if (rc)
+ goto error;
+ }
+
+ /*
+ * If we are finalizing a SW HMAC request, we just computed the result
+ * of: H(k ^ ipad || m).
+ *
+ * We now need to complete the HMAC calculation with the OPAD step,
+ * that is, we need to compute H(k ^ opad || digest), where digest is
+ * the digest we just obtained, i.e., H(k ^ ipad || m).
+ */
+ if (rctx->flags & REQ_FLAGS_HMAC_SW) {
+ /*
+ * Compute k ^ opad and store it in the request buffer (which
+ * is not used anymore at this point).
+ * Note: key has been padded / hashed already (so keylen ==
+ * blksz) .
+ */
+ WARN_ON(tctx->key_len != rctx->blk_sz);
+ for (i = 0; i < rctx->blk_sz; i++)
+ rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE;
+ /* Now append the digest to the rest of the buffer. */
+ for (i = 0; (i < rctx->dig_sz); i++)
+ rctx->buffer[rctx->blk_sz + i] = req->result[i];
+
+ /* Now hash the buffer to obtain the final HMAC. */
+ rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer,
+ rctx->blk_sz + rctx->dig_sz, req->result,
+ rctx->dig_sz);
+ if (rc)
+ goto error;
+ }
+
+ /* Perform secure clean-up. */
+ kmb_ocs_hcu_secure_cleanup(req);
+done:
+ crypto_finalize_hash_request(hcu_dev->engine, req, 0);
+
+ return 0;
+
+error:
+ kmb_ocs_hcu_secure_cleanup(req);
+ return rc;
+}
+
+static int kmb_ocs_hcu_init(struct ahash_request *req)
+{
+ struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
+
+ if (!hcu_dev)
+ return -ENOENT;
+
+ /* Initialize entire request context to zero. */
+ memset(rctx, 0, sizeof(*rctx));
+
+ rctx->hcu_dev = hcu_dev;
+ rctx->dig_sz = crypto_ahash_digestsize(tfm);
+
+ switch (rctx->dig_sz) {
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
+ case SHA224_DIGEST_SIZE:
+ rctx->blk_sz = SHA224_BLOCK_SIZE;
+ rctx->algo = OCS_HCU_ALGO_SHA224;
+ break;
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
+ case SHA256_DIGEST_SIZE:
+ rctx->blk_sz = SHA256_BLOCK_SIZE;
+ /*
+ * SHA256 and SM3 have the same digest size: use info from tfm
+ * context to find out which one we should use.
+ */
+ rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 :
+ OCS_HCU_ALGO_SHA256;
+ break;
+ case SHA384_DIGEST_SIZE:
+ rctx->blk_sz = SHA384_BLOCK_SIZE;
+ rctx->algo = OCS_HCU_ALGO_SHA384;
+ break;
+ case SHA512_DIGEST_SIZE:
+ rctx->blk_sz = SHA512_BLOCK_SIZE;
+ rctx->algo = OCS_HCU_ALGO_SHA512;
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ /* Initialize intermediate data. */
+ ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo);
+
+ /* If this a HMAC request, set HMAC flag. */
+ if (ctx->is_hmac_tfm)
+ rctx->flags |= REQ_FLAGS_HMAC;
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_update(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ int rc;
+
+ if (!req->nbytes)
+ return 0;
+
+ rctx->sg_data_total = req->nbytes;
+ rctx->sg_data_offset = 0;
+ rctx->sg = req->src;
+
+ /*
+ * If we are doing HMAC, then we must use SW-assisted HMAC, since HW
+ * HMAC does not support context switching (there it can only be used
+ * with finup() or digest()).
+ */
+ if (rctx->flags & REQ_FLAGS_HMAC &&
+ !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
+ rctx->flags |= REQ_FLAGS_HMAC_SW;
+ rc = prepare_ipad(req);
+ if (rc)
+ return rc;
+ }
+
+ /*
+ * If remaining sg_data fits into ctx buffer, just copy it there; we'll
+ * process it at the next update() or final().
+ */
+ if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt))
+ return flush_sg_to_ocs_buffer(rctx);
+
+ return kmb_ocs_hcu_handle_queue(req);
+}
+
+/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */
+static int kmb_ocs_hcu_fin_common(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
+ int rc;
+
+ rctx->flags |= REQ_FINAL;
+
+ /*
+ * If this is a HMAC request and, so far, we didn't have to switch to
+ * SW HMAC, check if we can use HW HMAC.
+ */
+ if (rctx->flags & REQ_FLAGS_HMAC &&
+ !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
+ /*
+ * If we are here, it means we never processed any data so far,
+ * so we can use HW HMAC, but only if there is some data to
+ * process (since OCS HW MAC does not support zero-length
+ * messages) and the key length is supported by the hardware
+ * (OCS HCU HW only supports length <= 64); if HW HMAC cannot
+ * be used, fall back to SW-assisted HMAC.
+ */
+ if (kmb_get_total_data(rctx) &&
+ ctx->key_len <= OCS_HCU_HW_KEY_LEN) {
+ rctx->flags |= REQ_FLAGS_HMAC_HW;
+ } else {
+ rctx->flags |= REQ_FLAGS_HMAC_SW;
+ rc = prepare_ipad(req);
+ if (rc)
+ return rc;
+ }
+ }
+
+ return kmb_ocs_hcu_handle_queue(req);
+}
+
+static int kmb_ocs_hcu_final(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+
+ rctx->sg_data_total = 0;
+ rctx->sg_data_offset = 0;
+ rctx->sg = NULL;
+
+ return kmb_ocs_hcu_fin_common(req);
+}
+
+static int kmb_ocs_hcu_finup(struct ahash_request *req)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+
+ rctx->sg_data_total = req->nbytes;
+ rctx->sg_data_offset = 0;
+ rctx->sg = req->src;
+
+ return kmb_ocs_hcu_fin_common(req);
+}
+
+static int kmb_ocs_hcu_digest(struct ahash_request *req)
+{
+ int rc = 0;
+ struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
+
+ if (!hcu_dev)
+ return -ENOENT;
+
+ rc = kmb_ocs_hcu_init(req);
+ if (rc)
+ return rc;
+
+ rc = kmb_ocs_hcu_finup(req);
+
+ return rc;
+}
+
+static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+
+ /* Intermediate data is always stored and applied per request. */
+ memcpy(out, rctx, sizeof(*rctx));
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in)
+{
+ struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
+
+ /* Intermediate data is always stored and applied per request. */
+ memcpy(rctx, in, sizeof(*rctx));
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ unsigned int digestsize = crypto_ahash_digestsize(tfm);
+ struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
+ size_t blk_sz = crypto_ahash_blocksize(tfm);
+ struct crypto_ahash *ahash_tfm;
+ struct ahash_request *req;
+ struct crypto_wait wait;
+ struct scatterlist sg;
+ const char *alg_name;
+ int rc;
+
+ /*
+ * Key length must be equal to block size:
+ * - If key is shorter, we are done for now (the key will be padded
+ * later on); this is to maximize the use of HW HMAC (which works
+ * only for keys <= 64 bytes).
+ * - If key is longer, we hash it.
+ */
+ if (keylen <= blk_sz) {
+ memcpy(ctx->key, key, keylen);
+ ctx->key_len = keylen;
+ return 0;
+ }
+
+ switch (digestsize) {
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
+ case SHA224_DIGEST_SIZE:
+ alg_name = "sha224-keembay-ocs";
+ break;
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
+ case SHA256_DIGEST_SIZE:
+ alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" :
+ "sha256-keembay-ocs";
+ break;
+ case SHA384_DIGEST_SIZE:
+ alg_name = "sha384-keembay-ocs";
+ break;
+ case SHA512_DIGEST_SIZE:
+ alg_name = "sha512-keembay-ocs";
+ break;
+ default:
+ return -EINVAL;
+ }
+
+ ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
+ if (IS_ERR(ahash_tfm))
+ return PTR_ERR(ahash_tfm);
+
+ req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
+ if (!req) {
+ rc = -ENOMEM;
+ goto err_free_ahash;
+ }
+
+ crypto_init_wait(&wait);
+ ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
+ crypto_req_done, &wait);
+ crypto_ahash_clear_flags(ahash_tfm, ~0);
+
+ sg_init_one(&sg, key, keylen);
+ ahash_request_set_crypt(req, &sg, ctx->key, keylen);
+
+ rc = crypto_wait_req(crypto_ahash_digest(req), &wait);
+ if (rc == 0)
+ ctx->key_len = digestsize;
+
+ ahash_request_free(req);
+err_free_ahash:
+ crypto_free_ahash(ahash_tfm);
+
+ return rc;
+}
+
+/* Set request size and initialize tfm context. */
+static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx)
+{
+ crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
+ sizeof(struct ocs_hcu_rctx));
+
+ /* Init context to 0. */
+ memzero_explicit(ctx, sizeof(*ctx));
+ /* Set engine ops. */
+ ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request;
+}
+
+static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm)
+{
+ struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ __cra_init(tfm, ctx);
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm)
+{
+ struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ __cra_init(tfm, ctx);
+
+ ctx->is_sm3_tfm = true;
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm)
+{
+ struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ __cra_init(tfm, ctx);
+
+ ctx->is_sm3_tfm = true;
+ ctx->is_hmac_tfm = true;
+
+ return 0;
+}
+
+static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm)
+{
+ struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ __cra_init(tfm, ctx);
+
+ ctx->is_hmac_tfm = true;
+
+ return 0;
+}
+
+/* Function called when 'tfm' is de-initialized. */
+static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm)
+{
+ struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ /* Clear the key. */
+ memzero_explicit(ctx->key, sizeof(ctx->key));
+}
+
+static struct ahash_alg ocs_hcu_algs[] = {
+#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "sha224",
+ .cra_driver_name = "sha224-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_sha_cra_init,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .setkey = kmb_ocs_hcu_setkey,
+ .halg = {
+ .digestsize = SHA224_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "hmac(sha224)",
+ .cra_driver_name = "hmac-sha224-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA224_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_hmac_cra_init,
+ .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
+ }
+ }
+},
+#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "sha256",
+ .cra_driver_name = "sha256-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_sha_cra_init,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .setkey = kmb_ocs_hcu_setkey,
+ .halg = {
+ .digestsize = SHA256_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "hmac(sha256)",
+ .cra_driver_name = "hmac-sha256-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA256_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_hmac_cra_init,
+ .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .halg = {
+ .digestsize = SM3_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "sm3",
+ .cra_driver_name = "sm3-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SM3_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_sm3_cra_init,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .setkey = kmb_ocs_hcu_setkey,
+ .halg = {
+ .digestsize = SM3_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "hmac(sm3)",
+ .cra_driver_name = "hmac-sm3-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SM3_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init,
+ .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "sha384",
+ .cra_driver_name = "sha384-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_sha_cra_init,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .setkey = kmb_ocs_hcu_setkey,
+ .halg = {
+ .digestsize = SHA384_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "hmac(sha384)",
+ .cra_driver_name = "hmac-sha384-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA384_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_hmac_cra_init,
+ .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "sha512",
+ .cra_driver_name = "sha512-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_sha_cra_init,
+ }
+ }
+},
+{
+ .init = kmb_ocs_hcu_init,
+ .update = kmb_ocs_hcu_update,
+ .final = kmb_ocs_hcu_final,
+ .finup = kmb_ocs_hcu_finup,
+ .digest = kmb_ocs_hcu_digest,
+ .export = kmb_ocs_hcu_export,
+ .import = kmb_ocs_hcu_import,
+ .setkey = kmb_ocs_hcu_setkey,
+ .halg = {
+ .digestsize = SHA512_DIGEST_SIZE,
+ .statesize = sizeof(struct ocs_hcu_rctx),
+ .base = {
+ .cra_name = "hmac(sha512)",
+ .cra_driver_name = "hmac-sha512-keembay-ocs",
+ .cra_priority = 255,
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SHA512_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
+ .cra_alignmask = 0,
+ .cra_module = THIS_MODULE,
+ .cra_init = kmb_ocs_hcu_hmac_cra_init,
+ .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
+ }
+ }
+},
+};
+
+/* Device tree driver match. */
+static const struct of_device_id kmb_ocs_hcu_of_match[] = {
+ {
+ .compatible = "intel,keembay-ocs-hcu",
+ },
+ {}
+};
+
+static int kmb_ocs_hcu_remove(struct platform_device *pdev)
+{
+ struct ocs_hcu_dev *hcu_dev;
+ int rc;
+
+ hcu_dev = platform_get_drvdata(pdev);
+ if (!hcu_dev)
+ return -ENODEV;
+
+ crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
+
+ rc = crypto_engine_exit(hcu_dev->engine);
+
+ spin_lock_bh(&ocs_hcu.lock);
+ list_del(&hcu_dev->list);
+ spin_unlock_bh(&ocs_hcu.lock);
+
+ return rc;
+}
+
+static int kmb_ocs_hcu_probe(struct platform_device *pdev)
+{
+ struct device *dev = &pdev->dev;
+ struct ocs_hcu_dev *hcu_dev;
+ struct resource *hcu_mem;
+ int rc;
+
+ hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL);
+ if (!hcu_dev)
+ return -ENOMEM;
+
+ hcu_dev->dev = dev;
+
+ platform_set_drvdata(pdev, hcu_dev);
+ rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK);
+ if (rc)
+ return rc;
+
+ /* Get the memory address and remap. */
+ hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+ if (!hcu_mem) {
+ dev_err(dev, "Could not retrieve io mem resource.\n");
+ return -ENODEV;
+ }
+
+ hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem);
+ if (IS_ERR(hcu_dev->io_base))
+ return PTR_ERR(hcu_dev->io_base);
+
+ init_completion(&hcu_dev->irq_done);
+
+ /* Get and request IRQ. */
+ hcu_dev->irq = platform_get_irq(pdev, 0);
+ if (hcu_dev->irq < 0)
+ return hcu_dev->irq;
+
+ rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq,
+ ocs_hcu_irq_handler, NULL, 0,
+ "keembay-ocs-hcu", hcu_dev);
+ if (rc < 0) {
+ dev_err(dev, "Could not request IRQ.\n");
+ return rc;
+ }
+
+ INIT_LIST_HEAD(&hcu_dev->list);
+
+ spin_lock_bh(&ocs_hcu.lock);
+ list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list);
+ spin_unlock_bh(&ocs_hcu.lock);
+
+ /* Initialize crypto engine */
+ hcu_dev->engine = crypto_engine_alloc_init(dev, 1);
+ if (!hcu_dev->engine) {
+ rc = -ENOMEM;
+ goto list_del;
+ }
+
+ rc = crypto_engine_start(hcu_dev->engine);
+ if (rc) {
+ dev_err(dev, "Could not start engine.\n");
+ goto cleanup;
+ }
+
+ /* Security infrastructure guarantees OCS clock is enabled. */
+
+ rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
+ if (rc) {
+ dev_err(dev, "Could not register algorithms.\n");
+ goto cleanup;
+ }
+
+ return 0;
+
+cleanup:
+ crypto_engine_exit(hcu_dev->engine);
+list_del:
+ spin_lock_bh(&ocs_hcu.lock);
+ list_del(&hcu_dev->list);
+ spin_unlock_bh(&ocs_hcu.lock);
+
+ return rc;
+}
+
+/* The OCS driver is a platform device. */
+static struct platform_driver kmb_ocs_hcu_driver = {
+ .probe = kmb_ocs_hcu_probe,
+ .remove = kmb_ocs_hcu_remove,
+ .driver = {
+ .name = DRV_NAME,
+ .of_match_table = kmb_ocs_hcu_of_match,
+ },
+};
+
+module_platform_driver(kmb_ocs_hcu_driver);
+
+MODULE_LICENSE("GPL");
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Keem Bay OCS AES Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/swab.h>
+
+#include <asm/byteorder.h>
+#include <asm/errno.h>
+
+#include <crypto/aes.h>
+#include <crypto/gcm.h>
+
+#include "ocs-aes.h"
+
+#define AES_COMMAND_OFFSET 0x0000
+#define AES_KEY_0_OFFSET 0x0004
+#define AES_KEY_1_OFFSET 0x0008
+#define AES_KEY_2_OFFSET 0x000C
+#define AES_KEY_3_OFFSET 0x0010
+#define AES_KEY_4_OFFSET 0x0014
+#define AES_KEY_5_OFFSET 0x0018
+#define AES_KEY_6_OFFSET 0x001C
+#define AES_KEY_7_OFFSET 0x0020
+#define AES_IV_0_OFFSET 0x0024
+#define AES_IV_1_OFFSET 0x0028
+#define AES_IV_2_OFFSET 0x002C
+#define AES_IV_3_OFFSET 0x0030
+#define AES_ACTIVE_OFFSET 0x0034
+#define AES_STATUS_OFFSET 0x0038
+#define AES_KEY_SIZE_OFFSET 0x0044
+#define AES_IER_OFFSET 0x0048
+#define AES_ISR_OFFSET 0x005C
+#define AES_MULTIPURPOSE1_0_OFFSET 0x0200
+#define AES_MULTIPURPOSE1_1_OFFSET 0x0204
+#define AES_MULTIPURPOSE1_2_OFFSET 0x0208
+#define AES_MULTIPURPOSE1_3_OFFSET 0x020C
+#define AES_MULTIPURPOSE2_0_OFFSET 0x0220
+#define AES_MULTIPURPOSE2_1_OFFSET 0x0224
+#define AES_MULTIPURPOSE2_2_OFFSET 0x0228
+#define AES_MULTIPURPOSE2_3_OFFSET 0x022C
+#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0
+#define AES_TLEN_OFFSET 0x0300
+#define AES_T_MAC_0_OFFSET 0x0304
+#define AES_T_MAC_1_OFFSET 0x0308
+#define AES_T_MAC_2_OFFSET 0x030C
+#define AES_T_MAC_3_OFFSET 0x0310
+#define AES_PLEN_OFFSET 0x0314
+#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400
+#define AES_A_DMA_DST_ADDR_OFFSET 0x0404
+#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408
+#define AES_A_DMA_DST_SIZE_OFFSET 0x040C
+#define AES_A_DMA_DMA_MODE_OFFSET 0x0410
+#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418
+#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C
+#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420
+#define AES_A_DMA_LOG_OFFSET 0x0424
+#define AES_A_DMA_STATUS_OFFSET 0x0428
+#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C
+#define AES_A_DMA_MSI_ISR_OFFSET 0x0480
+#define AES_A_DMA_MSI_IER_OFFSET 0x0484
+#define AES_A_DMA_MSI_MASK_OFFSET 0x0488
+#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600
+#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700
+
+/*
+ * AES_A_DMA_DMA_MODE register.
+ * Default: 0x00000000.
+ * bit[31] ACTIVE
+ * This bit activates the DMA. When the DMA finishes, it resets
+ * this bit to zero.
+ * bit[30:26] Unused by this driver.
+ * bit[25] SRC_LINK_LIST_EN
+ * Source link list enable bit. When the linked list is terminated
+ * this bit is reset by the DMA.
+ * bit[24] DST_LINK_LIST_EN
+ * Destination link list enable bit. When the linked list is
+ * terminated this bit is reset by the DMA.
+ * bit[23:0] Unused by this driver.
+ */
+#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31)
+#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25)
+#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24)
+
+/*
+ * AES_ACTIVE register
+ * default 0x00000000
+ * bit[31:10] Reserved
+ * bit[9] LAST_ADATA
+ * bit[8] LAST_GCX
+ * bit[7:2] Reserved
+ * bit[1] TERMINATION
+ * bit[0] TRIGGER
+ */
+#define AES_ACTIVE_LAST_ADATA BIT(9)
+#define AES_ACTIVE_LAST_CCM_GCM BIT(8)
+#define AES_ACTIVE_TERMINATION BIT(1)
+#define AES_ACTIVE_TRIGGER BIT(0)
+
+#define AES_DISABLE_INT 0x00000000
+#define AES_DMA_CPD_ERR_INT BIT(8)
+#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7)
+#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6)
+#define AES_DMA_INBUF_RD_ERR_INT BIT(5)
+#define AES_DMA_INBUF_WR_ERR_INT BIT(4)
+#define AES_DMA_BAD_COMP_INT BIT(3)
+#define AES_DMA_SAI_INT BIT(2)
+#define AES_DMA_SRC_DONE_INT BIT(0)
+#define AES_COMPLETE_INT BIT(1)
+
+#define AES_DMA_MSI_MASK_CLEAR BIT(0)
+
+#define AES_128_BIT_KEY 0x00000000
+#define AES_256_BIT_KEY BIT(0)
+
+#define AES_DEACTIVATE_PERF_CNTR 0x00000000
+#define AES_ACTIVATE_PERF_CNTR BIT(0)
+
+#define AES_MAX_TAG_SIZE_U32 4
+
+#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
+
+/*
+ * There is an inconsistency in the documentation. This is documented as a
+ * 11-bit value, but it is actually 10-bits.
+ */
+#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF
+
+/*
+ * During CCM decrypt, the OCS block needs to finish processing the ciphertext
+ * before the tag is written. For 128-bit mode this required delay is 28 OCS
+ * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
+ */
+#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL
+
+/*
+ * During CCM decrypt there must be a delay of at least 42 OCS clock cycles
+ * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM
+ * bit in the same register (as stated in the OCS databook)
+ */
+#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL
+
+/* See RFC3610 section 2.2 */
+#define L_PRIME_MIN (1)
+#define L_PRIME_MAX (7)
+/*
+ * CCM IV format from RFC 3610 section 2.3
+ *
+ * Octet Number Contents
+ * ------------ ---------
+ * 0 Flags
+ * 1 ... 15-L Nonce N
+ * 16-L ... 15 Counter i
+ *
+ * Flags = L' = L - 1
+ */
+#define L_PRIME_IDX 0
+#define COUNTER_START(lprime) (16 - ((lprime) + 1))
+#define COUNTER_LEN(lprime) ((lprime) + 1)
+
+enum aes_counter_mode {
+ AES_CTR_M_NO_INC = 0,
+ AES_CTR_M_32_INC = 1,
+ AES_CTR_M_64_INC = 2,
+ AES_CTR_M_128_INC = 3,
+};
+
+/**
+ * struct ocs_dma_linked_list - OCS DMA linked list entry.
+ * @src_addr: Source address of the data.
+ * @src_len: Length of data to be fetched.
+ * @next: Next dma_list to fetch.
+ * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
+ */
+struct ocs_dma_linked_list {
+ u32 src_addr;
+ u32 src_len;
+ u32 next;
+ u32 ll_flags;
+} __packed;
+
+/*
+ * Set endianness of inputs and outputs
+ * AES_BYTE_ORDER_CFG
+ * default 0x00000000
+ * bit [10] - KEY_HI_LO_SWAP
+ * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
+ * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
+ * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
+ * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
+ * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
+ * bit [4] - IV_SWAP_BYTES_IN_DWORD
+ * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
+ * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
+ * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
+ * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
+ */
+static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
+}
+
+/* Trigger AES process start. */
+static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
+}
+
+/* Indicate last bulk of data. */
+static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_ACTIVE_TERMINATION,
+ aes_dev->base_reg + AES_ACTIVE_OFFSET);
+}
+
+/*
+ * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits.
+ *
+ * Called when DMA is programmed to fetch the last batch of data.
+ * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
+ * data.
+ * - For AES-GCM, it is called for the last batch of Plaintext data and
+ * Ciphertext data.
+ */
+static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_ACTIVE_LAST_CCM_GCM,
+ aes_dev->base_reg + AES_ACTIVE_OFFSET);
+}
+
+/* Wait for LAST_CCM_GCM bit to be unset. */
+static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
+{
+ u32 aes_active_reg;
+
+ do {
+ aes_active_reg = ioread32(aes_dev->base_reg +
+ AES_ACTIVE_OFFSET);
+ } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
+}
+
+/* Wait for 10 bits of input occupancy. */
+static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
+{
+ u32 reg;
+
+ do {
+ reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
+ } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
+}
+
+ /*
+ * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all
+ * other bits).
+ *
+ * Called when DMA is programmed to fetch the last batch of Associated Data
+ * (CCM case) or Additional Authenticated Data (GCM case).
+ */
+static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
+ aes_dev->base_reg + AES_ACTIVE_OFFSET);
+}
+
+/* Set DMA src and dst transfer size to 0 */
+static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
+ iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
+}
+
+/* Activate DMA for zero-byte transfer case. */
+static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
+ aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
+}
+
+/* Activate DMA and enable src linked list */
+static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
+ AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
+ aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
+}
+
+/* Activate DMA and enable dst linked list */
+static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
+ AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
+ aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
+}
+
+/* Activate DMA and enable src and dst linked lists */
+static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
+ AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
+ AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
+ aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
+}
+
+/* Reset PERF_CNTR to 0 and activate it */
+static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
+{
+ iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
+ iowrite32(AES_ACTIVATE_PERF_CNTR,
+ aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
+}
+
+/* Wait until PERF_CNTR is > delay, then deactivate it */
+static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
+ int delay)
+{
+ while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
+ ;
+ iowrite32(AES_DEACTIVATE_PERF_CNTR,
+ aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
+}
+
+/* Disable AES and DMA IRQ. */
+static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
+{
+ u32 isr_val = 0;
+
+ /* Disable interrupts */
+ iowrite32(AES_DISABLE_INT,
+ aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
+ iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
+
+ /* Clear any pending interrupt */
+ isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
+ if (isr_val)
+ iowrite32(isr_val,
+ aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
+
+ isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
+ if (isr_val)
+ iowrite32(isr_val,
+ aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
+
+ isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
+ if (isr_val)
+ iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
+}
+
+/* Enable AES or DMA IRQ. IRQ is disabled once fired. */
+static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
+{
+ if (irq == AES_COMPLETE_INT) {
+ /* Ensure DMA error interrupts are enabled */
+ iowrite32(AES_DMA_CPD_ERR_INT |
+ AES_DMA_OUTBUF_RD_ERR_INT |
+ AES_DMA_OUTBUF_WR_ERR_INT |
+ AES_DMA_INBUF_RD_ERR_INT |
+ AES_DMA_INBUF_WR_ERR_INT |
+ AES_DMA_BAD_COMP_INT |
+ AES_DMA_SAI_INT,
+ aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
+ /*
+ * AES_IER
+ * default 0x00000000
+ * bits [31:3] - reserved
+ * bit [2] - EN_SKS_ERR
+ * bit [1] - EN_AES_COMPLETE
+ * bit [0] - reserved
+ */
+ iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
+ return;
+ }
+ if (irq == AES_DMA_SRC_DONE_INT) {
+ /* Ensure AES interrupts are disabled */
+ iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
+ /*
+ * DMA_MSI_IER
+ * default 0x00000000
+ * bits [31:9] - reserved
+ * bit [8] - CPD_ERR_INT_EN
+ * bit [7] - OUTBUF_RD_ERR_INT_EN
+ * bit [6] - OUTBUF_WR_ERR_INT_EN
+ * bit [5] - INBUF_RD_ERR_INT_EN
+ * bit [4] - INBUF_WR_ERR_INT_EN
+ * bit [3] - BAD_COMP_INT_EN
+ * bit [2] - SAI_INT_EN
+ * bit [1] - DST_DONE_INT_EN
+ * bit [0] - SRC_DONE_INT_EN
+ */
+ iowrite32(AES_DMA_CPD_ERR_INT |
+ AES_DMA_OUTBUF_RD_ERR_INT |
+ AES_DMA_OUTBUF_WR_ERR_INT |
+ AES_DMA_INBUF_RD_ERR_INT |
+ AES_DMA_INBUF_WR_ERR_INT |
+ AES_DMA_BAD_COMP_INT |
+ AES_DMA_SAI_INT |
+ AES_DMA_SRC_DONE_INT,
+ aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
+ }
+}
+
+/* Enable and wait for IRQ (either from OCS AES engine or DMA) */
+static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
+{
+ int rc;
+
+ reinit_completion(&aes_dev->irq_completion);
+ aes_irq_enable(aes_dev, irq);
+ rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
+ if (rc)
+ return rc;
+
+ return aes_dev->dma_err_mask ? -EIO : 0;
+}
+
+/* Configure DMA to OCS, linked list mode */
+static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
+ dma_addr_t dma_list)
+{
+ iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
+ iowrite32(dma_list,
+ aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
+}
+
+/* Configure DMA from OCS, linked list mode */
+static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
+ dma_addr_t dma_list)
+{
+ iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
+ iowrite32(dma_list,
+ aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
+}
+
+irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
+{
+ struct ocs_aes_dev *aes_dev = dev_id;
+ u32 aes_dma_isr;
+
+ /* Read DMA ISR status. */
+ aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
+
+ /* Disable and clear interrupts. */
+ aes_irq_disable(aes_dev);
+
+ /* Save DMA error status. */
+ aes_dev->dma_err_mask = aes_dma_isr &
+ (AES_DMA_CPD_ERR_INT |
+ AES_DMA_OUTBUF_RD_ERR_INT |
+ AES_DMA_OUTBUF_WR_ERR_INT |
+ AES_DMA_INBUF_RD_ERR_INT |
+ AES_DMA_INBUF_WR_ERR_INT |
+ AES_DMA_BAD_COMP_INT |
+ AES_DMA_SAI_INT);
+
+ /* Signal IRQ completion. */
+ complete(&aes_dev->irq_completion);
+
+ return IRQ_HANDLED;
+}
+
+/**
+ * ocs_aes_set_key() - Write key into OCS AES hardware.
+ * @aes_dev: The OCS AES device to write the key to.
+ * @key_size: The size of the key (in bytes).
+ * @key: The key to write.
+ * @cipher: The cipher the key is for.
+ *
+ * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
+ enum ocs_cipher cipher)
+{
+ const u32 *key_u32;
+ u32 val;
+ int i;
+
+ /* OCS AES supports 128-bit and 256-bit keys only. */
+ if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
+ dev_err(aes_dev->dev,
+ "%d-bit keys not supported by AES cipher\n",
+ key_size * 8);
+ return -EINVAL;
+ }
+ /* OCS SM4 supports 128-bit keys only. */
+ if (cipher == OCS_SM4 && key_size != 16) {
+ dev_err(aes_dev->dev,
+ "%d-bit keys not supported for SM4 cipher\n",
+ key_size * 8);
+ return -EINVAL;
+ }
+
+ if (!key)
+ return -EINVAL;
+
+ key_u32 = (const u32 *)key;
+
+ /* Write key to AES_KEY[0-7] registers */
+ for (i = 0; i < (key_size / sizeof(u32)); i++) {
+ iowrite32(key_u32[i],
+ aes_dev->base_reg + AES_KEY_0_OFFSET +
+ (i * sizeof(u32)));
+ }
+ /*
+ * Write key size
+ * bits [31:1] - reserved
+ * bit [0] - AES_KEY_SIZE
+ * 0 - 128 bit key
+ * 1 - 256 bit key
+ */
+ val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
+ iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
+
+ return 0;
+}
+
+/* Write AES_COMMAND */
+static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
+ enum ocs_cipher cipher,
+ enum ocs_mode mode,
+ enum ocs_instruction instruction)
+{
+ u32 val;
+
+ /* AES_COMMAND
+ * default 0x000000CC
+ * bit [14] - CIPHER_SELECT
+ * 0 - AES
+ * 1 - SM4
+ * bits [11:8] - OCS_AES_MODE
+ * 0000 - ECB
+ * 0001 - CBC
+ * 0010 - CTR
+ * 0110 - CCM
+ * 0111 - GCM
+ * 1001 - CTS
+ * bits [7:6] - AES_INSTRUCTION
+ * 00 - ENCRYPT
+ * 01 - DECRYPT
+ * 10 - EXPAND
+ * 11 - BYPASS
+ * bits [3:2] - CTR_M_BITS
+ * 00 - No increment
+ * 01 - Least significant 32 bits are incremented
+ * 10 - Least significant 64 bits are incremented
+ * 11 - Full 128 bits are incremented
+ */
+ val = (cipher << 14) | (mode << 8) | (instruction << 6) |
+ (AES_CTR_M_128_INC << 2);
+ iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
+}
+
+static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
+ enum ocs_mode mode,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction)
+{
+ /* Ensure interrupts are disabled and pending interrupts cleared. */
+ aes_irq_disable(aes_dev);
+
+ /* Set endianness recommended by data-sheet. */
+ aes_a_set_endianness(aes_dev);
+
+ /* Set AES_COMMAND register. */
+ set_ocs_aes_command(aes_dev, cipher, mode, instruction);
+}
+
+/*
+ * Write the byte length of the last AES/SM4 block of Payload data (without
+ * zero padding and without the length of the MAC) in register AES_PLEN.
+ */
+static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
+ u32 size)
+{
+ u32 val;
+
+ if (size == 0) {
+ val = 0;
+ goto exit;
+ }
+
+ val = size % AES_BLOCK_SIZE;
+ if (val == 0)
+ val = AES_BLOCK_SIZE;
+
+exit:
+ iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
+}
+
+/*
+ * Validate inputs according to mode.
+ * If OK return 0; else return -EINVAL.
+ */
+static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
+ const u8 *iv, u32 iv_size,
+ dma_addr_t aad_dma_list, u32 aad_size,
+ const u8 *tag, u32 tag_size,
+ enum ocs_cipher cipher, enum ocs_mode mode,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list)
+{
+ /* Ensure cipher, mode and instruction are valid. */
+ if (!(cipher == OCS_AES || cipher == OCS_SM4))
+ return -EINVAL;
+
+ if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
+ mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
+ mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
+ return -EINVAL;
+
+ if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
+ instruction != OCS_EXPAND && instruction != OCS_BYPASS)
+ return -EINVAL;
+
+ /*
+ * When instruction is OCS_BYPASS, OCS simply copies data from source
+ * to destination using DMA.
+ *
+ * AES mode is irrelevant, but both source and destination DMA
+ * linked-list must be defined.
+ */
+ if (instruction == OCS_BYPASS) {
+ if (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ return 0;
+ }
+
+ /*
+ * For performance reasons switch based on mode to limit unnecessary
+ * conditionals for each mode
+ */
+ switch (mode) {
+ case OCS_MODE_ECB:
+ /* Ensure input length is multiple of block size */
+ if (src_size % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ /* Ensure source and destination linked lists are created */
+ if (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CBC:
+ /* Ensure input length is multiple of block size */
+ if (src_size % AES_BLOCK_SIZE != 0)
+ return -EINVAL;
+
+ /* Ensure source and destination linked lists are created */
+ if (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* Ensure IV is present and block size in length */
+ if (!iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CTR:
+ /* Ensure input length of 1 byte or greater */
+ if (src_size == 0)
+ return -EINVAL;
+
+ /* Ensure source and destination linked lists are created */
+ if (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* Ensure IV is present and block size in length */
+ if (!iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CTS:
+ /* Ensure input length >= block size */
+ if (src_size < AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ /* Ensure source and destination linked lists are created */
+ if (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* Ensure IV is present and block size in length */
+ if (!iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_GCM:
+ /* Ensure IV is present and GCM_AES_IV_SIZE in length */
+ if (!iv || iv_size != GCM_AES_IV_SIZE)
+ return -EINVAL;
+
+ /*
+ * If input data present ensure source and destination linked
+ * lists are created
+ */
+ if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR))
+ return -EINVAL;
+
+ /* If aad present ensure aad linked list is created */
+ if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* Ensure tag destination is set */
+ if (!tag)
+ return -EINVAL;
+
+ /* Just ensure that tag_size doesn't cause overflows. */
+ if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
+ return -EINVAL;
+
+ return 0;
+
+ case OCS_MODE_CCM:
+ /* Ensure IV is present and block size in length */
+ if (!iv || iv_size != AES_BLOCK_SIZE)
+ return -EINVAL;
+
+ /* 2 <= L <= 8, so 1 <= L' <= 7 */
+ if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
+ iv[L_PRIME_IDX] > L_PRIME_MAX)
+ return -EINVAL;
+
+ /* If aad present ensure aad linked list is created */
+ if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* Just ensure that tag_size doesn't cause overflows. */
+ if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
+ return -EINVAL;
+
+ if (instruction == OCS_DECRYPT) {
+ /*
+ * If input data present ensure source and destination
+ * linked lists are created
+ */
+ if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
+ dst_dma_list == DMA_MAPPING_ERROR))
+ return -EINVAL;
+
+ /* Ensure input tag is present */
+ if (!tag)
+ return -EINVAL;
+
+ return 0;
+ }
+
+ /* Instruction == OCS_ENCRYPT */
+
+ /*
+ * Destination linked list always required (for tag even if no
+ * input data)
+ */
+ if (dst_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ /* If input data present ensure src linked list is created */
+ if (src_size && src_dma_list == DMA_MAPPING_ERROR)
+ return -EINVAL;
+
+ return 0;
+
+ default:
+ return -EINVAL;
+ }
+}
+
+/**
+ * ocs_aes_op() - Perform AES/SM4 operation.
+ * @aes_dev: The OCS AES device to use.
+ * @mode: The mode to use (ECB, CBC, CTR, or CTS).
+ * @cipher: The cipher to use (AES or SM4).
+ * @instruction: The instruction to perform (encrypt or decrypt).
+ * @dst_dma_list: The OCS DMA list mapping output memory.
+ * @src_dma_list: The OCS DMA list mapping input payload data.
+ * @src_size: The amount of data mapped by @src_dma_list.
+ * @iv: The IV vector.
+ * @iv_size: The size (in bytes) of @iv.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_aes_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_mode mode,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ u8 *iv,
+ u32 iv_size)
+{
+ u32 *iv32;
+ int rc;
+
+ rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
+ NULL, 0, cipher, mode, instruction,
+ dst_dma_list);
+ if (rc)
+ return rc;
+ /*
+ * ocs_aes_validate_inputs() is a generic check, now ensure mode is not
+ * GCM or CCM.
+ */
+ if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
+ return -EINVAL;
+
+ /* Cast IV to u32 array. */
+ iv32 = (u32 *)iv;
+
+ ocs_aes_init(aes_dev, mode, cipher, instruction);
+
+ if (mode == OCS_MODE_CTS) {
+ /* Write the byte length of the last data block to engine. */
+ ocs_aes_write_last_data_blk_len(aes_dev, src_size);
+ }
+
+ /* ECB is the only mode that doesn't use IV. */
+ if (mode != OCS_MODE_ECB) {
+ iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
+ iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
+ iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
+ iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
+ }
+
+ /* Set AES_ACTIVE.TRIGGER to start the operation. */
+ aes_a_op_trigger(aes_dev);
+
+ /* Configure and activate input / output DMA. */
+ dma_to_ocs_aes_ll(aes_dev, src_dma_list);
+ dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
+ aes_a_dma_active_src_dst_ll_en(aes_dev);
+
+ if (mode == OCS_MODE_CTS) {
+ /*
+ * For CTS mode, instruct engine to activate ciphertext
+ * stealing if last block of data is incomplete.
+ */
+ aes_a_set_last_gcx(aes_dev);
+ } else {
+ /* For all other modes, just write the 'termination' bit. */
+ aes_a_op_termination(aes_dev);
+ }
+
+ /* Wait for engine to complete processing. */
+ rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
+ if (rc)
+ return rc;
+
+ if (mode == OCS_MODE_CTR) {
+ /* Read back IV for streaming mode */
+ iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
+ iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
+ iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
+ iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
+ }
+
+ return 0;
+}
+
+/* Compute and write J0 to engine registers. */
+static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
+ const u8 *iv)
+{
+ const u32 *j0 = (u32 *)iv;
+
+ /*
+ * IV must be 12 bytes; Other sizes not supported as Linux crypto API
+ * does only expects/allows 12 byte IV for GCM
+ */
+ iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
+ iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
+ iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
+ iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
+}
+
+/* Read GCM tag from engine registers. */
+static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
+ u8 *tag, u32 tag_size)
+{
+ u32 tag_u32[AES_MAX_TAG_SIZE_U32];
+
+ /*
+ * The Authentication Tag T is stored in Little Endian order in the
+ * registers with the most significant bytes stored from AES_T_MAC[3]
+ * downward.
+ */
+ tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
+ tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
+ tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
+ tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
+
+ memcpy(tag, tag_u32, tag_size);
+}
+
+/**
+ * ocs_aes_gcm_op() - Perform GCM operation.
+ * @aes_dev: The OCS AES device to use.
+ * @cipher: The Cipher to use (AES or SM4).
+ * @instruction: The instruction to perform (encrypt or decrypt).
+ * @dst_dma_list: The OCS DMA list mapping output memory.
+ * @src_dma_list: The OCS DMA list mapping input payload data.
+ * @src_size: The amount of data mapped by @src_dma_list.
+ * @iv: The input IV vector.
+ * @aad_dma_list: The OCS DMA list mapping input AAD data.
+ * @aad_size: The amount of data mapped by @aad_dma_list.
+ * @out_tag: Where to store computed tag.
+ * @tag_size: The size (in bytes) of @out_tag.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ const u8 *iv,
+ dma_addr_t aad_dma_list,
+ u32 aad_size,
+ u8 *out_tag,
+ u32 tag_size)
+{
+ u64 bit_len;
+ u32 val;
+ int rc;
+
+ rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
+ GCM_AES_IV_SIZE, aad_dma_list,
+ aad_size, out_tag, tag_size, cipher,
+ OCS_MODE_GCM, instruction,
+ dst_dma_list);
+ if (rc)
+ return rc;
+
+ ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
+
+ /* Compute and write J0 to OCS HW. */
+ ocs_aes_gcm_write_j0(aes_dev, iv);
+
+ /* Write out_tag byte length */
+ iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
+
+ /* Write the byte length of the last plaintext / ciphertext block. */
+ ocs_aes_write_last_data_blk_len(aes_dev, src_size);
+
+ /* Write ciphertext bit length */
+ bit_len = (u64)src_size * 8;
+ val = bit_len & 0xFFFFFFFF;
+ iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
+ val = bit_len >> 32;
+ iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
+
+ /* Write aad bit length */
+ bit_len = (u64)aad_size * 8;
+ val = bit_len & 0xFFFFFFFF;
+ iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
+ val = bit_len >> 32;
+ iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
+
+ /* Set AES_ACTIVE.TRIGGER to start the operation. */
+ aes_a_op_trigger(aes_dev);
+
+ /* Process AAD. */
+ if (aad_size) {
+ /* If aad present, configure DMA to feed it to the engine. */
+ dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
+ aes_a_dma_active_src_ll_en(aes_dev);
+
+ /* Instructs engine to pad last block of aad, if needed. */
+ aes_a_set_last_gcx_and_adata(aes_dev);
+
+ /* Wait for DMA transfer to complete. */
+ rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
+ if (rc)
+ return rc;
+ } else {
+ aes_a_set_last_gcx_and_adata(aes_dev);
+ }
+
+ /* Wait until adata (if present) has been processed. */
+ aes_a_wait_last_gcx(aes_dev);
+ aes_a_dma_wait_input_buffer_occupancy(aes_dev);
+
+ /* Now process payload. */
+ if (src_size) {
+ /* Configure and activate DMA for both input and output data. */
+ dma_to_ocs_aes_ll(aes_dev, src_dma_list);
+ dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
+ aes_a_dma_active_src_dst_ll_en(aes_dev);
+ } else {
+ aes_a_dma_set_xfer_size_zero(aes_dev);
+ aes_a_dma_active(aes_dev);
+ }
+
+ /* Instruct AES/SMA4 engine payload processing is over. */
+ aes_a_set_last_gcx(aes_dev);
+
+ /* Wait for OCS AES engine to complete processing. */
+ rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
+ if (rc)
+ return rc;
+
+ ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
+
+ return 0;
+}
+
+/* Write encrypted tag to AES/SM4 engine. */
+static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
+ const u8 *in_tag, u32 tag_size)
+{
+ int i;
+
+ /* Ensure DMA input buffer is empty */
+ aes_a_dma_wait_input_buffer_occupancy(aes_dev);
+
+ /*
+ * During CCM decrypt, the OCS block needs to finish processing the
+ * ciphertext before the tag is written. So delay needed after DMA has
+ * completed writing the ciphertext
+ */
+ aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
+ aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
+ CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
+
+ /* Write encrypted tag to AES/SM4 engine. */
+ for (i = 0; i < tag_size; i++) {
+ iowrite8(in_tag[i], aes_dev->base_reg +
+ AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
+ }
+}
+
+/*
+ * Write B0 CCM block to OCS AES HW.
+ *
+ * Note: B0 format is documented in NIST Special Publication 800-38C
+ * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
+ * (see Section A.2.1)
+ */
+static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
+ const u8 *iv, u32 adata_size, u32 tag_size,
+ u32 cryptlen)
+{
+ u8 b0[16]; /* CCM B0 block is 16 bytes long. */
+ int i, q;
+
+ /* Initialize B0 to 0. */
+ memset(b0, 0, sizeof(b0));
+
+ /*
+ * B0[0] is the 'Flags Octet' and has the following structure:
+ * bit 7: Reserved
+ * bit 6: Adata flag
+ * bit 5-3: t value encoded as (t-2)/2
+ * bit 2-0: q value encoded as q - 1
+ */
+ /* If there is AAD data, set the Adata flag. */
+ if (adata_size)
+ b0[0] |= BIT(6);
+ /*
+ * t denotes the octet length of T.
+ * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is
+ * encoded as (t - 2) / 2
+ */
+ b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
+ /*
+ * q is the octet length of Q.
+ * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as
+ * q - 1 == iv[0] & 0x7;
+ */
+ b0[0] |= iv[0] & 0x7;
+ /*
+ * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
+ * and must be copied to b0[1]..b0[15-q].
+ * q == (iv[0] & 0x7) + 1
+ */
+ q = (iv[0] & 0x7) + 1;
+ for (i = 1; i <= 15 - q; i++)
+ b0[i] = iv[i];
+ /*
+ * The rest of B0 must contain Q, i.e., the message length.
+ * Q is encoded in q octets, in big-endian order, so to write it, we
+ * start from the end of B0 and we move backward.
+ */
+ i = sizeof(b0) - 1;
+ while (q) {
+ b0[i] = cryptlen & 0xff;
+ cryptlen >>= 8;
+ i--;
+ q--;
+ }
+ /*
+ * If cryptlen is not zero at this point, it means that its original
+ * value was too big.
+ */
+ if (cryptlen)
+ return -EOVERFLOW;
+ /* Now write B0 to OCS AES input buffer. */
+ for (i = 0; i < sizeof(b0); i++)
+ iowrite8(b0[i], aes_dev->base_reg +
+ AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
+ return 0;
+}
+
+/*
+ * Write adata length to OCS AES HW.
+ *
+ * Note: adata len encoding is documented in NIST Special Publication 800-38C
+ * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
+ * (see Section A.2.2)
+ */
+static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
+ u64 adata_len)
+{
+ u8 enc_a[10]; /* Maximum encoded size: 10 octets. */
+ int i, len;
+
+ /*
+ * adata_len ('a') is encoded as follows:
+ * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets
+ * (big endian).
+ * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
+ * i.e., six octets (big endian).
+ * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64,
+ * i.e., ten octets (big endian).
+ */
+ if (adata_len < 65280) {
+ len = 2;
+ *(__be16 *)enc_a = cpu_to_be16(adata_len);
+ } else if (adata_len <= 0xFFFFFFFF) {
+ len = 6;
+ *(__be16 *)enc_a = cpu_to_be16(0xfffe);
+ *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
+ } else { /* adata_len >= 2^32 */
+ len = 10;
+ *(__be16 *)enc_a = cpu_to_be16(0xffff);
+ *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
+ }
+ for (i = 0; i < len; i++)
+ iowrite8(enc_a[i],
+ aes_dev->base_reg +
+ AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
+}
+
+static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
+ dma_addr_t adata_dma_list, u32 adata_size)
+{
+ int rc;
+
+ if (!adata_size) {
+ /* Since no aad the LAST_GCX bit can be set now */
+ aes_a_set_last_gcx_and_adata(aes_dev);
+ goto exit;
+ }
+
+ /* Adata case. */
+
+ /*
+ * Form the encoding of the Associated data length and write it
+ * to the AES/SM4 input buffer.
+ */
+ ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
+
+ /* Configure the AES/SM4 DMA to fetch the Associated Data */
+ dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
+
+ /* Activate DMA to fetch Associated data. */
+ aes_a_dma_active_src_ll_en(aes_dev);
+
+ /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */
+ aes_a_set_last_gcx_and_adata(aes_dev);
+
+ /* Wait for DMA transfer to complete. */
+ rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
+ if (rc)
+ return rc;
+
+exit:
+ /* Wait until adata (if present) has been processed. */
+ aes_a_wait_last_gcx(aes_dev);
+ aes_a_dma_wait_input_buffer_occupancy(aes_dev);
+
+ return 0;
+}
+
+static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size)
+{
+ if (src_size) {
+ /*
+ * Configure and activate DMA for both input and output
+ * data.
+ */
+ dma_to_ocs_aes_ll(aes_dev, src_dma_list);
+ dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
+ aes_a_dma_active_src_dst_ll_en(aes_dev);
+ } else {
+ /* Configure and activate DMA for output data only. */
+ dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
+ aes_a_dma_active_dst_ll_en(aes_dev);
+ }
+
+ /*
+ * Set the LAST GCX bit in AES_ACTIVE Register to instruct
+ * AES/SM4 engine to pad the last block of data.
+ */
+ aes_a_set_last_gcx(aes_dev);
+
+ /* We are done, wait for IRQ and return. */
+ return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
+}
+
+static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size)
+{
+ if (!src_size) {
+ /* Let engine process 0-length input. */
+ aes_a_dma_set_xfer_size_zero(aes_dev);
+ aes_a_dma_active(aes_dev);
+ aes_a_set_last_gcx(aes_dev);
+
+ return 0;
+ }
+
+ /*
+ * Configure and activate DMA for both input and output
+ * data.
+ */
+ dma_to_ocs_aes_ll(aes_dev, src_dma_list);
+ dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
+ aes_a_dma_active_src_dst_ll_en(aes_dev);
+ /*
+ * Set the LAST GCX bit in AES_ACTIVE Register; this allows the
+ * AES/SM4 engine to differentiate between encrypted data and
+ * encrypted MAC.
+ */
+ aes_a_set_last_gcx(aes_dev);
+ /*
+ * Enable DMA DONE interrupt; once DMA transfer is over,
+ * interrupt handler will process the MAC/tag.
+ */
+ return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
+}
+
+/*
+ * Compare Tag to Yr.
+ *
+ * Only used at the end of CCM decrypt. If tag == yr, message authentication
+ * has succeeded.
+ */
+static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
+ u8 tag_size_bytes)
+{
+ u32 tag[AES_MAX_TAG_SIZE_U32];
+ u32 yr[AES_MAX_TAG_SIZE_U32];
+ u8 i;
+
+ /* Read Tag and Yr from AES registers. */
+ for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
+ tag[i] = ioread32(aes_dev->base_reg +
+ AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
+ yr[i] = ioread32(aes_dev->base_reg +
+ AES_MULTIPURPOSE2_0_OFFSET +
+ (i * sizeof(u32)));
+ }
+
+ return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
+}
+
+/**
+ * ocs_aes_ccm_op() - Perform CCM operation.
+ * @aes_dev: The OCS AES device to use.
+ * @cipher: The Cipher to use (AES or SM4).
+ * @instruction: The instruction to perform (encrypt or decrypt).
+ * @dst_dma_list: The OCS DMA list mapping output memory.
+ * @src_dma_list: The OCS DMA list mapping input payload data.
+ * @src_size: The amount of data mapped by @src_dma_list.
+ * @iv: The input IV vector.
+ * @adata_dma_list: The OCS DMA list mapping input A-data.
+ * @adata_size: The amount of data mapped by @adata_dma_list.
+ * @in_tag: Input tag.
+ * @tag_size: The size (in bytes) of @in_tag.
+ *
+ * Note: for encrypt the tag is appended to the ciphertext (in the memory
+ * mapped by @dst_dma_list).
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ u8 *iv,
+ dma_addr_t adata_dma_list,
+ u32 adata_size,
+ u8 *in_tag,
+ u32 tag_size)
+{
+ u32 *iv_32;
+ u8 lprime;
+ int rc;
+
+ rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
+ AES_BLOCK_SIZE, adata_dma_list, adata_size,
+ in_tag, tag_size, cipher, OCS_MODE_CCM,
+ instruction, dst_dma_list);
+ if (rc)
+ return rc;
+
+ ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
+
+ /*
+ * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
+ * auth tag so ensure this is the case
+ */
+ lprime = iv[L_PRIME_IDX];
+ memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
+
+ /*
+ * Nonce is already converted to ctr0 before being passed into this
+ * function as iv.
+ */
+ iv_32 = (u32 *)iv;
+ iowrite32(__swab32(iv_32[0]),
+ aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
+ iowrite32(__swab32(iv_32[1]),
+ aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
+ iowrite32(__swab32(iv_32[2]),
+ aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
+ iowrite32(__swab32(iv_32[3]),
+ aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
+
+ /* Write MAC/tag length in register AES_TLEN */
+ iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
+ /*
+ * Write the byte length of the last AES/SM4 block of Payload data
+ * (without zero padding and without the length of the MAC) in register
+ * AES_PLEN.
+ */
+ ocs_aes_write_last_data_blk_len(aes_dev, src_size);
+
+ /* Set AES_ACTIVE.TRIGGER to start the operation. */
+ aes_a_op_trigger(aes_dev);
+
+ aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
+
+ /* Form block B0 and write it to the AES/SM4 input buffer. */
+ rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
+ if (rc)
+ return rc;
+ /*
+ * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT
+ * clock cycles since TRIGGER bit was set
+ */
+ aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
+ CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
+
+ /* Process Adata. */
+ ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
+
+ /* For Encrypt case we just process the payload and return. */
+ if (instruction == OCS_ENCRYPT) {
+ return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
+ src_dma_list, src_size);
+ }
+ /* For Decypt we need to process the payload and then the tag. */
+ rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
+ src_dma_list, src_size);
+ if (rc)
+ return rc;
+
+ /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
+ ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
+ rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
+ if (rc)
+ return rc;
+
+ return ccm_compare_tag_to_yr(aes_dev, tag_size);
+}
+
+/**
+ * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
+ * @aes_dev: The OCS AES device the list will be created for.
+ * @sg: The SG list OCS DMA linked list will be created from. When
+ * passed to this function, @sg must have been already mapped
+ * with dma_map_sg().
+ * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
+ * value returned by dma_map_sg() when @sg was mapped.
+ * @dll_desc: The OCS DMA dma_list to use to store information about the
+ * created linked list.
+ * @data_size: The size of the data (from the SG list) to be mapped into the
+ * OCS DMA linked list.
+ * @data_offset: The offset (within the SG list) of the data to be mapped.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
+ struct scatterlist *sg,
+ int sg_dma_count,
+ struct ocs_dll_desc *dll_desc,
+ size_t data_size, size_t data_offset)
+{
+ struct ocs_dma_linked_list *ll = NULL;
+ struct scatterlist *sg_tmp;
+ unsigned int tmp;
+ int dma_nents;
+ int i;
+
+ if (!dll_desc || !sg || !aes_dev)
+ return -EINVAL;
+
+ /* Default values for when no ddl_desc is created. */
+ dll_desc->vaddr = NULL;
+ dll_desc->dma_addr = DMA_MAPPING_ERROR;
+ dll_desc->size = 0;
+
+ if (data_size == 0)
+ return 0;
+
+ /* Loop over sg_list until we reach entry at specified offset. */
+ while (data_offset >= sg_dma_len(sg)) {
+ data_offset -= sg_dma_len(sg);
+ sg_dma_count--;
+ sg = sg_next(sg);
+ /* If we reach the end of the list, offset was invalid. */
+ if (!sg || sg_dma_count == 0)
+ return -EINVAL;
+ }
+
+ /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
+ dma_nents = 0;
+ tmp = 0;
+ sg_tmp = sg;
+ while (tmp < data_offset + data_size) {
+ /* If we reach the end of the list, data_size was invalid. */
+ if (!sg_tmp)
+ return -EINVAL;
+ tmp += sg_dma_len(sg_tmp);
+ dma_nents++;
+ sg_tmp = sg_next(sg_tmp);
+ }
+ if (dma_nents > sg_dma_count)
+ return -EINVAL;
+
+ /* Allocate the DMA list, one entry for each SG entry. */
+ dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
+ dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
+ &dll_desc->dma_addr, GFP_KERNEL);
+ if (!dll_desc->vaddr)
+ return -ENOMEM;
+
+ /* Populate DMA linked list entries. */
+ ll = dll_desc->vaddr;
+ for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
+ ll[i].src_addr = sg_dma_address(sg) + data_offset;
+ ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
+ (sg_dma_len(sg) - data_offset) : data_size;
+ data_offset = 0;
+ data_size -= ll[i].src_len;
+ /* Current element points to the DMA address of the next one. */
+ ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
+ ll[i].ll_flags = 0;
+ }
+ /* Terminate last element. */
+ ll[i - 1].next = 0;
+ ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
+
+ return 0;
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Intel Keem Bay OCS AES Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#ifndef _CRYPTO_OCS_AES_H
+#define _CRYPTO_OCS_AES_H
+
+#include <linux/dma-mapping.h>
+
+enum ocs_cipher {
+ OCS_AES = 0,
+ OCS_SM4 = 1,
+};
+
+enum ocs_mode {
+ OCS_MODE_ECB = 0,
+ OCS_MODE_CBC = 1,
+ OCS_MODE_CTR = 2,
+ OCS_MODE_CCM = 6,
+ OCS_MODE_GCM = 7,
+ OCS_MODE_CTS = 9,
+};
+
+enum ocs_instruction {
+ OCS_ENCRYPT = 0,
+ OCS_DECRYPT = 1,
+ OCS_EXPAND = 2,
+ OCS_BYPASS = 3,
+};
+
+/**
+ * struct ocs_aes_dev - AES device context.
+ * @list: List head for insertion into device list hold
+ * by driver.
+ * @dev: OCS AES device.
+ * @irq: IRQ number.
+ * @base_reg: IO base address of OCS AES.
+ * @irq_copy_completion: Completion to indicate IRQ has been triggered.
+ * @dma_err_mask: Error reported by OCS DMA interrupts.
+ * @engine: Crypto engine for the device.
+ */
+struct ocs_aes_dev {
+ struct list_head list;
+ struct device *dev;
+ int irq;
+ void __iomem *base_reg;
+ struct completion irq_completion;
+ u32 dma_err_mask;
+ struct crypto_engine *engine;
+};
+
+/**
+ * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List.
+ * @vaddr: Virtual address of the linked list head.
+ * @dma_addr: DMA address of the linked list head.
+ * @size: Size (in bytes) of the linked list.
+ */
+struct ocs_dll_desc {
+ void *vaddr;
+ dma_addr_t dma_addr;
+ size_t size;
+};
+
+int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size,
+ const u8 *key, const enum ocs_cipher cipher);
+
+int ocs_aes_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_mode mode,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ u8 *iv,
+ u32 iv_size);
+
+/**
+ * ocs_aes_bypass_op() - Use OCS DMA to copy data.
+ * @aes_dev: The OCS AES device to use.
+ * @dst_dma_list: The OCS DMA list mapping the memory where input data
+ * will be copied to.
+ * @src_dma_list: The OCS DMA list mapping input data.
+ * @src_size: The amount of data to copy.
+ */
+static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list, u32 src_size)
+{
+ return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS,
+ dst_dma_list, src_dma_list, src_size, NULL, 0);
+}
+
+int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ const u8 *iv,
+ dma_addr_t aad_dma_list,
+ u32 aad_size,
+ u8 *out_tag,
+ u32 tag_size);
+
+int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
+ enum ocs_cipher cipher,
+ enum ocs_instruction instruction,
+ dma_addr_t dst_dma_list,
+ dma_addr_t src_dma_list,
+ u32 src_size,
+ u8 *iv,
+ dma_addr_t adata_dma_list,
+ u32 adata_size,
+ u8 *in_tag,
+ u32 tag_size);
+
+int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
+ struct scatterlist *sg,
+ int sg_dma_count,
+ struct ocs_dll_desc *dll_desc,
+ size_t data_size,
+ size_t data_offset);
+
+irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id);
+
+#endif
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Intel Keem Bay OCS HCU Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/irq.h>
+#include <linux/module.h>
+
+#include <crypto/sha2.h>
+
+#include "ocs-hcu.h"
+
+/* Registers. */
+#define OCS_HCU_MODE 0x00
+#define OCS_HCU_CHAIN 0x04
+#define OCS_HCU_OPERATION 0x08
+#define OCS_HCU_KEY_0 0x0C
+#define OCS_HCU_ISR 0x50
+#define OCS_HCU_IER 0x54
+#define OCS_HCU_STATUS 0x58
+#define OCS_HCU_MSG_LEN_LO 0x60
+#define OCS_HCU_MSG_LEN_HI 0x64
+#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80
+#define OCS_HCU_DMA_SRC_ADDR 0x400
+#define OCS_HCU_DMA_SRC_SIZE 0x408
+#define OCS_HCU_DMA_DST_SIZE 0x40C
+#define OCS_HCU_DMA_DMA_MODE 0x410
+#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418
+#define OCS_HCU_DMA_MSI_ISR 0x480
+#define OCS_HCU_DMA_MSI_IER 0x484
+#define OCS_HCU_DMA_MSI_MASK 0x488
+
+/* Register bit definitions. */
+#define HCU_MODE_ALGO_SHIFT 16
+#define HCU_MODE_HMAC_SHIFT 22
+
+#define HCU_STATUS_BUSY BIT(0)
+
+#define HCU_BYTE_ORDER_SWAP BIT(0)
+
+#define HCU_IRQ_HASH_DONE BIT(2)
+#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0))
+
+#define HCU_DMA_IRQ_SRC_DONE BIT(0)
+#define HCU_DMA_IRQ_SAI_ERR BIT(2)
+#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3)
+#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4)
+#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5)
+#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6)
+#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7)
+#define HCU_DMA_IRQ_CRD_ERR BIT(8)
+#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \
+ HCU_DMA_IRQ_BAD_COMP_ERR | \
+ HCU_DMA_IRQ_INBUF_RD_ERR | \
+ HCU_DMA_IRQ_INBUF_WD_ERR | \
+ HCU_DMA_IRQ_OUTBUF_WR_ERR | \
+ HCU_DMA_IRQ_OUTBUF_RD_ERR | \
+ HCU_DMA_IRQ_CRD_ERR)
+
+#define HCU_DMA_SNOOP_MASK (0x7 << 28)
+#define HCU_DMA_SRC_LL_EN BIT(25)
+#define HCU_DMA_EN BIT(31)
+
+#define OCS_HCU_ENDIANNESS_VALUE 0x2A
+
+#define HCU_DMA_MSI_UNMASK BIT(0)
+#define HCU_DMA_MSI_DISABLE 0
+#define HCU_IRQ_DISABLE 0
+
+#define OCS_HCU_START BIT(0)
+#define OCS_HCU_TERMINATE BIT(1)
+
+#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
+
+#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32))
+
+#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26
+
+#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32))
+#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32))
+
+/*
+ * While polling on a busy HCU, wait maximum 200us between one check and the
+ * other.
+ */
+#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200
+/* Wait on a busy HCU for maximum 1 second. */
+#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000
+
+/**
+ * struct ocs_hcu_dma_entry - An entry in an OCS DMA linked list.
+ * @src_addr: Source address of the data.
+ * @src_len: Length of data to be fetched.
+ * @nxt_desc: Next descriptor to fetch.
+ * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
+ */
+struct ocs_hcu_dma_entry {
+ u32 src_addr;
+ u32 src_len;
+ u32 nxt_desc;
+ u32 ll_flags;
+};
+
+/**
+ * struct ocs_hcu_dma_list - OCS-specific DMA linked list.
+ * @head: The head of the list (points to the array backing the list).
+ * @tail: The current tail of the list; NULL if the list is empty.
+ * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing
+ * array).
+ * @max_nents: Maximum number of entries in the list (i.e., number of elements
+ * in the backing array).
+ *
+ * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array
+ * backing the list is allocated with dma_alloc_coherent() and pointed by
+ * @head.
+ */
+struct ocs_hcu_dma_list {
+ struct ocs_hcu_dma_entry *head;
+ struct ocs_hcu_dma_entry *tail;
+ dma_addr_t dma_addr;
+ size_t max_nents;
+};
+
+static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo)
+{
+ switch (algo) {
+ case OCS_HCU_ALGO_SHA224:
+ case OCS_HCU_ALGO_SHA256:
+ case OCS_HCU_ALGO_SM3:
+ return OCS_HCU_NUM_CHAINS_SHA256_224_SM3;
+ case OCS_HCU_ALGO_SHA384:
+ case OCS_HCU_ALGO_SHA512:
+ return OCS_HCU_NUM_CHAINS_SHA384_512;
+ default:
+ return 0;
+ };
+}
+
+static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo)
+{
+ switch (algo) {
+ case OCS_HCU_ALGO_SHA224:
+ return SHA224_DIGEST_SIZE;
+ case OCS_HCU_ALGO_SHA256:
+ case OCS_HCU_ALGO_SM3:
+ /* SM3 shares the same block size. */
+ return SHA256_DIGEST_SIZE;
+ case OCS_HCU_ALGO_SHA384:
+ return SHA384_DIGEST_SIZE;
+ case OCS_HCU_ALGO_SHA512:
+ return SHA512_DIGEST_SIZE;
+ default:
+ return 0;
+ }
+}
+
+/**
+ * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable.
+ * @hcu_dev: OCS HCU device to wait for.
+ *
+ * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has
+ * expired.
+ */
+static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev)
+{
+ long val;
+
+ return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val,
+ !(val & HCU_STATUS_BUSY),
+ OCS_HCU_WAIT_BUSY_RETRY_DELAY_US,
+ OCS_HCU_WAIT_BUSY_TIMEOUT_US);
+}
+
+static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev)
+{
+ /* Clear any pending interrupts. */
+ writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR);
+ hcu_dev->irq_err = false;
+ /* Enable error and HCU done interrupts. */
+ writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK,
+ hcu_dev->io_base + OCS_HCU_IER);
+}
+
+static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev)
+{
+ /* Clear any pending interrupts. */
+ writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
+ hcu_dev->irq_err = false;
+ /* Only operating on DMA source completion and error interrupts. */
+ writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE,
+ hcu_dev->io_base + OCS_HCU_DMA_MSI_IER);
+ /* Unmask */
+ writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK);
+}
+
+static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev)
+{
+ writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER);
+ writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER);
+}
+
+static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev)
+{
+ int rc;
+
+ rc = wait_for_completion_interruptible(&hcu_dev->irq_done);
+ if (rc)
+ goto exit;
+
+ if (hcu_dev->irq_err) {
+ /* Unset flag and return error. */
+ hcu_dev->irq_err = false;
+ rc = -EIO;
+ goto exit;
+ }
+
+exit:
+ ocs_hcu_irq_dis(hcu_dev);
+
+ return rc;
+}
+
+/**
+ * ocs_hcu_get_intermediate_data() - Get intermediate data.
+ * @hcu_dev: The target HCU device.
+ * @data: Where to store the intermediate.
+ * @algo: The algorithm being used.
+ *
+ * This function is used to save the current hashing process state in order to
+ * continue it in the future.
+ *
+ * Note: once all data has been processed, the intermediate data actually
+ * contains the hashing result. So this function is also used to retrieve the
+ * final result of a hashing process.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_idata *data,
+ enum ocs_hcu_algo algo)
+{
+ const int n = ocs_hcu_num_chains(algo);
+ u32 *chain;
+ int rc;
+ int i;
+
+ /* Data not requested. */
+ if (!data)
+ return -EINVAL;
+
+ chain = (u32 *)data->digest;
+
+ /* Ensure that the OCS is no longer busy before reading the chains. */
+ rc = ocs_hcu_wait_busy(hcu_dev);
+ if (rc)
+ return rc;
+
+ /*
+ * This loops is safe because data->digest is an array of
+ * SHA512_DIGEST_SIZE bytes and the maximum value returned by
+ * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal
+ * to SHA512_DIGEST_SIZE / sizeof(u32).
+ */
+ for (i = 0; i < n; i++)
+ chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN);
+
+ data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO);
+ data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI);
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_set_intermediate_data() - Set intermediate data.
+ * @hcu_dev: The target HCU device.
+ * @data: The intermediate data to be set.
+ * @algo: The algorithm being used.
+ *
+ * This function is used to continue a previous hashing process.
+ */
+static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_idata *data,
+ enum ocs_hcu_algo algo)
+{
+ const int n = ocs_hcu_num_chains(algo);
+ u32 *chain = (u32 *)data->digest;
+ int i;
+
+ /*
+ * This loops is safe because data->digest is an array of
+ * SHA512_DIGEST_SIZE bytes and the maximum value returned by
+ * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal
+ * to SHA512_DIGEST_SIZE / sizeof(u32).
+ */
+ for (i = 0; i < n; i++)
+ writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN);
+
+ writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO);
+ writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI);
+}
+
+static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev,
+ enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len)
+{
+ u32 *chain;
+ int rc;
+ int i;
+
+ if (!dgst)
+ return -EINVAL;
+
+ /* Length of the output buffer must match the algo digest size. */
+ if (dgst_len != ocs_hcu_digest_size(algo))
+ return -EINVAL;
+
+ /* Ensure that the OCS is no longer busy before reading the chains. */
+ rc = ocs_hcu_wait_busy(hcu_dev);
+ if (rc)
+ return rc;
+
+ chain = (u32 *)dgst;
+ for (i = 0; i < dgst_len / sizeof(u32); i++)
+ chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN);
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_hw_cfg() - Configure the HCU hardware.
+ * @hcu_dev: The HCU device to configure.
+ * @algo: The algorithm to be used by the HCU device.
+ * @use_hmac: Whether or not HW HMAC should be used.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
+ bool use_hmac)
+{
+ u32 cfg;
+ int rc;
+
+ if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 &&
+ algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 &&
+ algo != OCS_HCU_ALGO_SM3)
+ return -EINVAL;
+
+ rc = ocs_hcu_wait_busy(hcu_dev);
+ if (rc)
+ return rc;
+
+ /* Ensure interrupts are disabled. */
+ ocs_hcu_irq_dis(hcu_dev);
+
+ /* Configure endianness, hashing algorithm and HW HMAC (if needed) */
+ cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET;
+ cfg |= algo << HCU_MODE_ALGO_SHIFT;
+ if (use_hmac)
+ cfg |= BIT(HCU_MODE_HMAC_SHIFT);
+
+ writel(cfg, hcu_dev->io_base + OCS_HCU_MODE);
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers.
+ * @hcu_dev: The OCS HCU device whose key registers should be cleared.
+ */
+static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev)
+{
+ int reg_off;
+
+ /* Clear OCS_HCU_KEY_[0..15] */
+ for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32))
+ writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off);
+}
+
+/**
+ * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers.
+ * @hcu_dev: The OCS HCU device the key should be written to.
+ * @key: The key to be written.
+ * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len)
+{
+ u32 key_u32[OCS_HCU_HW_KEY_LEN_U32];
+ int i;
+
+ if (len > OCS_HCU_HW_KEY_LEN)
+ return -EINVAL;
+
+ /* Copy key into temporary u32 array. */
+ memcpy(key_u32, key, len);
+
+ /*
+ * Hardware requires all the bytes of the HW Key vector to be
+ * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN.
+ */
+ memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len);
+
+ /*
+ * OCS hardware expects the MSB of the key to be written at the highest
+ * address of the HCU Key vector; in other word, the key must be
+ * written in reverse order.
+ *
+ * Therefore, we first enable byte swapping for the HCU key vector;
+ * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be
+ * swapped:
+ * 3 <---> 0, 2 <---> 1.
+ */
+ writel(HCU_BYTE_ORDER_SWAP,
+ hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG);
+ /*
+ * And then we write the 32-bit words composing the key starting from
+ * the end of the key.
+ */
+ for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++)
+ writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i],
+ hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i));
+
+ memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN);
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA
+ * @hcu_dev: The OCS HCU device to use.
+ * @dma_list: The OCS DMA list mapping the data to hash.
+ * @finalize: Whether or not this is the last hashing operation and therefore
+ * the final hash should be compute even if data is not
+ * block-aligned.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_dma_list *dma_list,
+ bool finalize)
+{
+ u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN;
+ int rc;
+
+ if (!dma_list)
+ return -EINVAL;
+
+ /*
+ * For final requests we use HCU_DONE IRQ to be notified when all input
+ * data has been processed by the HCU; however, we cannot do so for
+ * non-final requests, because we don't get a HCU_DONE IRQ when we
+ * don't terminate the operation.
+ *
+ * Therefore, for non-final requests, we use the DMA IRQ, which
+ * triggers when DMA has finishing feeding all the input data to the
+ * HCU, but the HCU may still be processing it. This is fine, since we
+ * will wait for the HCU processing to be completed when we try to read
+ * intermediate results, in ocs_hcu_get_intermediate_data().
+ */
+ if (finalize)
+ ocs_hcu_done_irq_en(hcu_dev);
+ else
+ ocs_hcu_dma_irq_en(hcu_dev);
+
+ reinit_completion(&hcu_dev->irq_done);
+ writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR);
+ writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE);
+ writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE);
+
+ writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION);
+
+ writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE);
+
+ if (finalize)
+ writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
+
+ rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
+ if (rc)
+ return rc;
+
+ return 0;
+}
+
+struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev,
+ int max_nents)
+{
+ struct ocs_hcu_dma_list *dma_list;
+
+ dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL);
+ if (!dma_list)
+ return NULL;
+
+ /* Total size of the DMA list to allocate. */
+ dma_list->head = dma_alloc_coherent(hcu_dev->dev,
+ sizeof(*dma_list->head) * max_nents,
+ &dma_list->dma_addr, GFP_KERNEL);
+ if (!dma_list->head) {
+ kfree(dma_list);
+ return NULL;
+ }
+ dma_list->max_nents = max_nents;
+ dma_list->tail = NULL;
+
+ return dma_list;
+}
+
+void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_dma_list *dma_list)
+{
+ if (!dma_list)
+ return;
+
+ dma_free_coherent(hcu_dev->dev,
+ sizeof(*dma_list->head) * dma_list->max_nents,
+ dma_list->head, dma_list->dma_addr);
+
+ kfree(dma_list);
+}
+
+/* Add a new DMA entry at the end of the OCS DMA list. */
+int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_dma_list *dma_list,
+ dma_addr_t addr, u32 len)
+{
+ struct device *dev = hcu_dev->dev;
+ struct ocs_hcu_dma_entry *old_tail;
+ struct ocs_hcu_dma_entry *new_tail;
+
+ if (!len)
+ return 0;
+
+ if (!dma_list)
+ return -EINVAL;
+
+ if (addr & ~OCS_HCU_DMA_BIT_MASK) {
+ dev_err(dev,
+ "Unexpected error: Invalid DMA address for OCS HCU\n");
+ return -EINVAL;
+ }
+
+ old_tail = dma_list->tail;
+ new_tail = old_tail ? old_tail + 1 : dma_list->head;
+
+ /* Check if list is full. */
+ if (new_tail - dma_list->head >= dma_list->max_nents)
+ return -ENOMEM;
+
+ /*
+ * If there was an old tail (i.e., this is not the first element we are
+ * adding), un-terminate the old tail and make it point to the new one.
+ */
+ if (old_tail) {
+ old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE;
+ /*
+ * The old tail 'nxt_desc' must point to the DMA address of the
+ * new tail.
+ */
+ old_tail->nxt_desc = dma_list->dma_addr +
+ sizeof(*dma_list->tail) * (new_tail -
+ dma_list->head);
+ }
+
+ new_tail->src_addr = (u32)addr;
+ new_tail->src_len = (u32)len;
+ new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
+ new_tail->nxt_desc = 0;
+
+ /* Update list tail with new tail. */
+ dma_list->tail = new_tail;
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_hash_init() - Initialize hash operation context.
+ * @ctx: The context to initialize.
+ * @algo: The hashing algorithm to use.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo)
+{
+ if (!ctx)
+ return -EINVAL;
+
+ ctx->algo = algo;
+ ctx->idata.msg_len_lo = 0;
+ ctx->idata.msg_len_hi = 0;
+ /* No need to set idata.digest to 0. */
+
+ return 0;
+}
+
+/**
+ * ocs_hcu_hash_update() - Perform a hashing iteration.
+ * @hcu_dev: The OCS HCU device to use.
+ * @ctx: The OCS HCU hashing context.
+ * @dma_list: The OCS DMA list mapping the input data to process.
+ *
+ * Return: 0 on success; negative error code otherwise.
+ */
+int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_hash_ctx *ctx,
+ const struct ocs_hcu_dma_list *dma_list)
+{
+ int rc;
+
+ if (!hcu_dev || !ctx)
+ return -EINVAL;
+
+ /* Configure the hardware for the current request. */
+ rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
+ if (rc)
+ return rc;
+
+ /* If we already processed some data, idata needs to be set. */
+ if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
+ ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
+
+ /* Start linked-list DMA hashing. */
+ rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false);
+ if (rc)
+ return rc;
+
+ /* Update idata and return. */
+ return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
+}
+
+/**
+ * ocs_hcu_hash_finup() - Update and finalize hash computation.
+ * @hcu_dev: The OCS HCU device to use.
+ * @ctx: The OCS HCU hashing context.
+ * @dma_list: The OCS DMA list mapping the input data to process.
+ * @dgst: The buffer where to save the computed digest.
+ * @dgst_len: The length of @dgst.
+ *
+ * Return: 0 on success; negative error code otherwise.
+ */
+int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_hash_ctx *ctx,
+ const struct ocs_hcu_dma_list *dma_list,
+ u8 *dgst, size_t dgst_len)
+{
+ int rc;
+
+ if (!hcu_dev || !ctx)
+ return -EINVAL;
+
+ /* Configure the hardware for the current request. */
+ rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
+ if (rc)
+ return rc;
+
+ /* If we already processed some data, idata needs to be set. */
+ if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
+ ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
+
+ /* Start linked-list DMA hashing. */
+ rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true);
+ if (rc)
+ return rc;
+
+ /* Get digest and return. */
+ return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len);
+}
+
+/**
+ * ocs_hcu_hash_final() - Finalize hash computation.
+ * @hcu_dev: The OCS HCU device to use.
+ * @ctx: The OCS HCU hashing context.
+ * @dgst: The buffer where to save the computed digest.
+ * @dgst_len: The length of @dgst.
+ *
+ * Return: 0 on success; negative error code otherwise.
+ */
+int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_hash_ctx *ctx, u8 *dgst,
+ size_t dgst_len)
+{
+ int rc;
+
+ if (!hcu_dev || !ctx)
+ return -EINVAL;
+
+ /* Configure the hardware for the current request. */
+ rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
+ if (rc)
+ return rc;
+
+ /* If we already processed some data, idata needs to be set. */
+ if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
+ ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
+
+ /*
+ * Enable HCU interrupts, so that HCU_DONE will be triggered once the
+ * final hash is computed.
+ */
+ ocs_hcu_done_irq_en(hcu_dev);
+ reinit_completion(&hcu_dev->irq_done);
+ writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
+
+ rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
+ if (rc)
+ return rc;
+
+ /* Get digest and return. */
+ return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len);
+}
+
+/**
+ * ocs_hcu_digest() - Compute hash digest.
+ * @hcu_dev: The OCS HCU device to use.
+ * @algo: The hash algorithm to use.
+ * @data: The input data to process.
+ * @data_len: The length of @data.
+ * @dgst: The buffer where to save the computed digest.
+ * @dgst_len: The length of @dgst.
+ *
+ * Return: 0 on success; negative error code otherwise.
+ */
+int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
+ void *data, size_t data_len, u8 *dgst, size_t dgst_len)
+{
+ struct device *dev = hcu_dev->dev;
+ dma_addr_t dma_handle;
+ u32 reg;
+ int rc;
+
+ /* Configure the hardware for the current request. */
+ rc = ocs_hcu_hw_cfg(hcu_dev, algo, false);
+ if (rc)
+ return rc;
+
+ dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE);
+ if (dma_mapping_error(dev, dma_handle))
+ return -EIO;
+
+ reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN;
+
+ ocs_hcu_done_irq_en(hcu_dev);
+
+ reinit_completion(&hcu_dev->irq_done);
+
+ writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR);
+ writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE);
+ writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION);
+ writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE);
+
+ writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
+
+ rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
+ if (rc)
+ return rc;
+
+ dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE);
+
+ return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len);
+}
+
+/**
+ * ocs_hcu_hmac() - Compute HMAC.
+ * @hcu_dev: The OCS HCU device to use.
+ * @algo: The hash algorithm to use with HMAC.
+ * @key: The key to use.
+ * @dma_list: The OCS DMA list mapping the input data to process.
+ * @key_len: The length of @key.
+ * @dgst: The buffer where to save the computed HMAC.
+ * @dgst_len: The length of @dgst.
+ *
+ * Return: 0 on success; negative error code otherwise.
+ */
+int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
+ const u8 *key, size_t key_len,
+ const struct ocs_hcu_dma_list *dma_list,
+ u8 *dgst, size_t dgst_len)
+{
+ int rc;
+
+ /* Ensure 'key' is not NULL. */
+ if (!key || key_len == 0)
+ return -EINVAL;
+
+ /* Configure the hardware for the current request. */
+ rc = ocs_hcu_hw_cfg(hcu_dev, algo, true);
+ if (rc)
+ return rc;
+
+ rc = ocs_hcu_write_key(hcu_dev, key, key_len);
+ if (rc)
+ return rc;
+
+ rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true);
+
+ /* Clear HW key before processing return code. */
+ ocs_hcu_clear_key(hcu_dev);
+
+ if (rc)
+ return rc;
+
+ return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len);
+}
+
+irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id)
+{
+ struct ocs_hcu_dev *hcu_dev = dev_id;
+ u32 hcu_irq;
+ u32 dma_irq;
+
+ /* Read and clear the HCU interrupt. */
+ hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR);
+ writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR);
+
+ /* Read and clear the HCU DMA interrupt. */
+ dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
+ writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
+
+ /* Check for errors. */
+ if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) {
+ hcu_dev->irq_err = true;
+ goto complete;
+ }
+
+ /* Check for DONE IRQs. */
+ if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE)
+ goto complete;
+
+ return IRQ_NONE;
+
+complete:
+ complete(&hcu_dev->irq_done);
+
+ return IRQ_HANDLED;
+}
+
+MODULE_LICENSE("GPL");
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Intel Keem Bay OCS HCU Crypto Driver.
+ *
+ * Copyright (C) 2018-2020 Intel Corporation
+ */
+
+#include <linux/dma-mapping.h>
+
+#ifndef _CRYPTO_OCS_HCU_H
+#define _CRYPTO_OCS_HCU_H
+
+#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32)
+
+#define OCS_HCU_HW_KEY_LEN 64
+
+struct ocs_hcu_dma_list;
+
+enum ocs_hcu_algo {
+ OCS_HCU_ALGO_SHA256 = 2,
+ OCS_HCU_ALGO_SHA224 = 3,
+ OCS_HCU_ALGO_SHA384 = 4,
+ OCS_HCU_ALGO_SHA512 = 5,
+ OCS_HCU_ALGO_SM3 = 6,
+};
+
+/**
+ * struct ocs_hcu_dev - OCS HCU device context.
+ * @list: List of device contexts.
+ * @dev: OCS HCU device.
+ * @io_base: Base address of OCS HCU registers.
+ * @engine: Crypto engine for the device.
+ * @irq: IRQ number.
+ * @irq_done: Completion for IRQ.
+ * @irq_err: Flag indicating an IRQ error has happened.
+ */
+struct ocs_hcu_dev {
+ struct list_head list;
+ struct device *dev;
+ void __iomem *io_base;
+ struct crypto_engine *engine;
+ int irq;
+ struct completion irq_done;
+ bool irq_err;
+};
+
+/**
+ * struct ocs_hcu_idata - Intermediate data generated by the HCU.
+ * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b.
+ * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b.
+ * @digest: The digest read from the HCU. If the HCU is terminated, it will
+ * contain the actual hash digest. Otherwise it is the intermediate
+ * state.
+ */
+struct ocs_hcu_idata {
+ u32 msg_len_lo;
+ u32 msg_len_hi;
+ u8 digest[SHA512_DIGEST_SIZE];
+};
+
+/**
+ * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation.
+ * @algo: The hashing algorithm being used.
+ * @idata: The current intermediate data.
+ */
+struct ocs_hcu_hash_ctx {
+ enum ocs_hcu_algo algo;
+ struct ocs_hcu_idata idata;
+};
+
+irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id);
+
+struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev,
+ int max_nents);
+
+void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_dma_list *dma_list);
+
+int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_dma_list *dma_list,
+ dma_addr_t addr, u32 len);
+
+int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo);
+
+int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev,
+ struct ocs_hcu_hash_ctx *ctx,
+ const struct ocs_hcu_dma_list *dma_list);
+
+int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_hash_ctx *ctx,
+ const struct ocs_hcu_dma_list *dma_list,
+ u8 *dgst, size_t dgst_len);
+
+int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev,
+ const struct ocs_hcu_hash_ctx *ctx, u8 *dgst,
+ size_t dgst_len);
+
+int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
+ void *data, size_t data_len, u8 *dgst, size_t dgst_len);
+
+int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
+ const u8 *key, size_t key_len,
+ const struct ocs_hcu_dma_list *dma_list,
+ u8 *dgst, size_t dgst_len);
+
+#endif /* _CRYPTO_OCS_HCU_H */
+++ /dev/null
-config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
- tristate "Support for Intel Keem Bay OCS AES/SM4 HW acceleration"
- depends on HAS_IOMEM
- depends on ARCH_KEEMBAY || COMPILE_TEST
- select CRYPTO_SKCIPHER
- select CRYPTO_AEAD
- select CRYPTO_ENGINE
- help
- Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) AES and
- SM4 cipher hardware acceleration for use with Crypto API.
-
- Provides HW acceleration for the following transformations:
- cbc(aes), ctr(aes), ccm(aes), gcm(aes), cbc(sm4), ctr(sm4), ccm(sm4)
- and gcm(sm4).
-
- Optionally, support for the following transformations can also be
- enabled: ecb(aes), cts(cbc(aes)), ecb(sm4) and cts(cbc(sm4)).
-
-config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
- bool "Support for Intel Keem Bay OCS AES/SM4 ECB HW acceleration"
- depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
- help
- Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
- AES/SM4 ECB mode hardware acceleration for use with Crypto API.
-
- Provides OCS version of ecb(aes) and ecb(sm4)
-
- Intel does not recommend use of ECB mode with AES/SM4.
-
-config CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
- bool "Support for Intel Keem Bay OCS AES/SM4 CTS HW acceleration"
- depends on CRYPTO_DEV_KEEMBAY_OCS_AES_SM4
- help
- Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
- AES/SM4 CBC with CTS mode hardware acceleration for use with
- Crypto API.
-
- Provides OCS version of cts(cbc(aes)) and cts(cbc(sm4)).
-
- Intel does not recommend use of CTS mode with AES/SM4.
-
-config CRYPTO_DEV_KEEMBAY_OCS_ECC
- tristate "Support for Intel Keem Bay OCS ECC HW acceleration"
- depends on ARCH_KEEMBAY || COMPILE_TEST
- depends on OF
- depends on HAS_IOMEM
- select CRYPTO_ECDH
- select CRYPTO_ENGINE
- help
- Support for Intel Keem Bay Offload and Crypto Subsystem (OCS)
- Elliptic Curve Cryptography (ECC) hardware acceleration for use with
- Crypto API.
-
- Provides OCS acceleration for ECDH-256 and ECDH-384.
-
- Say Y or M if you are compiling for the Intel Keem Bay SoC. The
- module will be called keembay-ocs-ecc.
-
- If unsure, say N.
-
-config CRYPTO_DEV_KEEMBAY_OCS_HCU
- tristate "Support for Intel Keem Bay OCS HCU HW acceleration"
- select CRYPTO_HASH
- select CRYPTO_ENGINE
- depends on HAS_IOMEM
- depends on ARCH_KEEMBAY || COMPILE_TEST
- depends on OF
- help
- Support for Intel Keem Bay Offload and Crypto Subsystem (OCS) Hash
- Control Unit (HCU) hardware acceleration for use with Crypto API.
-
- Provides OCS HCU hardware acceleration of sha256, sha384, sha512, and
- sm3, as well as the HMAC variant of these algorithms.
-
- Say Y or M if you're building for the Intel Keem Bay SoC. If compiled
- as a module, the module will be called keembay-ocs-hcu.
-
- If unsure, say N.
-
-config CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
- bool "Enable sha224 and hmac(sha224) support in Intel Keem Bay OCS HCU"
- depends on CRYPTO_DEV_KEEMBAY_OCS_HCU
- help
- Enables support for sha224 and hmac(sha224) algorithms in the Intel
- Keem Bay OCS HCU driver. Intel recommends not to use these
- algorithms.
-
- Provides OCS HCU hardware acceleration of sha224 and hmac(224).
-
- If unsure, say N.
+++ /dev/null
-#
-# Makefile for Intel Keem Bay OCS Crypto API Linux drivers
-#
-obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4) += keembay-ocs-aes.o
-keembay-ocs-aes-objs := keembay-ocs-aes-core.o ocs-aes.o
-
-obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_ECC) += keembay-ocs-ecc.o
-
-obj-$(CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU) += keembay-ocs-hcu.o
-keembay-ocs-hcu-objs := keembay-ocs-hcu-core.o ocs-hcu.o
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Intel Keem Bay OCS AES Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#include <linux/clk.h>
-#include <linux/completion.h>
-#include <linux/crypto.h>
-#include <linux/dma-mapping.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/types.h>
-
-#include <crypto/aes.h>
-#include <crypto/engine.h>
-#include <crypto/gcm.h>
-#include <crypto/scatterwalk.h>
-
-#include <crypto/internal/aead.h>
-#include <crypto/internal/skcipher.h>
-
-#include "ocs-aes.h"
-
-#define KMB_OCS_PRIORITY 350
-#define DRV_NAME "keembay-ocs-aes"
-
-#define OCS_AES_MIN_KEY_SIZE 16
-#define OCS_AES_MAX_KEY_SIZE 32
-#define OCS_AES_KEYSIZE_128 16
-#define OCS_AES_KEYSIZE_192 24
-#define OCS_AES_KEYSIZE_256 32
-#define OCS_SM4_KEY_SIZE 16
-
-/**
- * struct ocs_aes_tctx - OCS AES Transform context
- * @engine_ctx: Engine context.
- * @aes_dev: The OCS AES device.
- * @key: AES/SM4 key.
- * @key_len: The length (in bytes) of @key.
- * @cipher: OCS cipher to use (either AES or SM4).
- * @sw_cipher: The cipher to use as fallback.
- * @use_fallback: Whether or not fallback cipher should be used.
- */
-struct ocs_aes_tctx {
- struct crypto_engine_ctx engine_ctx;
- struct ocs_aes_dev *aes_dev;
- u8 key[OCS_AES_KEYSIZE_256];
- unsigned int key_len;
- enum ocs_cipher cipher;
- union {
- struct crypto_sync_skcipher *sk;
- struct crypto_aead *aead;
- } sw_cipher;
- bool use_fallback;
-};
-
-/**
- * struct ocs_aes_rctx - OCS AES Request context.
- * @instruction: Instruction to be executed (encrypt / decrypt).
- * @mode: Mode to use (ECB, CBC, CTR, CCm, GCM, CTS)
- * @src_nents: Number of source SG entries.
- * @dst_nents: Number of destination SG entries.
- * @src_dma_count: The number of DMA-mapped entries of the source SG.
- * @dst_dma_count: The number of DMA-mapped entries of the destination SG.
- * @in_place: Whether or not this is an in place request, i.e.,
- * src_sg == dst_sg.
- * @src_dll: OCS DMA linked list for input data.
- * @dst_dll: OCS DMA linked list for output data.
- * @last_ct_blk: Buffer to hold last cipher text block (only used in CBC
- * mode).
- * @cts_swap: Whether or not CTS swap must be performed.
- * @aad_src_dll: OCS DMA linked list for input AAD data.
- * @aad_dst_dll: OCS DMA linked list for output AAD data.
- * @in_tag: Buffer to hold input encrypted tag (only used for
- * CCM/GCM decrypt).
- * @out_tag: Buffer to hold output encrypted / decrypted tag (only
- * used for GCM encrypt / decrypt).
- */
-struct ocs_aes_rctx {
- /* Fields common across all modes. */
- enum ocs_instruction instruction;
- enum ocs_mode mode;
- int src_nents;
- int dst_nents;
- int src_dma_count;
- int dst_dma_count;
- bool in_place;
- struct ocs_dll_desc src_dll;
- struct ocs_dll_desc dst_dll;
-
- /* CBC specific */
- u8 last_ct_blk[AES_BLOCK_SIZE];
-
- /* CTS specific */
- int cts_swap;
-
- /* CCM/GCM specific */
- struct ocs_dll_desc aad_src_dll;
- struct ocs_dll_desc aad_dst_dll;
- u8 in_tag[AES_BLOCK_SIZE];
-
- /* GCM specific */
- u8 out_tag[AES_BLOCK_SIZE];
-};
-
-/* Driver data. */
-struct ocs_aes_drv {
- struct list_head dev_list;
- spinlock_t lock; /* Protects dev_list. */
-};
-
-static struct ocs_aes_drv ocs_aes = {
- .dev_list = LIST_HEAD_INIT(ocs_aes.dev_list),
- .lock = __SPIN_LOCK_UNLOCKED(ocs_aes.lock),
-};
-
-static struct ocs_aes_dev *kmb_ocs_aes_find_dev(struct ocs_aes_tctx *tctx)
-{
- struct ocs_aes_dev *aes_dev;
-
- spin_lock(&ocs_aes.lock);
-
- if (tctx->aes_dev) {
- aes_dev = tctx->aes_dev;
- goto exit;
- }
-
- /* Only a single OCS device available */
- aes_dev = list_first_entry(&ocs_aes.dev_list, struct ocs_aes_dev, list);
- tctx->aes_dev = aes_dev;
-
-exit:
- spin_unlock(&ocs_aes.lock);
-
- return aes_dev;
-}
-
-/*
- * Ensure key is 128-bit or 256-bit for AES or 128-bit for SM4 and an actual
- * key is being passed in.
- *
- * Return: 0 if key is valid, -EINVAL otherwise.
- */
-static int check_key(const u8 *in_key, size_t key_len, enum ocs_cipher cipher)
-{
- if (!in_key)
- return -EINVAL;
-
- /* For AES, only 128-byte or 256-byte keys are supported. */
- if (cipher == OCS_AES && (key_len == OCS_AES_KEYSIZE_128 ||
- key_len == OCS_AES_KEYSIZE_256))
- return 0;
-
- /* For SM4, only 128-byte keys are supported. */
- if (cipher == OCS_SM4 && key_len == OCS_AES_KEYSIZE_128)
- return 0;
-
- /* Everything else is unsupported. */
- return -EINVAL;
-}
-
-/* Save key into transformation context. */
-static int save_key(struct ocs_aes_tctx *tctx, const u8 *in_key, size_t key_len,
- enum ocs_cipher cipher)
-{
- int ret;
-
- ret = check_key(in_key, key_len, cipher);
- if (ret)
- return ret;
-
- memcpy(tctx->key, in_key, key_len);
- tctx->key_len = key_len;
- tctx->cipher = cipher;
-
- return 0;
-}
-
-/* Set key for symmetric cypher. */
-static int kmb_ocs_sk_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- size_t key_len, enum ocs_cipher cipher)
-{
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
-
- /* Fallback is used for AES with 192-bit key. */
- tctx->use_fallback = (cipher == OCS_AES &&
- key_len == OCS_AES_KEYSIZE_192);
-
- if (!tctx->use_fallback)
- return save_key(tctx, in_key, key_len, cipher);
-
- crypto_sync_skcipher_clear_flags(tctx->sw_cipher.sk,
- CRYPTO_TFM_REQ_MASK);
- crypto_sync_skcipher_set_flags(tctx->sw_cipher.sk,
- tfm->base.crt_flags &
- CRYPTO_TFM_REQ_MASK);
-
- return crypto_sync_skcipher_setkey(tctx->sw_cipher.sk, in_key, key_len);
-}
-
-/* Set key for AEAD cipher. */
-static int kmb_ocs_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
- size_t key_len, enum ocs_cipher cipher)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
-
- /* Fallback is used for AES with 192-bit key. */
- tctx->use_fallback = (cipher == OCS_AES &&
- key_len == OCS_AES_KEYSIZE_192);
-
- if (!tctx->use_fallback)
- return save_key(tctx, in_key, key_len, cipher);
-
- crypto_aead_clear_flags(tctx->sw_cipher.aead, CRYPTO_TFM_REQ_MASK);
- crypto_aead_set_flags(tctx->sw_cipher.aead,
- crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK);
-
- return crypto_aead_setkey(tctx->sw_cipher.aead, in_key, key_len);
-}
-
-/* Swap two AES blocks in SG lists. */
-static void sg_swap_blocks(struct scatterlist *sgl, unsigned int nents,
- off_t blk1_offset, off_t blk2_offset)
-{
- u8 tmp_buf1[AES_BLOCK_SIZE], tmp_buf2[AES_BLOCK_SIZE];
-
- /*
- * No easy way to copy within sg list, so copy both blocks to temporary
- * buffers first.
- */
- sg_pcopy_to_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk1_offset);
- sg_pcopy_to_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk2_offset);
- sg_pcopy_from_buffer(sgl, nents, tmp_buf1, AES_BLOCK_SIZE, blk2_offset);
- sg_pcopy_from_buffer(sgl, nents, tmp_buf2, AES_BLOCK_SIZE, blk1_offset);
-}
-
-/* Initialize request context to default values. */
-static void ocs_aes_init_rctx(struct ocs_aes_rctx *rctx)
-{
- /* Zero everything. */
- memset(rctx, 0, sizeof(*rctx));
-
- /* Set initial value for DMA addresses. */
- rctx->src_dll.dma_addr = DMA_MAPPING_ERROR;
- rctx->dst_dll.dma_addr = DMA_MAPPING_ERROR;
- rctx->aad_src_dll.dma_addr = DMA_MAPPING_ERROR;
- rctx->aad_dst_dll.dma_addr = DMA_MAPPING_ERROR;
-}
-
-static int kmb_ocs_sk_validate_input(struct skcipher_request *req,
- enum ocs_mode mode)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- int iv_size = crypto_skcipher_ivsize(tfm);
-
- switch (mode) {
- case OCS_MODE_ECB:
- /* Ensure input length is multiple of block size */
- if (req->cryptlen % AES_BLOCK_SIZE != 0)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CBC:
- /* Ensure input length is multiple of block size */
- if (req->cryptlen % AES_BLOCK_SIZE != 0)
- return -EINVAL;
-
- /* Ensure IV is present and block size in length */
- if (!req->iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
- /*
- * NOTE: Since req->cryptlen == 0 case was already handled in
- * kmb_ocs_sk_common(), the above two conditions also guarantee
- * that: cryptlen >= iv_size
- */
- return 0;
-
- case OCS_MODE_CTR:
- /* Ensure IV is present and block size in length */
- if (!req->iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
- return 0;
-
- case OCS_MODE_CTS:
- /* Ensure input length >= block size */
- if (req->cryptlen < AES_BLOCK_SIZE)
- return -EINVAL;
-
- /* Ensure IV is present and block size in length */
- if (!req->iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- return 0;
- default:
- return -EINVAL;
- }
-}
-
-/*
- * Called by encrypt() / decrypt() skcipher functions.
- *
- * Use fallback if needed, otherwise initialize context and enqueue request
- * into engine.
- */
-static int kmb_ocs_sk_common(struct skcipher_request *req,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- enum ocs_mode mode)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- struct ocs_aes_dev *aes_dev;
- int rc;
-
- if (tctx->use_fallback) {
- SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, tctx->sw_cipher.sk);
-
- skcipher_request_set_sync_tfm(subreq, tctx->sw_cipher.sk);
- skcipher_request_set_callback(subreq, req->base.flags, NULL,
- NULL);
- skcipher_request_set_crypt(subreq, req->src, req->dst,
- req->cryptlen, req->iv);
-
- if (instruction == OCS_ENCRYPT)
- rc = crypto_skcipher_encrypt(subreq);
- else
- rc = crypto_skcipher_decrypt(subreq);
-
- skcipher_request_zero(subreq);
-
- return rc;
- }
-
- /*
- * If cryptlen == 0, no processing needed for ECB, CBC and CTR.
- *
- * For CTS continue: kmb_ocs_sk_validate_input() will return -EINVAL.
- */
- if (!req->cryptlen && mode != OCS_MODE_CTS)
- return 0;
-
- rc = kmb_ocs_sk_validate_input(req, mode);
- if (rc)
- return rc;
-
- aes_dev = kmb_ocs_aes_find_dev(tctx);
- if (!aes_dev)
- return -ENODEV;
-
- if (cipher != tctx->cipher)
- return -EINVAL;
-
- ocs_aes_init_rctx(rctx);
- rctx->instruction = instruction;
- rctx->mode = mode;
-
- return crypto_transfer_skcipher_request_to_engine(aes_dev->engine, req);
-}
-
-static void cleanup_ocs_dma_linked_list(struct device *dev,
- struct ocs_dll_desc *dll)
-{
- if (dll->vaddr)
- dma_free_coherent(dev, dll->size, dll->vaddr, dll->dma_addr);
- dll->vaddr = NULL;
- dll->size = 0;
- dll->dma_addr = DMA_MAPPING_ERROR;
-}
-
-static void kmb_ocs_sk_dma_cleanup(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- struct device *dev = tctx->aes_dev->dev;
-
- if (rctx->src_dma_count) {
- dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
- rctx->src_dma_count = 0;
- }
-
- if (rctx->dst_dma_count) {
- dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
- DMA_BIDIRECTIONAL :
- DMA_FROM_DEVICE);
- rctx->dst_dma_count = 0;
- }
-
- /* Clean up OCS DMA linked lists */
- cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
- cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
-}
-
-static int kmb_ocs_sk_prepare_inplace(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- int iv_size = crypto_skcipher_ivsize(tfm);
- int rc;
-
- /*
- * For CBC decrypt, save last block (iv) to last_ct_blk buffer.
- *
- * Note: if we are here, we already checked that cryptlen >= iv_size
- * and iv_size == AES_BLOCK_SIZE (i.e., the size of last_ct_blk); see
- * kmb_ocs_sk_validate_input().
- */
- if (rctx->mode == OCS_MODE_CBC && rctx->instruction == OCS_DECRYPT)
- scatterwalk_map_and_copy(rctx->last_ct_blk, req->src,
- req->cryptlen - iv_size, iv_size, 0);
-
- /* For CTS decrypt, swap last two blocks, if needed. */
- if (rctx->cts_swap && rctx->instruction == OCS_DECRYPT)
- sg_swap_blocks(req->dst, rctx->dst_nents,
- req->cryptlen - AES_BLOCK_SIZE,
- req->cryptlen - (2 * AES_BLOCK_SIZE));
-
- /* src and dst buffers are the same, use bidirectional DMA mapping. */
- rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
- rctx->dst_nents, DMA_BIDIRECTIONAL);
- if (rctx->dst_dma_count == 0) {
- dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
- return -ENOMEM;
- }
-
- /* Create DST linked list */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count, &rctx->dst_dll,
- req->cryptlen, 0);
- if (rc)
- return rc;
- /*
- * If descriptor creation was successful, set the src_dll.dma_addr to
- * the value of dst_dll.dma_addr, as we do in-place AES operation on
- * the src.
- */
- rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
-
- return 0;
-}
-
-static int kmb_ocs_sk_prepare_notinplace(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- int rc;
-
- rctx->src_nents = sg_nents_for_len(req->src, req->cryptlen);
- if (rctx->src_nents < 0)
- return -EBADMSG;
-
- /* Map SRC SG. */
- rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
- rctx->src_nents, DMA_TO_DEVICE);
- if (rctx->src_dma_count == 0) {
- dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
- return -ENOMEM;
- }
-
- /* Create SRC linked list */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
- rctx->src_dma_count, &rctx->src_dll,
- req->cryptlen, 0);
- if (rc)
- return rc;
-
- /* Map DST SG. */
- rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
- rctx->dst_nents, DMA_FROM_DEVICE);
- if (rctx->dst_dma_count == 0) {
- dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
- return -ENOMEM;
- }
-
- /* Create DST linked list */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count, &rctx->dst_dll,
- req->cryptlen, 0);
- if (rc)
- return rc;
-
- /* If this is not a CTS decrypt operation with swapping, we are done. */
- if (!(rctx->cts_swap && rctx->instruction == OCS_DECRYPT))
- return 0;
-
- /*
- * Otherwise, we have to copy src to dst (as we cannot modify src).
- * Use OCS AES bypass mode to copy src to dst via DMA.
- *
- * NOTE: for anything other than small data sizes this is rather
- * inefficient.
- */
- rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->dst_dll.dma_addr,
- rctx->src_dll.dma_addr, req->cryptlen);
- if (rc)
- return rc;
-
- /*
- * Now dst == src, so clean up what we did so far and use in_place
- * logic.
- */
- kmb_ocs_sk_dma_cleanup(req);
- rctx->in_place = true;
-
- return kmb_ocs_sk_prepare_inplace(req);
-}
-
-static int kmb_ocs_sk_run(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_rctx *rctx = skcipher_request_ctx(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- struct ocs_aes_dev *aes_dev = tctx->aes_dev;
- int iv_size = crypto_skcipher_ivsize(tfm);
- int rc;
-
- rctx->dst_nents = sg_nents_for_len(req->dst, req->cryptlen);
- if (rctx->dst_nents < 0)
- return -EBADMSG;
-
- /*
- * If 2 blocks or greater, and multiple of block size swap last two
- * blocks to be compatible with other crypto API CTS implementations:
- * OCS mode uses CBC-CS2, whereas other crypto API implementations use
- * CBC-CS3.
- * CBC-CS2 and CBC-CS3 defined by:
- * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a-add.pdf
- */
- rctx->cts_swap = (rctx->mode == OCS_MODE_CTS &&
- req->cryptlen > AES_BLOCK_SIZE &&
- req->cryptlen % AES_BLOCK_SIZE == 0);
-
- rctx->in_place = (req->src == req->dst);
-
- if (rctx->in_place)
- rc = kmb_ocs_sk_prepare_inplace(req);
- else
- rc = kmb_ocs_sk_prepare_notinplace(req);
-
- if (rc)
- goto error;
-
- rc = ocs_aes_op(aes_dev, rctx->mode, tctx->cipher, rctx->instruction,
- rctx->dst_dll.dma_addr, rctx->src_dll.dma_addr,
- req->cryptlen, req->iv, iv_size);
- if (rc)
- goto error;
-
- /* Clean-up DMA before further processing output. */
- kmb_ocs_sk_dma_cleanup(req);
-
- /* For CTS Encrypt, swap last 2 blocks, if needed. */
- if (rctx->cts_swap && rctx->instruction == OCS_ENCRYPT) {
- sg_swap_blocks(req->dst, rctx->dst_nents,
- req->cryptlen - AES_BLOCK_SIZE,
- req->cryptlen - (2 * AES_BLOCK_SIZE));
- return 0;
- }
-
- /* For CBC copy IV to req->IV. */
- if (rctx->mode == OCS_MODE_CBC) {
- /* CBC encrypt case. */
- if (rctx->instruction == OCS_ENCRYPT) {
- scatterwalk_map_and_copy(req->iv, req->dst,
- req->cryptlen - iv_size,
- iv_size, 0);
- return 0;
- }
- /* CBC decrypt case. */
- if (rctx->in_place)
- memcpy(req->iv, rctx->last_ct_blk, iv_size);
- else
- scatterwalk_map_and_copy(req->iv, req->src,
- req->cryptlen - iv_size,
- iv_size, 0);
- return 0;
- }
- /* For all other modes there's nothing to do. */
-
- return 0;
-
-error:
- kmb_ocs_sk_dma_cleanup(req);
-
- return rc;
-}
-
-static int kmb_ocs_aead_validate_input(struct aead_request *req,
- enum ocs_instruction instruction,
- enum ocs_mode mode)
-{
- struct crypto_aead *tfm = crypto_aead_reqtfm(req);
- int tag_size = crypto_aead_authsize(tfm);
- int iv_size = crypto_aead_ivsize(tfm);
-
- /* For decrypt crytplen == len(PT) + len(tag). */
- if (instruction == OCS_DECRYPT && req->cryptlen < tag_size)
- return -EINVAL;
-
- /* IV is mandatory. */
- if (!req->iv)
- return -EINVAL;
-
- switch (mode) {
- case OCS_MODE_GCM:
- if (iv_size != GCM_AES_IV_SIZE)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CCM:
- /* Ensure IV is present and block size in length */
- if (iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- return 0;
-
- default:
- return -EINVAL;
- }
-}
-
-/*
- * Called by encrypt() / decrypt() aead functions.
- *
- * Use fallback if needed, otherwise initialize context and enqueue request
- * into engine.
- */
-static int kmb_ocs_aead_common(struct aead_request *req,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- enum ocs_mode mode)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- struct ocs_aes_rctx *rctx = aead_request_ctx(req);
- struct ocs_aes_dev *dd;
- int rc;
-
- if (tctx->use_fallback) {
- struct aead_request *subreq = aead_request_ctx(req);
-
- aead_request_set_tfm(subreq, tctx->sw_cipher.aead);
- aead_request_set_callback(subreq, req->base.flags,
- req->base.complete, req->base.data);
- aead_request_set_crypt(subreq, req->src, req->dst,
- req->cryptlen, req->iv);
- aead_request_set_ad(subreq, req->assoclen);
- rc = crypto_aead_setauthsize(tctx->sw_cipher.aead,
- crypto_aead_authsize(crypto_aead_reqtfm(req)));
- if (rc)
- return rc;
-
- return (instruction == OCS_ENCRYPT) ?
- crypto_aead_encrypt(subreq) :
- crypto_aead_decrypt(subreq);
- }
-
- rc = kmb_ocs_aead_validate_input(req, instruction, mode);
- if (rc)
- return rc;
-
- dd = kmb_ocs_aes_find_dev(tctx);
- if (!dd)
- return -ENODEV;
-
- if (cipher != tctx->cipher)
- return -EINVAL;
-
- ocs_aes_init_rctx(rctx);
- rctx->instruction = instruction;
- rctx->mode = mode;
-
- return crypto_transfer_aead_request_to_engine(dd->engine, req);
-}
-
-static void kmb_ocs_aead_dma_cleanup(struct aead_request *req)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- struct ocs_aes_rctx *rctx = aead_request_ctx(req);
- struct device *dev = tctx->aes_dev->dev;
-
- if (rctx->src_dma_count) {
- dma_unmap_sg(dev, req->src, rctx->src_nents, DMA_TO_DEVICE);
- rctx->src_dma_count = 0;
- }
-
- if (rctx->dst_dma_count) {
- dma_unmap_sg(dev, req->dst, rctx->dst_nents, rctx->in_place ?
- DMA_BIDIRECTIONAL :
- DMA_FROM_DEVICE);
- rctx->dst_dma_count = 0;
- }
- /* Clean up OCS DMA linked lists */
- cleanup_ocs_dma_linked_list(dev, &rctx->src_dll);
- cleanup_ocs_dma_linked_list(dev, &rctx->dst_dll);
- cleanup_ocs_dma_linked_list(dev, &rctx->aad_src_dll);
- cleanup_ocs_dma_linked_list(dev, &rctx->aad_dst_dll);
-}
-
-/**
- * kmb_ocs_aead_dma_prepare() - Do DMA mapping for AEAD processing.
- * @req: The AEAD request being processed.
- * @src_dll_size: Where to store the length of the data mapped into the
- * src_dll OCS DMA list.
- *
- * Do the following:
- * - DMA map req->src and req->dst
- * - Initialize the following OCS DMA linked lists: rctx->src_dll,
- * rctx->dst_dll, rctx->aad_src_dll and rxtc->aad_dst_dll.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int kmb_ocs_aead_dma_prepare(struct aead_request *req, u32 *src_dll_size)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
- struct ocs_aes_rctx *rctx = aead_request_ctx(req);
- u32 in_size; /* The length of the data to be mapped by src_dll. */
- u32 out_size; /* The length of the data to be mapped by dst_dll. */
- u32 dst_size; /* The length of the data in dst_sg. */
- int rc;
-
- /* Get number of entries in input data SG list. */
- rctx->src_nents = sg_nents_for_len(req->src,
- req->assoclen + req->cryptlen);
- if (rctx->src_nents < 0)
- return -EBADMSG;
-
- if (rctx->instruction == OCS_DECRYPT) {
- /*
- * For decrypt:
- * - src sg list is: AAD|CT|tag
- * - dst sg list expects: AAD|PT
- *
- * in_size == len(CT); out_size == len(PT)
- */
-
- /* req->cryptlen includes both CT and tag. */
- in_size = req->cryptlen - tag_size;
-
- /* out_size = PT size == CT size */
- out_size = in_size;
-
- /* len(dst_sg) == len(AAD) + len(PT) */
- dst_size = req->assoclen + out_size;
-
- /*
- * Copy tag from source SG list to 'in_tag' buffer.
- *
- * Note: this needs to be done here, before DMA mapping src_sg.
- */
- sg_pcopy_to_buffer(req->src, rctx->src_nents, rctx->in_tag,
- tag_size, req->assoclen + in_size);
-
- } else { /* OCS_ENCRYPT */
- /*
- * For encrypt:
- * src sg list is: AAD|PT
- * dst sg list expects: AAD|CT|tag
- */
- /* in_size == len(PT) */
- in_size = req->cryptlen;
-
- /*
- * In CCM mode the OCS engine appends the tag to the ciphertext,
- * but in GCM mode the tag must be read from the tag registers
- * and appended manually below
- */
- out_size = (rctx->mode == OCS_MODE_CCM) ? in_size + tag_size :
- in_size;
- /* len(dst_sg) == len(AAD) + len(CT) + len(tag) */
- dst_size = req->assoclen + in_size + tag_size;
- }
- *src_dll_size = in_size;
-
- /* Get number of entries in output data SG list. */
- rctx->dst_nents = sg_nents_for_len(req->dst, dst_size);
- if (rctx->dst_nents < 0)
- return -EBADMSG;
-
- rctx->in_place = (req->src == req->dst) ? 1 : 0;
-
- /* Map destination; use bidirectional mapping for in-place case. */
- rctx->dst_dma_count = dma_map_sg(tctx->aes_dev->dev, req->dst,
- rctx->dst_nents,
- rctx->in_place ? DMA_BIDIRECTIONAL :
- DMA_FROM_DEVICE);
- if (rctx->dst_dma_count == 0 && rctx->dst_nents != 0) {
- dev_err(tctx->aes_dev->dev, "Failed to map destination sg\n");
- return -ENOMEM;
- }
-
- /* Create AAD DST list: maps dst[0:AAD_SIZE-1]. */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count,
- &rctx->aad_dst_dll, req->assoclen,
- 0);
- if (rc)
- return rc;
-
- /* Create DST list: maps dst[AAD_SIZE:out_size] */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count, &rctx->dst_dll,
- out_size, req->assoclen);
- if (rc)
- return rc;
-
- if (rctx->in_place) {
- /* If this is not CCM encrypt, we are done. */
- if (!(rctx->mode == OCS_MODE_CCM &&
- rctx->instruction == OCS_ENCRYPT)) {
- /*
- * SRC and DST are the same, so re-use the same DMA
- * addresses (to avoid allocating new DMA lists
- * identical to the dst ones).
- */
- rctx->src_dll.dma_addr = rctx->dst_dll.dma_addr;
- rctx->aad_src_dll.dma_addr = rctx->aad_dst_dll.dma_addr;
-
- return 0;
- }
- /*
- * For CCM encrypt the input and output linked lists contain
- * different amounts of data, so, we need to create different
- * SRC and AAD SRC lists, even for the in-place case.
- */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count,
- &rctx->aad_src_dll,
- req->assoclen, 0);
- if (rc)
- return rc;
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->dst,
- rctx->dst_dma_count,
- &rctx->src_dll, in_size,
- req->assoclen);
- if (rc)
- return rc;
-
- return 0;
- }
- /* Not in-place case. */
-
- /* Map source SG. */
- rctx->src_dma_count = dma_map_sg(tctx->aes_dev->dev, req->src,
- rctx->src_nents, DMA_TO_DEVICE);
- if (rctx->src_dma_count == 0 && rctx->src_nents != 0) {
- dev_err(tctx->aes_dev->dev, "Failed to map source sg\n");
- return -ENOMEM;
- }
-
- /* Create AAD SRC list. */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
- rctx->src_dma_count,
- &rctx->aad_src_dll,
- req->assoclen, 0);
- if (rc)
- return rc;
-
- /* Create SRC list. */
- rc = ocs_create_linked_list_from_sg(tctx->aes_dev, req->src,
- rctx->src_dma_count,
- &rctx->src_dll, in_size,
- req->assoclen);
- if (rc)
- return rc;
-
- if (req->assoclen == 0)
- return 0;
-
- /* Copy AAD from src sg to dst sg using OCS DMA. */
- rc = ocs_aes_bypass_op(tctx->aes_dev, rctx->aad_dst_dll.dma_addr,
- rctx->aad_src_dll.dma_addr, req->cryptlen);
- if (rc)
- dev_err(tctx->aes_dev->dev,
- "Failed to copy source AAD to destination AAD\n");
-
- return rc;
-}
-
-static int kmb_ocs_aead_run(struct aead_request *req)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- const int tag_size = crypto_aead_authsize(crypto_aead_reqtfm(req));
- struct ocs_aes_rctx *rctx = aead_request_ctx(req);
- u32 in_size; /* The length of the data mapped by src_dll. */
- int rc;
-
- rc = kmb_ocs_aead_dma_prepare(req, &in_size);
- if (rc)
- goto exit;
-
- /* For CCM, we just call the OCS processing and we are done. */
- if (rctx->mode == OCS_MODE_CCM) {
- rc = ocs_aes_ccm_op(tctx->aes_dev, tctx->cipher,
- rctx->instruction, rctx->dst_dll.dma_addr,
- rctx->src_dll.dma_addr, in_size,
- req->iv,
- rctx->aad_src_dll.dma_addr, req->assoclen,
- rctx->in_tag, tag_size);
- goto exit;
- }
- /* GCM case; invoke OCS processing. */
- rc = ocs_aes_gcm_op(tctx->aes_dev, tctx->cipher,
- rctx->instruction,
- rctx->dst_dll.dma_addr,
- rctx->src_dll.dma_addr, in_size,
- req->iv,
- rctx->aad_src_dll.dma_addr, req->assoclen,
- rctx->out_tag, tag_size);
- if (rc)
- goto exit;
-
- /* For GCM decrypt, we have to compare in_tag with out_tag. */
- if (rctx->instruction == OCS_DECRYPT) {
- rc = memcmp(rctx->in_tag, rctx->out_tag, tag_size) ?
- -EBADMSG : 0;
- goto exit;
- }
-
- /* For GCM encrypt, we must manually copy out_tag to DST sg. */
-
- /* Clean-up must be called before the sg_pcopy_from_buffer() below. */
- kmb_ocs_aead_dma_cleanup(req);
-
- /* Copy tag to destination sg after AAD and CT. */
- sg_pcopy_from_buffer(req->dst, rctx->dst_nents, rctx->out_tag,
- tag_size, req->assoclen + req->cryptlen);
-
- /* Return directly as DMA cleanup already done. */
- return 0;
-
-exit:
- kmb_ocs_aead_dma_cleanup(req);
-
- return rc;
-}
-
-static int kmb_ocs_aes_sk_do_one_request(struct crypto_engine *engine,
- void *areq)
-{
- struct skcipher_request *req =
- container_of(areq, struct skcipher_request, base);
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- int err;
-
- if (!tctx->aes_dev) {
- err = -ENODEV;
- goto exit;
- }
-
- err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
- tctx->cipher);
- if (err)
- goto exit;
-
- err = kmb_ocs_sk_run(req);
-
-exit:
- crypto_finalize_skcipher_request(engine, req, err);
-
- return 0;
-}
-
-static int kmb_ocs_aes_aead_do_one_request(struct crypto_engine *engine,
- void *areq)
-{
- struct aead_request *req = container_of(areq,
- struct aead_request, base);
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
- int err;
-
- if (!tctx->aes_dev)
- return -ENODEV;
-
- err = ocs_aes_set_key(tctx->aes_dev, tctx->key_len, tctx->key,
- tctx->cipher);
- if (err)
- goto exit;
-
- err = kmb_ocs_aead_run(req);
-
-exit:
- crypto_finalize_aead_request(tctx->aes_dev->engine, req, err);
-
- return 0;
-}
-
-static int kmb_ocs_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_AES);
-}
-
-static int kmb_ocs_aes_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_AES);
-}
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
-static int kmb_ocs_aes_ecb_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_ECB);
-}
-
-static int kmb_ocs_aes_ecb_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_ECB);
-}
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
-
-static int kmb_ocs_aes_cbc_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CBC);
-}
-
-static int kmb_ocs_aes_cbc_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CBC);
-}
-
-static int kmb_ocs_aes_ctr_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTR);
-}
-
-static int kmb_ocs_aes_ctr_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTR);
-}
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
-static int kmb_ocs_aes_cts_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CTS);
-}
-
-static int kmb_ocs_aes_cts_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CTS);
-}
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
-
-static int kmb_ocs_aes_gcm_encrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_GCM);
-}
-
-static int kmb_ocs_aes_gcm_decrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_GCM);
-}
-
-static int kmb_ocs_aes_ccm_encrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_AES, OCS_ENCRYPT, OCS_MODE_CCM);
-}
-
-static int kmb_ocs_aes_ccm_decrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_AES, OCS_DECRYPT, OCS_MODE_CCM);
-}
-
-static int kmb_ocs_sm4_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- return kmb_ocs_sk_set_key(tfm, in_key, key_len, OCS_SM4);
-}
-
-static int kmb_ocs_sm4_aead_set_key(struct crypto_aead *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- return kmb_ocs_aead_set_key(tfm, in_key, key_len, OCS_SM4);
-}
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
-static int kmb_ocs_sm4_ecb_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_ECB);
-}
-
-static int kmb_ocs_sm4_ecb_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_ECB);
-}
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
-
-static int kmb_ocs_sm4_cbc_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CBC);
-}
-
-static int kmb_ocs_sm4_cbc_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CBC);
-}
-
-static int kmb_ocs_sm4_ctr_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTR);
-}
-
-static int kmb_ocs_sm4_ctr_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTR);
-}
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
-static int kmb_ocs_sm4_cts_encrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CTS);
-}
-
-static int kmb_ocs_sm4_cts_decrypt(struct skcipher_request *req)
-{
- return kmb_ocs_sk_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CTS);
-}
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
-
-static int kmb_ocs_sm4_gcm_encrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_GCM);
-}
-
-static int kmb_ocs_sm4_gcm_decrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_GCM);
-}
-
-static int kmb_ocs_sm4_ccm_encrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_SM4, OCS_ENCRYPT, OCS_MODE_CCM);
-}
-
-static int kmb_ocs_sm4_ccm_decrypt(struct aead_request *req)
-{
- return kmb_ocs_aead_common(req, OCS_SM4, OCS_DECRYPT, OCS_MODE_CCM);
-}
-
-static inline int ocs_common_init(struct ocs_aes_tctx *tctx)
-{
- tctx->engine_ctx.op.prepare_request = NULL;
- tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_sk_do_one_request;
- tctx->engine_ctx.op.unprepare_request = NULL;
-
- return 0;
-}
-
-static int ocs_aes_init_tfm(struct crypto_skcipher *tfm)
-{
- const char *alg_name = crypto_tfm_alg_name(&tfm->base);
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
- struct crypto_sync_skcipher *blk;
-
- /* set fallback cipher in case it will be needed */
- blk = crypto_alloc_sync_skcipher(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
- if (IS_ERR(blk))
- return PTR_ERR(blk);
-
- tctx->sw_cipher.sk = blk;
-
- crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
-
- return ocs_common_init(tctx);
-}
-
-static int ocs_sm4_init_tfm(struct crypto_skcipher *tfm)
-{
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
-
- crypto_skcipher_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
-
- return ocs_common_init(tctx);
-}
-
-static inline void clear_key(struct ocs_aes_tctx *tctx)
-{
- memzero_explicit(tctx->key, OCS_AES_KEYSIZE_256);
-
- /* Zero key registers if set */
- if (tctx->aes_dev)
- ocs_aes_set_key(tctx->aes_dev, OCS_AES_KEYSIZE_256,
- tctx->key, OCS_AES);
-}
-
-static void ocs_exit_tfm(struct crypto_skcipher *tfm)
-{
- struct ocs_aes_tctx *tctx = crypto_skcipher_ctx(tfm);
-
- clear_key(tctx);
-
- if (tctx->sw_cipher.sk) {
- crypto_free_sync_skcipher(tctx->sw_cipher.sk);
- tctx->sw_cipher.sk = NULL;
- }
-}
-
-static inline int ocs_common_aead_init(struct ocs_aes_tctx *tctx)
-{
- tctx->engine_ctx.op.prepare_request = NULL;
- tctx->engine_ctx.op.do_one_request = kmb_ocs_aes_aead_do_one_request;
- tctx->engine_ctx.op.unprepare_request = NULL;
-
- return 0;
-}
-
-static int ocs_aes_aead_cra_init(struct crypto_aead *tfm)
-{
- const char *alg_name = crypto_tfm_alg_name(&tfm->base);
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
- struct crypto_aead *blk;
-
- /* Set fallback cipher in case it will be needed */
- blk = crypto_alloc_aead(alg_name, 0, CRYPTO_ALG_NEED_FALLBACK);
- if (IS_ERR(blk))
- return PTR_ERR(blk);
-
- tctx->sw_cipher.aead = blk;
-
- crypto_aead_set_reqsize(tfm,
- max(sizeof(struct ocs_aes_rctx),
- (sizeof(struct aead_request) +
- crypto_aead_reqsize(tctx->sw_cipher.aead))));
-
- return ocs_common_aead_init(tctx);
-}
-
-static int kmb_ocs_aead_ccm_setauthsize(struct crypto_aead *tfm,
- unsigned int authsize)
-{
- switch (authsize) {
- case 4:
- case 6:
- case 8:
- case 10:
- case 12:
- case 14:
- case 16:
- return 0;
- default:
- return -EINVAL;
- }
-}
-
-static int kmb_ocs_aead_gcm_setauthsize(struct crypto_aead *tfm,
- unsigned int authsize)
-{
- return crypto_gcm_check_authsize(authsize);
-}
-
-static int ocs_sm4_aead_cra_init(struct crypto_aead *tfm)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
-
- crypto_aead_set_reqsize(tfm, sizeof(struct ocs_aes_rctx));
-
- return ocs_common_aead_init(tctx);
-}
-
-static void ocs_aead_cra_exit(struct crypto_aead *tfm)
-{
- struct ocs_aes_tctx *tctx = crypto_aead_ctx(tfm);
-
- clear_key(tctx);
-
- if (tctx->sw_cipher.aead) {
- crypto_free_aead(tctx->sw_cipher.aead);
- tctx->sw_cipher.aead = NULL;
- }
-}
-
-static struct skcipher_alg algs[] = {
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
- {
- .base.cra_name = "ecb(aes)",
- .base.cra_driver_name = "ecb-aes-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_AES_MIN_KEY_SIZE,
- .max_keysize = OCS_AES_MAX_KEY_SIZE,
- .setkey = kmb_ocs_aes_set_key,
- .encrypt = kmb_ocs_aes_ecb_encrypt,
- .decrypt = kmb_ocs_aes_ecb_decrypt,
- .init = ocs_aes_init_tfm,
- .exit = ocs_exit_tfm,
- },
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
- {
- .base.cra_name = "cbc(aes)",
- .base.cra_driver_name = "cbc-aes-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_AES_MIN_KEY_SIZE,
- .max_keysize = OCS_AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_aes_set_key,
- .encrypt = kmb_ocs_aes_cbc_encrypt,
- .decrypt = kmb_ocs_aes_cbc_decrypt,
- .init = ocs_aes_init_tfm,
- .exit = ocs_exit_tfm,
- },
- {
- .base.cra_name = "ctr(aes)",
- .base.cra_driver_name = "ctr-aes-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = 1,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_AES_MIN_KEY_SIZE,
- .max_keysize = OCS_AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_aes_set_key,
- .encrypt = kmb_ocs_aes_ctr_encrypt,
- .decrypt = kmb_ocs_aes_ctr_decrypt,
- .init = ocs_aes_init_tfm,
- .exit = ocs_exit_tfm,
- },
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
- {
- .base.cra_name = "cts(cbc(aes))",
- .base.cra_driver_name = "cts-aes-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_AES_MIN_KEY_SIZE,
- .max_keysize = OCS_AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_aes_set_key,
- .encrypt = kmb_ocs_aes_cts_encrypt,
- .decrypt = kmb_ocs_aes_cts_decrypt,
- .init = ocs_aes_init_tfm,
- .exit = ocs_exit_tfm,
- },
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
- {
- .base.cra_name = "ecb(sm4)",
- .base.cra_driver_name = "ecb-sm4-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_SM4_KEY_SIZE,
- .max_keysize = OCS_SM4_KEY_SIZE,
- .setkey = kmb_ocs_sm4_set_key,
- .encrypt = kmb_ocs_sm4_ecb_encrypt,
- .decrypt = kmb_ocs_sm4_ecb_decrypt,
- .init = ocs_sm4_init_tfm,
- .exit = ocs_exit_tfm,
- },
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
- {
- .base.cra_name = "cbc(sm4)",
- .base.cra_driver_name = "cbc-sm4-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_SM4_KEY_SIZE,
- .max_keysize = OCS_SM4_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_sm4_set_key,
- .encrypt = kmb_ocs_sm4_cbc_encrypt,
- .decrypt = kmb_ocs_sm4_cbc_decrypt,
- .init = ocs_sm4_init_tfm,
- .exit = ocs_exit_tfm,
- },
- {
- .base.cra_name = "ctr(sm4)",
- .base.cra_driver_name = "ctr-sm4-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .base.cra_blocksize = 1,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_SM4_KEY_SIZE,
- .max_keysize = OCS_SM4_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_sm4_set_key,
- .encrypt = kmb_ocs_sm4_ctr_encrypt,
- .decrypt = kmb_ocs_sm4_ctr_decrypt,
- .init = ocs_sm4_init_tfm,
- .exit = ocs_exit_tfm,
- },
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
- {
- .base.cra_name = "cts(cbc(sm4))",
- .base.cra_driver_name = "cts-sm4-keembay-ocs",
- .base.cra_priority = KMB_OCS_PRIORITY,
- .base.cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .base.cra_module = THIS_MODULE,
- .base.cra_alignmask = 0,
-
- .min_keysize = OCS_SM4_KEY_SIZE,
- .max_keysize = OCS_SM4_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = kmb_ocs_sm4_set_key,
- .encrypt = kmb_ocs_sm4_cts_encrypt,
- .decrypt = kmb_ocs_sm4_cts_decrypt,
- .init = ocs_sm4_init_tfm,
- .exit = ocs_exit_tfm,
- }
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
-};
-
-static struct aead_alg algs_aead[] = {
- {
- .base = {
- .cra_name = "gcm(aes)",
- .cra_driver_name = "gcm-aes-keembay-ocs",
- .cra_priority = KMB_OCS_PRIORITY,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- },
- .init = ocs_aes_aead_cra_init,
- .exit = ocs_aead_cra_exit,
- .ivsize = GCM_AES_IV_SIZE,
- .maxauthsize = AES_BLOCK_SIZE,
- .setauthsize = kmb_ocs_aead_gcm_setauthsize,
- .setkey = kmb_ocs_aes_aead_set_key,
- .encrypt = kmb_ocs_aes_gcm_encrypt,
- .decrypt = kmb_ocs_aes_gcm_decrypt,
- },
- {
- .base = {
- .cra_name = "ccm(aes)",
- .cra_driver_name = "ccm-aes-keembay-ocs",
- .cra_priority = KMB_OCS_PRIORITY,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY |
- CRYPTO_ALG_NEED_FALLBACK,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- },
- .init = ocs_aes_aead_cra_init,
- .exit = ocs_aead_cra_exit,
- .ivsize = AES_BLOCK_SIZE,
- .maxauthsize = AES_BLOCK_SIZE,
- .setauthsize = kmb_ocs_aead_ccm_setauthsize,
- .setkey = kmb_ocs_aes_aead_set_key,
- .encrypt = kmb_ocs_aes_ccm_encrypt,
- .decrypt = kmb_ocs_aes_ccm_decrypt,
- },
- {
- .base = {
- .cra_name = "gcm(sm4)",
- .cra_driver_name = "gcm-sm4-keembay-ocs",
- .cra_priority = KMB_OCS_PRIORITY,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- },
- .init = ocs_sm4_aead_cra_init,
- .exit = ocs_aead_cra_exit,
- .ivsize = GCM_AES_IV_SIZE,
- .maxauthsize = AES_BLOCK_SIZE,
- .setauthsize = kmb_ocs_aead_gcm_setauthsize,
- .setkey = kmb_ocs_sm4_aead_set_key,
- .encrypt = kmb_ocs_sm4_gcm_encrypt,
- .decrypt = kmb_ocs_sm4_gcm_decrypt,
- },
- {
- .base = {
- .cra_name = "ccm(sm4)",
- .cra_driver_name = "ccm-sm4-keembay-ocs",
- .cra_priority = KMB_OCS_PRIORITY,
- .cra_flags = CRYPTO_ALG_ASYNC |
- CRYPTO_ALG_KERN_DRIVER_ONLY,
- .cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct ocs_aes_tctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- },
- .init = ocs_sm4_aead_cra_init,
- .exit = ocs_aead_cra_exit,
- .ivsize = AES_BLOCK_SIZE,
- .maxauthsize = AES_BLOCK_SIZE,
- .setauthsize = kmb_ocs_aead_ccm_setauthsize,
- .setkey = kmb_ocs_sm4_aead_set_key,
- .encrypt = kmb_ocs_sm4_ccm_encrypt,
- .decrypt = kmb_ocs_sm4_ccm_decrypt,
- }
-};
-
-static void unregister_aes_algs(struct ocs_aes_dev *aes_dev)
-{
- crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs_aead));
- crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
-}
-
-static int register_aes_algs(struct ocs_aes_dev *aes_dev)
-{
- int ret;
-
- /*
- * If any algorithm fails to register, all preceding algorithms that
- * were successfully registered will be automatically unregistered.
- */
- ret = crypto_register_aeads(algs_aead, ARRAY_SIZE(algs_aead));
- if (ret)
- return ret;
-
- ret = crypto_register_skciphers(algs, ARRAY_SIZE(algs));
- if (ret)
- crypto_unregister_aeads(algs_aead, ARRAY_SIZE(algs));
-
- return ret;
-}
-
-/* Device tree driver match. */
-static const struct of_device_id kmb_ocs_aes_of_match[] = {
- {
- .compatible = "intel,keembay-ocs-aes",
- },
- {}
-};
-
-static int kmb_ocs_aes_remove(struct platform_device *pdev)
-{
- struct ocs_aes_dev *aes_dev;
-
- aes_dev = platform_get_drvdata(pdev);
-
- unregister_aes_algs(aes_dev);
-
- spin_lock(&ocs_aes.lock);
- list_del(&aes_dev->list);
- spin_unlock(&ocs_aes.lock);
-
- crypto_engine_exit(aes_dev->engine);
-
- return 0;
-}
-
-static int kmb_ocs_aes_probe(struct platform_device *pdev)
-{
- struct device *dev = &pdev->dev;
- struct ocs_aes_dev *aes_dev;
- int rc;
-
- aes_dev = devm_kzalloc(dev, sizeof(*aes_dev), GFP_KERNEL);
- if (!aes_dev)
- return -ENOMEM;
-
- aes_dev->dev = dev;
-
- platform_set_drvdata(pdev, aes_dev);
-
- rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
- if (rc) {
- dev_err(dev, "Failed to set 32 bit dma mask %d\n", rc);
- return rc;
- }
-
- /* Get base register address. */
- aes_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);
- if (IS_ERR(aes_dev->base_reg))
- return PTR_ERR(aes_dev->base_reg);
-
- /* Get and request IRQ */
- aes_dev->irq = platform_get_irq(pdev, 0);
- if (aes_dev->irq < 0)
- return aes_dev->irq;
-
- rc = devm_request_threaded_irq(dev, aes_dev->irq, ocs_aes_irq_handler,
- NULL, 0, "keembay-ocs-aes", aes_dev);
- if (rc < 0) {
- dev_err(dev, "Could not request IRQ\n");
- return rc;
- }
-
- INIT_LIST_HEAD(&aes_dev->list);
- spin_lock(&ocs_aes.lock);
- list_add_tail(&aes_dev->list, &ocs_aes.dev_list);
- spin_unlock(&ocs_aes.lock);
-
- init_completion(&aes_dev->irq_completion);
-
- /* Initialize crypto engine */
- aes_dev->engine = crypto_engine_alloc_init(dev, true);
- if (!aes_dev->engine) {
- rc = -ENOMEM;
- goto list_del;
- }
-
- rc = crypto_engine_start(aes_dev->engine);
- if (rc) {
- dev_err(dev, "Could not start crypto engine\n");
- goto cleanup;
- }
-
- rc = register_aes_algs(aes_dev);
- if (rc) {
- dev_err(dev,
- "Could not register OCS algorithms with Crypto API\n");
- goto cleanup;
- }
-
- return 0;
-
-cleanup:
- crypto_engine_exit(aes_dev->engine);
-list_del:
- spin_lock(&ocs_aes.lock);
- list_del(&aes_dev->list);
- spin_unlock(&ocs_aes.lock);
-
- return rc;
-}
-
-/* The OCS driver is a platform device. */
-static struct platform_driver kmb_ocs_aes_driver = {
- .probe = kmb_ocs_aes_probe,
- .remove = kmb_ocs_aes_remove,
- .driver = {
- .name = DRV_NAME,
- .of_match_table = kmb_ocs_aes_of_match,
- },
-};
-
-module_platform_driver(kmb_ocs_aes_driver);
-
-MODULE_DESCRIPTION("Intel Keem Bay Offload and Crypto Subsystem (OCS) AES/SM4 Driver");
-MODULE_LICENSE("GPL");
-
-MODULE_ALIAS_CRYPTO("cbc-aes-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ctr-aes-keembay-ocs");
-MODULE_ALIAS_CRYPTO("gcm-aes-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ccm-aes-keembay-ocs");
-
-MODULE_ALIAS_CRYPTO("cbc-sm4-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ctr-sm4-keembay-ocs");
-MODULE_ALIAS_CRYPTO("gcm-sm4-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ccm-sm4-keembay-ocs");
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB
-MODULE_ALIAS_CRYPTO("ecb-aes-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ecb-sm4-keembay-ocs");
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_ECB */
-
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS
-MODULE_ALIAS_CRYPTO("cts-aes-keembay-ocs");
-MODULE_ALIAS_CRYPTO("cts-sm4-keembay-ocs");
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_AES_SM4_CTS */
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Intel Keem Bay OCS ECC Crypto Driver.
- *
- * Copyright (C) 2019-2021 Intel Corporation
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/clk.h>
-#include <linux/completion.h>
-#include <linux/crypto.h>
-#include <linux/delay.h>
-#include <linux/fips.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/iopoll.h>
-#include <linux/irq.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
-#include <linux/scatterlist.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-
-#include <crypto/ecc_curve.h>
-#include <crypto/ecdh.h>
-#include <crypto/engine.h>
-#include <crypto/kpp.h>
-#include <crypto/rng.h>
-
-#include <crypto/internal/ecc.h>
-#include <crypto/internal/kpp.h>
-
-#define DRV_NAME "keembay-ocs-ecc"
-
-#define KMB_OCS_ECC_PRIORITY 350
-
-#define HW_OFFS_OCS_ECC_COMMAND 0x00000000
-#define HW_OFFS_OCS_ECC_STATUS 0x00000004
-#define HW_OFFS_OCS_ECC_DATA_IN 0x00000080
-#define HW_OFFS_OCS_ECC_CX_DATA_OUT 0x00000100
-#define HW_OFFS_OCS_ECC_CY_DATA_OUT 0x00000180
-#define HW_OFFS_OCS_ECC_ISR 0x00000400
-#define HW_OFFS_OCS_ECC_IER 0x00000404
-
-#define HW_OCS_ECC_ISR_INT_STATUS_DONE BIT(0)
-#define HW_OCS_ECC_COMMAND_INS_BP BIT(0)
-
-#define HW_OCS_ECC_COMMAND_START_VAL BIT(0)
-
-#define OCS_ECC_OP_SIZE_384 BIT(8)
-#define OCS_ECC_OP_SIZE_256 0
-
-/* ECC Instruction : for ECC_COMMAND */
-#define OCS_ECC_INST_WRITE_AX (0x1 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_WRITE_AY (0x2 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_WRITE_BX_D (0x3 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_WRITE_BY_L (0x4 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_WRITE_P (0x5 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_WRITE_A (0x6 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_CALC_D_IDX_A (0x8 << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_CALC_A_POW_B_MODP (0xB << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_CALC_A_MUL_B_MODP (0xC << HW_OCS_ECC_COMMAND_INS_BP)
-#define OCS_ECC_INST_CALC_A_ADD_B_MODP (0xD << HW_OCS_ECC_COMMAND_INS_BP)
-
-#define ECC_ENABLE_INTR 1
-
-#define POLL_USEC 100
-#define TIMEOUT_USEC 10000
-
-#define KMB_ECC_VLI_MAX_DIGITS ECC_CURVE_NIST_P384_DIGITS
-#define KMB_ECC_VLI_MAX_BYTES (KMB_ECC_VLI_MAX_DIGITS \
- << ECC_DIGITS_TO_BYTES_SHIFT)
-
-#define POW_CUBE 3
-
-/**
- * struct ocs_ecc_dev - ECC device context
- * @list: List of device contexts
- * @dev: OCS ECC device
- * @base_reg: IO base address of OCS ECC
- * @engine: Crypto engine for the device
- * @irq_done: IRQ done completion.
- * @irq: IRQ number
- */
-struct ocs_ecc_dev {
- struct list_head list;
- struct device *dev;
- void __iomem *base_reg;
- struct crypto_engine *engine;
- struct completion irq_done;
- int irq;
-};
-
-/**
- * struct ocs_ecc_ctx - Transformation context.
- * @engine_ctx: Crypto engine ctx.
- * @ecc_dev: The ECC driver associated with this context.
- * @curve: The elliptic curve used by this transformation.
- * @private_key: The private key.
- */
-struct ocs_ecc_ctx {
- struct crypto_engine_ctx engine_ctx;
- struct ocs_ecc_dev *ecc_dev;
- const struct ecc_curve *curve;
- u64 private_key[KMB_ECC_VLI_MAX_DIGITS];
-};
-
-/* Driver data. */
-struct ocs_ecc_drv {
- struct list_head dev_list;
- spinlock_t lock; /* Protects dev_list. */
-};
-
-/* Global variable holding the list of OCS ECC devices (only one expected). */
-static struct ocs_ecc_drv ocs_ecc = {
- .dev_list = LIST_HEAD_INIT(ocs_ecc.dev_list),
- .lock = __SPIN_LOCK_UNLOCKED(ocs_ecc.lock),
-};
-
-/* Get OCS ECC tfm context from kpp_request. */
-static inline struct ocs_ecc_ctx *kmb_ocs_ecc_tctx(struct kpp_request *req)
-{
- return kpp_tfm_ctx(crypto_kpp_reqtfm(req));
-}
-
-/* Converts number of digits to number of bytes. */
-static inline unsigned int digits_to_bytes(unsigned int n)
-{
- return n << ECC_DIGITS_TO_BYTES_SHIFT;
-}
-
-/*
- * Wait for ECC idle i.e when an operation (other than write operations)
- * is done.
- */
-static inline int ocs_ecc_wait_idle(struct ocs_ecc_dev *dev)
-{
- u32 value;
-
- return readl_poll_timeout((dev->base_reg + HW_OFFS_OCS_ECC_STATUS),
- value,
- !(value & HW_OCS_ECC_ISR_INT_STATUS_DONE),
- POLL_USEC, TIMEOUT_USEC);
-}
-
-static void ocs_ecc_cmd_start(struct ocs_ecc_dev *ecc_dev, u32 op_size)
-{
- iowrite32(op_size | HW_OCS_ECC_COMMAND_START_VAL,
- ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
-}
-
-/* Direct write of u32 buffer to ECC engine with associated instruction. */
-static void ocs_ecc_write_cmd_and_data(struct ocs_ecc_dev *dev,
- u32 op_size,
- u32 inst,
- const void *data_in,
- size_t data_size)
-{
- iowrite32(op_size | inst, dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
-
- /* MMIO Write src uint32 to dst. */
- memcpy_toio(dev->base_reg + HW_OFFS_OCS_ECC_DATA_IN, data_in,
- data_size);
-}
-
-/* Start OCS ECC operation and wait for its completion. */
-static int ocs_ecc_trigger_op(struct ocs_ecc_dev *ecc_dev, u32 op_size,
- u32 inst)
-{
- reinit_completion(&ecc_dev->irq_done);
-
- iowrite32(ECC_ENABLE_INTR, ecc_dev->base_reg + HW_OFFS_OCS_ECC_IER);
- iowrite32(op_size | inst, ecc_dev->base_reg + HW_OFFS_OCS_ECC_COMMAND);
-
- return wait_for_completion_interruptible(&ecc_dev->irq_done);
-}
-
-/**
- * ocs_ecc_read_cx_out() - Read the CX data output buffer.
- * @dev: The OCS ECC device to read from.
- * @cx_out: The buffer where to store the CX value. Must be at least
- * @byte_count byte long.
- * @byte_count: The amount of data to read.
- */
-static inline void ocs_ecc_read_cx_out(struct ocs_ecc_dev *dev, void *cx_out,
- size_t byte_count)
-{
- memcpy_fromio(cx_out, dev->base_reg + HW_OFFS_OCS_ECC_CX_DATA_OUT,
- byte_count);
-}
-
-/**
- * ocs_ecc_read_cy_out() - Read the CX data output buffer.
- * @dev: The OCS ECC device to read from.
- * @cy_out: The buffer where to store the CY value. Must be at least
- * @byte_count byte long.
- * @byte_count: The amount of data to read.
- */
-static inline void ocs_ecc_read_cy_out(struct ocs_ecc_dev *dev, void *cy_out,
- size_t byte_count)
-{
- memcpy_fromio(cy_out, dev->base_reg + HW_OFFS_OCS_ECC_CY_DATA_OUT,
- byte_count);
-}
-
-static struct ocs_ecc_dev *kmb_ocs_ecc_find_dev(struct ocs_ecc_ctx *tctx)
-{
- if (tctx->ecc_dev)
- return tctx->ecc_dev;
-
- spin_lock(&ocs_ecc.lock);
-
- /* Only a single OCS device available. */
- tctx->ecc_dev = list_first_entry(&ocs_ecc.dev_list, struct ocs_ecc_dev,
- list);
-
- spin_unlock(&ocs_ecc.lock);
-
- return tctx->ecc_dev;
-}
-
-/* Do point multiplication using OCS ECC HW. */
-static int kmb_ecc_point_mult(struct ocs_ecc_dev *ecc_dev,
- struct ecc_point *result,
- const struct ecc_point *point,
- u64 *scalar,
- const struct ecc_curve *curve)
-{
- u8 sca[KMB_ECC_VLI_MAX_BYTES]; /* Use the maximum data size. */
- u32 op_size = (curve->g.ndigits > ECC_CURVE_NIST_P256_DIGITS) ?
- OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;
- size_t nbytes = digits_to_bytes(curve->g.ndigits);
- int rc = 0;
-
- /* Generate random nbytes for Simple and Differential SCA protection. */
- rc = crypto_get_default_rng();
- if (rc)
- return rc;
-
- rc = crypto_rng_get_bytes(crypto_default_rng, sca, nbytes);
- crypto_put_default_rng();
- if (rc)
- return rc;
-
- /* Wait engine to be idle before starting new operation. */
- rc = ocs_ecc_wait_idle(ecc_dev);
- if (rc)
- return rc;
-
- /* Send ecc_start pulse as well as indicating operation size. */
- ocs_ecc_cmd_start(ecc_dev, op_size);
-
- /* Write ax param; Base point (Gx). */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,
- point->x, nbytes);
-
- /* Write ay param; Base point (Gy). */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,
- point->y, nbytes);
-
- /*
- * Write the private key into DATA_IN reg.
- *
- * Since DATA_IN register is used to write different values during the
- * computation private Key value is overwritten with
- * side-channel-resistance value.
- */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BX_D,
- scalar, nbytes);
-
- /* Write operand by/l. */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_BY_L,
- sca, nbytes);
- memzero_explicit(sca, sizeof(sca));
-
- /* Write p = curve prime(GF modulus). */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,
- curve->p, nbytes);
-
- /* Write a = curve coefficient. */
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_A,
- curve->a, nbytes);
-
- /* Make hardware perform the multiplication. */
- rc = ocs_ecc_trigger_op(ecc_dev, op_size, OCS_ECC_INST_CALC_D_IDX_A);
- if (rc)
- return rc;
-
- /* Read result. */
- ocs_ecc_read_cx_out(ecc_dev, result->x, nbytes);
- ocs_ecc_read_cy_out(ecc_dev, result->y, nbytes);
-
- return 0;
-}
-
-/**
- * kmb_ecc_do_scalar_op() - Perform Scalar operation using OCS ECC HW.
- * @ecc_dev: The OCS ECC device to use.
- * @scalar_out: Where to store the output scalar.
- * @scalar_a: Input scalar operand 'a'.
- * @scalar_b: Input scalar operand 'b'
- * @curve: The curve on which the operation is performed.
- * @ndigits: The size of the operands (in digits).
- * @inst: The operation to perform (as an OCS ECC instruction).
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int kmb_ecc_do_scalar_op(struct ocs_ecc_dev *ecc_dev, u64 *scalar_out,
- const u64 *scalar_a, const u64 *scalar_b,
- const struct ecc_curve *curve,
- unsigned int ndigits, const u32 inst)
-{
- u32 op_size = (ndigits > ECC_CURVE_NIST_P256_DIGITS) ?
- OCS_ECC_OP_SIZE_384 : OCS_ECC_OP_SIZE_256;
- size_t nbytes = digits_to_bytes(ndigits);
- int rc;
-
- /* Wait engine to be idle before starting new operation. */
- rc = ocs_ecc_wait_idle(ecc_dev);
- if (rc)
- return rc;
-
- /* Send ecc_start pulse as well as indicating operation size. */
- ocs_ecc_cmd_start(ecc_dev, op_size);
-
- /* Write ax param (Base point (Gx).*/
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AX,
- scalar_a, nbytes);
-
- /* Write ay param Base point (Gy).*/
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_AY,
- scalar_b, nbytes);
-
- /* Write p = curve prime(GF modulus).*/
- ocs_ecc_write_cmd_and_data(ecc_dev, op_size, OCS_ECC_INST_WRITE_P,
- curve->p, nbytes);
-
- /* Give instruction A.B or A+B to ECC engine. */
- rc = ocs_ecc_trigger_op(ecc_dev, op_size, inst);
- if (rc)
- return rc;
-
- ocs_ecc_read_cx_out(ecc_dev, scalar_out, nbytes);
-
- if (vli_is_zero(scalar_out, ndigits))
- return -EINVAL;
-
- return 0;
-}
-
-/* SP800-56A section 5.6.2.3.4 partial verification: ephemeral keys only */
-static int kmb_ocs_ecc_is_pubkey_valid_partial(struct ocs_ecc_dev *ecc_dev,
- const struct ecc_curve *curve,
- struct ecc_point *pk)
-{
- u64 xxx[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
- u64 yy[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
- u64 w[KMB_ECC_VLI_MAX_DIGITS] = { 0 };
- int rc;
-
- if (WARN_ON(pk->ndigits != curve->g.ndigits))
- return -EINVAL;
-
- /* Check 1: Verify key is not the zero point. */
- if (ecc_point_is_zero(pk))
- return -EINVAL;
-
- /* Check 2: Verify key is in the range [0, p-1]. */
- if (vli_cmp(curve->p, pk->x, pk->ndigits) != 1)
- return -EINVAL;
-
- if (vli_cmp(curve->p, pk->y, pk->ndigits) != 1)
- return -EINVAL;
-
- /* Check 3: Verify that y^2 == (x^3 + a·x + b) mod p */
-
- /* y^2 */
- /* Compute y^2 -> store in yy */
- rc = kmb_ecc_do_scalar_op(ecc_dev, yy, pk->y, pk->y, curve, pk->ndigits,
- OCS_ECC_INST_CALC_A_MUL_B_MODP);
- if (rc)
- goto exit;
-
- /* x^3 */
- /* Assigning w = 3, used for calculating x^3. */
- w[0] = POW_CUBE;
- /* Load the next stage.*/
- rc = kmb_ecc_do_scalar_op(ecc_dev, xxx, pk->x, w, curve, pk->ndigits,
- OCS_ECC_INST_CALC_A_POW_B_MODP);
- if (rc)
- goto exit;
-
- /* Do a*x -> store in w. */
- rc = kmb_ecc_do_scalar_op(ecc_dev, w, curve->a, pk->x, curve,
- pk->ndigits,
- OCS_ECC_INST_CALC_A_MUL_B_MODP);
- if (rc)
- goto exit;
-
- /* Do ax + b == w + b; store in w. */
- rc = kmb_ecc_do_scalar_op(ecc_dev, w, w, curve->b, curve,
- pk->ndigits,
- OCS_ECC_INST_CALC_A_ADD_B_MODP);
- if (rc)
- goto exit;
-
- /* x^3 + ax + b == x^3 + w -> store in w. */
- rc = kmb_ecc_do_scalar_op(ecc_dev, w, xxx, w, curve, pk->ndigits,
- OCS_ECC_INST_CALC_A_ADD_B_MODP);
- if (rc)
- goto exit;
-
- /* Compare y^2 == x^3 + a·x + b. */
- rc = vli_cmp(yy, w, pk->ndigits);
- if (rc)
- rc = -EINVAL;
-
-exit:
- memzero_explicit(xxx, sizeof(xxx));
- memzero_explicit(yy, sizeof(yy));
- memzero_explicit(w, sizeof(w));
-
- return rc;
-}
-
-/* SP800-56A section 5.6.2.3.3 full verification */
-static int kmb_ocs_ecc_is_pubkey_valid_full(struct ocs_ecc_dev *ecc_dev,
- const struct ecc_curve *curve,
- struct ecc_point *pk)
-{
- struct ecc_point *nQ;
- int rc;
-
- /* Checks 1 through 3 */
- rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);
- if (rc)
- return rc;
-
- /* Check 4: Verify that nQ is the zero point. */
- nQ = ecc_alloc_point(pk->ndigits);
- if (!nQ)
- return -ENOMEM;
-
- rc = kmb_ecc_point_mult(ecc_dev, nQ, pk, curve->n, curve);
- if (rc)
- goto exit;
-
- if (!ecc_point_is_zero(nQ))
- rc = -EINVAL;
-
-exit:
- ecc_free_point(nQ);
-
- return rc;
-}
-
-static int kmb_ecc_is_key_valid(const struct ecc_curve *curve,
- const u64 *private_key, size_t private_key_len)
-{
- size_t ndigits = curve->g.ndigits;
- u64 one[KMB_ECC_VLI_MAX_DIGITS] = {1};
- u64 res[KMB_ECC_VLI_MAX_DIGITS];
-
- if (private_key_len != digits_to_bytes(ndigits))
- return -EINVAL;
-
- if (!private_key)
- return -EINVAL;
-
- /* Make sure the private key is in the range [2, n-3]. */
- if (vli_cmp(one, private_key, ndigits) != -1)
- return -EINVAL;
-
- vli_sub(res, curve->n, one, ndigits);
- vli_sub(res, res, one, ndigits);
- if (vli_cmp(res, private_key, ndigits) != 1)
- return -EINVAL;
-
- return 0;
-}
-
-/*
- * ECC private keys are generated using the method of extra random bits,
- * equivalent to that described in FIPS 186-4, Appendix B.4.1.
- *
- * d = (c mod(n–1)) + 1 where c is a string of random bits, 64 bits longer
- * than requested
- * 0 <= c mod(n-1) <= n-2 and implies that
- * 1 <= d <= n-1
- *
- * This method generates a private key uniformly distributed in the range
- * [1, n-1].
- */
-static int kmb_ecc_gen_privkey(const struct ecc_curve *curve, u64 *privkey)
-{
- size_t nbytes = digits_to_bytes(curve->g.ndigits);
- u64 priv[KMB_ECC_VLI_MAX_DIGITS];
- size_t nbits;
- int rc;
-
- nbits = vli_num_bits(curve->n, curve->g.ndigits);
-
- /* Check that N is included in Table 1 of FIPS 186-4, section 6.1.1 */
- if (nbits < 160 || curve->g.ndigits > ARRAY_SIZE(priv))
- return -EINVAL;
-
- /*
- * FIPS 186-4 recommends that the private key should be obtained from a
- * RBG with a security strength equal to or greater than the security
- * strength associated with N.
- *
- * The maximum security strength identified by NIST SP800-57pt1r4 for
- * ECC is 256 (N >= 512).
- *
- * This condition is met by the default RNG because it selects a favored
- * DRBG with a security strength of 256.
- */
- if (crypto_get_default_rng())
- return -EFAULT;
-
- rc = crypto_rng_get_bytes(crypto_default_rng, (u8 *)priv, nbytes);
- crypto_put_default_rng();
- if (rc)
- goto cleanup;
-
- rc = kmb_ecc_is_key_valid(curve, priv, nbytes);
- if (rc)
- goto cleanup;
-
- ecc_swap_digits(priv, privkey, curve->g.ndigits);
-
-cleanup:
- memzero_explicit(&priv, sizeof(priv));
-
- return rc;
-}
-
-static int kmb_ocs_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf,
- unsigned int len)
-{
- struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
- struct ecdh params;
- int rc = 0;
-
- rc = crypto_ecdh_decode_key(buf, len, ¶ms);
- if (rc)
- goto cleanup;
-
- /* Ensure key size is not bigger then expected. */
- if (params.key_size > digits_to_bytes(tctx->curve->g.ndigits)) {
- rc = -EINVAL;
- goto cleanup;
- }
-
- /* Auto-generate private key is not provided. */
- if (!params.key || !params.key_size) {
- rc = kmb_ecc_gen_privkey(tctx->curve, tctx->private_key);
- goto cleanup;
- }
-
- rc = kmb_ecc_is_key_valid(tctx->curve, (const u64 *)params.key,
- params.key_size);
- if (rc)
- goto cleanup;
-
- ecc_swap_digits((const u64 *)params.key, tctx->private_key,
- tctx->curve->g.ndigits);
-cleanup:
- memzero_explicit(¶ms, sizeof(params));
-
- if (rc)
- tctx->curve = NULL;
-
- return rc;
-}
-
-/* Compute shared secret. */
-static int kmb_ecc_do_shared_secret(struct ocs_ecc_ctx *tctx,
- struct kpp_request *req)
-{
- struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;
- const struct ecc_curve *curve = tctx->curve;
- u64 shared_secret[KMB_ECC_VLI_MAX_DIGITS];
- u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];
- size_t copied, nbytes, pubk_len;
- struct ecc_point *pk, *result;
- int rc;
-
- nbytes = digits_to_bytes(curve->g.ndigits);
-
- /* Public key is a point, thus it has two coordinates */
- pubk_len = 2 * nbytes;
-
- /* Copy public key from SG list to pubk_buf. */
- copied = sg_copy_to_buffer(req->src,
- sg_nents_for_len(req->src, pubk_len),
- pubk_buf, pubk_len);
- if (copied != pubk_len)
- return -EINVAL;
-
- /* Allocate and initialize public key point. */
- pk = ecc_alloc_point(curve->g.ndigits);
- if (!pk)
- return -ENOMEM;
-
- ecc_swap_digits(pubk_buf, pk->x, curve->g.ndigits);
- ecc_swap_digits(&pubk_buf[curve->g.ndigits], pk->y, curve->g.ndigits);
-
- /*
- * Check the public key for following
- * Check 1: Verify key is not the zero point.
- * Check 2: Verify key is in the range [1, p-1].
- * Check 3: Verify that y^2 == (x^3 + a·x + b) mod p
- */
- rc = kmb_ocs_ecc_is_pubkey_valid_partial(ecc_dev, curve, pk);
- if (rc)
- goto exit_free_pk;
-
- /* Allocate point for storing computed shared secret. */
- result = ecc_alloc_point(pk->ndigits);
- if (!result) {
- rc = -ENOMEM;
- goto exit_free_pk;
- }
-
- /* Calculate the shared secret.*/
- rc = kmb_ecc_point_mult(ecc_dev, result, pk, tctx->private_key, curve);
- if (rc)
- goto exit_free_result;
-
- if (ecc_point_is_zero(result)) {
- rc = -EFAULT;
- goto exit_free_result;
- }
-
- /* Copy shared secret from point to buffer. */
- ecc_swap_digits(result->x, shared_secret, result->ndigits);
-
- /* Request might ask for less bytes than what we have. */
- nbytes = min_t(size_t, nbytes, req->dst_len);
-
- copied = sg_copy_from_buffer(req->dst,
- sg_nents_for_len(req->dst, nbytes),
- shared_secret, nbytes);
-
- if (copied != nbytes)
- rc = -EINVAL;
-
- memzero_explicit(shared_secret, sizeof(shared_secret));
-
-exit_free_result:
- ecc_free_point(result);
-
-exit_free_pk:
- ecc_free_point(pk);
-
- return rc;
-}
-
-/* Compute public key. */
-static int kmb_ecc_do_public_key(struct ocs_ecc_ctx *tctx,
- struct kpp_request *req)
-{
- const struct ecc_curve *curve = tctx->curve;
- u64 pubk_buf[KMB_ECC_VLI_MAX_DIGITS * 2];
- struct ecc_point *pk;
- size_t pubk_len;
- size_t copied;
- int rc;
-
- /* Public key is a point, so it has double the digits. */
- pubk_len = 2 * digits_to_bytes(curve->g.ndigits);
-
- pk = ecc_alloc_point(curve->g.ndigits);
- if (!pk)
- return -ENOMEM;
-
- /* Public Key(pk) = priv * G. */
- rc = kmb_ecc_point_mult(tctx->ecc_dev, pk, &curve->g, tctx->private_key,
- curve);
- if (rc)
- goto exit;
-
- /* SP800-56A rev 3 5.6.2.1.3 key check */
- if (kmb_ocs_ecc_is_pubkey_valid_full(tctx->ecc_dev, curve, pk)) {
- rc = -EAGAIN;
- goto exit;
- }
-
- /* Copy public key from point to buffer. */
- ecc_swap_digits(pk->x, pubk_buf, pk->ndigits);
- ecc_swap_digits(pk->y, &pubk_buf[pk->ndigits], pk->ndigits);
-
- /* Copy public key to req->dst. */
- copied = sg_copy_from_buffer(req->dst,
- sg_nents_for_len(req->dst, pubk_len),
- pubk_buf, pubk_len);
-
- if (copied != pubk_len)
- rc = -EINVAL;
-
-exit:
- ecc_free_point(pk);
-
- return rc;
-}
-
-static int kmb_ocs_ecc_do_one_request(struct crypto_engine *engine,
- void *areq)
-{
- struct kpp_request *req = container_of(areq, struct kpp_request, base);
- struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
- struct ocs_ecc_dev *ecc_dev = tctx->ecc_dev;
- int rc;
-
- if (req->src)
- rc = kmb_ecc_do_shared_secret(tctx, req);
- else
- rc = kmb_ecc_do_public_key(tctx, req);
-
- crypto_finalize_kpp_request(ecc_dev->engine, req, rc);
-
- return 0;
-}
-
-static int kmb_ocs_ecdh_generate_public_key(struct kpp_request *req)
-{
- struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
- const struct ecc_curve *curve = tctx->curve;
-
- /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */
- if (!tctx->curve)
- return -EINVAL;
-
- /* Ensure dst is present. */
- if (!req->dst)
- return -EINVAL;
-
- /* Check the request dst is big enough to hold the public key. */
- if (req->dst_len < (2 * digits_to_bytes(curve->g.ndigits)))
- return -EINVAL;
-
- /* 'src' is not supposed to be present when generate pubk is called. */
- if (req->src)
- return -EINVAL;
-
- return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,
- req);
-}
-
-static int kmb_ocs_ecdh_compute_shared_secret(struct kpp_request *req)
-{
- struct ocs_ecc_ctx *tctx = kmb_ocs_ecc_tctx(req);
- const struct ecc_curve *curve = tctx->curve;
-
- /* Ensure kmb_ocs_ecdh_set_secret() has been successfully called. */
- if (!tctx->curve)
- return -EINVAL;
-
- /* Ensure dst is present. */
- if (!req->dst)
- return -EINVAL;
-
- /* Ensure src is present. */
- if (!req->src)
- return -EINVAL;
-
- /*
- * req->src is expected to the (other-side) public key, so its length
- * must be 2 * coordinate size (in bytes).
- */
- if (req->src_len != 2 * digits_to_bytes(curve->g.ndigits))
- return -EINVAL;
-
- return crypto_transfer_kpp_request_to_engine(tctx->ecc_dev->engine,
- req);
-}
-
-static int kmb_ecc_tctx_init(struct ocs_ecc_ctx *tctx, unsigned int curve_id)
-{
- memset(tctx, 0, sizeof(*tctx));
-
- tctx->ecc_dev = kmb_ocs_ecc_find_dev(tctx);
-
- if (IS_ERR(tctx->ecc_dev)) {
- pr_err("Failed to find the device : %ld\n",
- PTR_ERR(tctx->ecc_dev));
- return PTR_ERR(tctx->ecc_dev);
- }
-
- tctx->curve = ecc_get_curve(curve_id);
- if (!tctx->curve)
- return -EOPNOTSUPP;
-
- tctx->engine_ctx.op.prepare_request = NULL;
- tctx->engine_ctx.op.do_one_request = kmb_ocs_ecc_do_one_request;
- tctx->engine_ctx.op.unprepare_request = NULL;
-
- return 0;
-}
-
-static int kmb_ocs_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm)
-{
- struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
-
- return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P256);
-}
-
-static int kmb_ocs_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm)
-{
- struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
-
- return kmb_ecc_tctx_init(tctx, ECC_CURVE_NIST_P384);
-}
-
-static void kmb_ocs_ecdh_exit_tfm(struct crypto_kpp *tfm)
-{
- struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
-
- memzero_explicit(tctx->private_key, sizeof(*tctx->private_key));
-}
-
-static unsigned int kmb_ocs_ecdh_max_size(struct crypto_kpp *tfm)
-{
- struct ocs_ecc_ctx *tctx = kpp_tfm_ctx(tfm);
-
- /* Public key is made of two coordinates, so double the digits. */
- return digits_to_bytes(tctx->curve->g.ndigits) * 2;
-}
-
-static struct kpp_alg ocs_ecdh_p256 = {
- .set_secret = kmb_ocs_ecdh_set_secret,
- .generate_public_key = kmb_ocs_ecdh_generate_public_key,
- .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,
- .init = kmb_ocs_ecdh_nist_p256_init_tfm,
- .exit = kmb_ocs_ecdh_exit_tfm,
- .max_size = kmb_ocs_ecdh_max_size,
- .base = {
- .cra_name = "ecdh-nist-p256",
- .cra_driver_name = "ecdh-nist-p256-keembay-ocs",
- .cra_priority = KMB_OCS_ECC_PRIORITY,
- .cra_module = THIS_MODULE,
- .cra_ctxsize = sizeof(struct ocs_ecc_ctx),
- },
-};
-
-static struct kpp_alg ocs_ecdh_p384 = {
- .set_secret = kmb_ocs_ecdh_set_secret,
- .generate_public_key = kmb_ocs_ecdh_generate_public_key,
- .compute_shared_secret = kmb_ocs_ecdh_compute_shared_secret,
- .init = kmb_ocs_ecdh_nist_p384_init_tfm,
- .exit = kmb_ocs_ecdh_exit_tfm,
- .max_size = kmb_ocs_ecdh_max_size,
- .base = {
- .cra_name = "ecdh-nist-p384",
- .cra_driver_name = "ecdh-nist-p384-keembay-ocs",
- .cra_priority = KMB_OCS_ECC_PRIORITY,
- .cra_module = THIS_MODULE,
- .cra_ctxsize = sizeof(struct ocs_ecc_ctx),
- },
-};
-
-static irqreturn_t ocs_ecc_irq_handler(int irq, void *dev_id)
-{
- struct ocs_ecc_dev *ecc_dev = dev_id;
- u32 status;
-
- /*
- * Read the status register and write it back to clear the
- * DONE_INT_STATUS bit.
- */
- status = ioread32(ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);
- iowrite32(status, ecc_dev->base_reg + HW_OFFS_OCS_ECC_ISR);
-
- if (!(status & HW_OCS_ECC_ISR_INT_STATUS_DONE))
- return IRQ_NONE;
-
- complete(&ecc_dev->irq_done);
-
- return IRQ_HANDLED;
-}
-
-static int kmb_ocs_ecc_probe(struct platform_device *pdev)
-{
- struct device *dev = &pdev->dev;
- struct ocs_ecc_dev *ecc_dev;
- int rc;
-
- ecc_dev = devm_kzalloc(dev, sizeof(*ecc_dev), GFP_KERNEL);
- if (!ecc_dev)
- return -ENOMEM;
-
- ecc_dev->dev = dev;
-
- platform_set_drvdata(pdev, ecc_dev);
-
- INIT_LIST_HEAD(&ecc_dev->list);
- init_completion(&ecc_dev->irq_done);
-
- /* Get base register address. */
- ecc_dev->base_reg = devm_platform_ioremap_resource(pdev, 0);
- if (IS_ERR(ecc_dev->base_reg)) {
- dev_err(dev, "Failed to get base address\n");
- rc = PTR_ERR(ecc_dev->base_reg);
- goto list_del;
- }
-
- /* Get and request IRQ */
- ecc_dev->irq = platform_get_irq(pdev, 0);
- if (ecc_dev->irq < 0) {
- rc = ecc_dev->irq;
- goto list_del;
- }
-
- rc = devm_request_threaded_irq(dev, ecc_dev->irq, ocs_ecc_irq_handler,
- NULL, 0, "keembay-ocs-ecc", ecc_dev);
- if (rc < 0) {
- dev_err(dev, "Could not request IRQ\n");
- goto list_del;
- }
-
- /* Add device to the list of OCS ECC devices. */
- spin_lock(&ocs_ecc.lock);
- list_add_tail(&ecc_dev->list, &ocs_ecc.dev_list);
- spin_unlock(&ocs_ecc.lock);
-
- /* Initialize crypto engine. */
- ecc_dev->engine = crypto_engine_alloc_init(dev, 1);
- if (!ecc_dev->engine) {
- dev_err(dev, "Could not allocate crypto engine\n");
- rc = -ENOMEM;
- goto list_del;
- }
-
- rc = crypto_engine_start(ecc_dev->engine);
- if (rc) {
- dev_err(dev, "Could not start crypto engine\n");
- goto cleanup;
- }
-
- /* Register the KPP algo. */
- rc = crypto_register_kpp(&ocs_ecdh_p256);
- if (rc) {
- dev_err(dev,
- "Could not register OCS algorithms with Crypto API\n");
- goto cleanup;
- }
-
- rc = crypto_register_kpp(&ocs_ecdh_p384);
- if (rc) {
- dev_err(dev,
- "Could not register OCS algorithms with Crypto API\n");
- goto ocs_ecdh_p384_error;
- }
-
- return 0;
-
-ocs_ecdh_p384_error:
- crypto_unregister_kpp(&ocs_ecdh_p256);
-
-cleanup:
- crypto_engine_exit(ecc_dev->engine);
-
-list_del:
- spin_lock(&ocs_ecc.lock);
- list_del(&ecc_dev->list);
- spin_unlock(&ocs_ecc.lock);
-
- return rc;
-}
-
-static int kmb_ocs_ecc_remove(struct platform_device *pdev)
-{
- struct ocs_ecc_dev *ecc_dev;
-
- ecc_dev = platform_get_drvdata(pdev);
-
- crypto_unregister_kpp(&ocs_ecdh_p384);
- crypto_unregister_kpp(&ocs_ecdh_p256);
-
- spin_lock(&ocs_ecc.lock);
- list_del(&ecc_dev->list);
- spin_unlock(&ocs_ecc.lock);
-
- crypto_engine_exit(ecc_dev->engine);
-
- return 0;
-}
-
-/* Device tree driver match. */
-static const struct of_device_id kmb_ocs_ecc_of_match[] = {
- {
- .compatible = "intel,keembay-ocs-ecc",
- },
- {}
-};
-
-/* The OCS driver is a platform device. */
-static struct platform_driver kmb_ocs_ecc_driver = {
- .probe = kmb_ocs_ecc_probe,
- .remove = kmb_ocs_ecc_remove,
- .driver = {
- .name = DRV_NAME,
- .of_match_table = kmb_ocs_ecc_of_match,
- },
-};
-module_platform_driver(kmb_ocs_ecc_driver);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Intel Keem Bay OCS ECC Driver");
-MODULE_ALIAS_CRYPTO("ecdh-nist-p256");
-MODULE_ALIAS_CRYPTO("ecdh-nist-p384");
-MODULE_ALIAS_CRYPTO("ecdh-nist-p256-keembay-ocs");
-MODULE_ALIAS_CRYPTO("ecdh-nist-p384-keembay-ocs");
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Intel Keem Bay OCS HCU Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#include <linux/completion.h>
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/interrupt.h>
-#include <linux/module.h>
-#include <linux/of_device.h>
-
-#include <crypto/engine.h>
-#include <crypto/scatterwalk.h>
-#include <crypto/sha2.h>
-#include <crypto/sm3.h>
-#include <crypto/hmac.h>
-#include <crypto/internal/hash.h>
-
-#include "ocs-hcu.h"
-
-#define DRV_NAME "keembay-ocs-hcu"
-
-/* Flag marking a final request. */
-#define REQ_FINAL BIT(0)
-/* Flag marking a HMAC request. */
-#define REQ_FLAGS_HMAC BIT(1)
-/* Flag set when HW HMAC is being used. */
-#define REQ_FLAGS_HMAC_HW BIT(2)
-/* Flag set when SW HMAC is being used. */
-#define REQ_FLAGS_HMAC_SW BIT(3)
-
-/**
- * struct ocs_hcu_ctx: OCS HCU Transform context.
- * @engine_ctx: Crypto Engine context.
- * @hcu_dev: The OCS HCU device used by the transformation.
- * @key: The key (used only for HMAC transformations).
- * @key_len: The length of the key.
- * @is_sm3_tfm: Whether or not this is an SM3 transformation.
- * @is_hmac_tfm: Whether or not this is a HMAC transformation.
- */
-struct ocs_hcu_ctx {
- struct crypto_engine_ctx engine_ctx;
- struct ocs_hcu_dev *hcu_dev;
- u8 key[SHA512_BLOCK_SIZE];
- size_t key_len;
- bool is_sm3_tfm;
- bool is_hmac_tfm;
-};
-
-/**
- * struct ocs_hcu_rctx - Context for the request.
- * @hcu_dev: OCS HCU device to be used to service the request.
- * @flags: Flags tracking request status.
- * @algo: Algorithm to use for the request.
- * @blk_sz: Block size of the transformation / request.
- * @dig_sz: Digest size of the transformation / request.
- * @dma_list: OCS DMA linked list.
- * @hash_ctx: OCS HCU hashing context.
- * @buffer: Buffer to store: partial block of data and SW HMAC
- * artifacts (ipad, opad, etc.).
- * @buf_cnt: Number of bytes currently stored in the buffer.
- * @buf_dma_addr: The DMA address of @buffer (when mapped).
- * @buf_dma_count: The number of bytes in @buffer currently DMA-mapped.
- * @sg: Head of the scatterlist entries containing data.
- * @sg_data_total: Total data in the SG list at any time.
- * @sg_data_offset: Offset into the data of the current individual SG node.
- * @sg_dma_nents: Number of sg entries mapped in dma_list.
- */
-struct ocs_hcu_rctx {
- struct ocs_hcu_dev *hcu_dev;
- u32 flags;
- enum ocs_hcu_algo algo;
- size_t blk_sz;
- size_t dig_sz;
- struct ocs_hcu_dma_list *dma_list;
- struct ocs_hcu_hash_ctx hash_ctx;
- /*
- * Buffer is double the block size because we need space for SW HMAC
- * artifacts, i.e:
- * - ipad (1 block) + a possible partial block of data.
- * - opad (1 block) + digest of H(k ^ ipad || m)
- */
- u8 buffer[2 * SHA512_BLOCK_SIZE];
- size_t buf_cnt;
- dma_addr_t buf_dma_addr;
- size_t buf_dma_count;
- struct scatterlist *sg;
- unsigned int sg_data_total;
- unsigned int sg_data_offset;
- unsigned int sg_dma_nents;
-};
-
-/**
- * struct ocs_hcu_drv - Driver data
- * @dev_list: The list of HCU devices.
- * @lock: The lock protecting dev_list.
- */
-struct ocs_hcu_drv {
- struct list_head dev_list;
- spinlock_t lock; /* Protects dev_list. */
-};
-
-static struct ocs_hcu_drv ocs_hcu = {
- .dev_list = LIST_HEAD_INIT(ocs_hcu.dev_list),
- .lock = __SPIN_LOCK_UNLOCKED(ocs_hcu.lock),
-};
-
-/*
- * Return the total amount of data in the request; that is: the data in the
- * request buffer + the data in the sg list.
- */
-static inline unsigned int kmb_get_total_data(struct ocs_hcu_rctx *rctx)
-{
- return rctx->sg_data_total + rctx->buf_cnt;
-}
-
-/* Move remaining content of scatter-gather list to context buffer. */
-static int flush_sg_to_ocs_buffer(struct ocs_hcu_rctx *rctx)
-{
- size_t count;
-
- if (rctx->sg_data_total > (sizeof(rctx->buffer) - rctx->buf_cnt)) {
- WARN(1, "%s: sg data does not fit in buffer\n", __func__);
- return -EINVAL;
- }
-
- while (rctx->sg_data_total) {
- if (!rctx->sg) {
- WARN(1, "%s: unexpected NULL sg\n", __func__);
- return -EINVAL;
- }
- /*
- * If current sg has been fully processed, skip to the next
- * one.
- */
- if (rctx->sg_data_offset == rctx->sg->length) {
- rctx->sg = sg_next(rctx->sg);
- rctx->sg_data_offset = 0;
- continue;
- }
- /*
- * Determine the maximum data available to copy from the node.
- * Minimum of the length left in the sg node, or the total data
- * in the request.
- */
- count = min(rctx->sg->length - rctx->sg_data_offset,
- rctx->sg_data_total);
- /* Copy from scatter-list entry to context buffer. */
- scatterwalk_map_and_copy(&rctx->buffer[rctx->buf_cnt],
- rctx->sg, rctx->sg_data_offset,
- count, 0);
-
- rctx->sg_data_offset += count;
- rctx->sg_data_total -= count;
- rctx->buf_cnt += count;
- }
-
- return 0;
-}
-
-static struct ocs_hcu_dev *kmb_ocs_hcu_find_dev(struct ahash_request *req)
-{
- struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
-
- /* If the HCU device for the request was previously set, return it. */
- if (tctx->hcu_dev)
- return tctx->hcu_dev;
-
- /*
- * Otherwise, get the first HCU device available (there should be one
- * and only one device).
- */
- spin_lock_bh(&ocs_hcu.lock);
- tctx->hcu_dev = list_first_entry_or_null(&ocs_hcu.dev_list,
- struct ocs_hcu_dev,
- list);
- spin_unlock_bh(&ocs_hcu.lock);
-
- return tctx->hcu_dev;
-}
-
-/* Free OCS DMA linked list and DMA-able context buffer. */
-static void kmb_ocs_hcu_dma_cleanup(struct ahash_request *req,
- struct ocs_hcu_rctx *rctx)
-{
- struct ocs_hcu_dev *hcu_dev = rctx->hcu_dev;
- struct device *dev = hcu_dev->dev;
-
- /* Unmap rctx->buffer (if mapped). */
- if (rctx->buf_dma_count) {
- dma_unmap_single(dev, rctx->buf_dma_addr, rctx->buf_dma_count,
- DMA_TO_DEVICE);
- rctx->buf_dma_count = 0;
- }
-
- /* Unmap req->src (if mapped). */
- if (rctx->sg_dma_nents) {
- dma_unmap_sg(dev, req->src, rctx->sg_dma_nents, DMA_TO_DEVICE);
- rctx->sg_dma_nents = 0;
- }
-
- /* Free dma_list (if allocated). */
- if (rctx->dma_list) {
- ocs_hcu_dma_list_free(hcu_dev, rctx->dma_list);
- rctx->dma_list = NULL;
- }
-}
-
-/*
- * Prepare for DMA operation:
- * - DMA-map request context buffer (if needed)
- * - DMA-map SG list (only the entries to be processed, see note below)
- * - Allocate OCS HCU DMA linked list (number of elements = SG entries to
- * process + context buffer (if not empty)).
- * - Add DMA-mapped request context buffer to OCS HCU DMA list.
- * - Add SG entries to DMA list.
- *
- * Note: if this is a final request, we process all the data in the SG list,
- * otherwise we can only process up to the maximum amount of block-aligned data
- * (the remainder will be put into the context buffer and processed in the next
- * request).
- */
-static int kmb_ocs_dma_prepare(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- struct device *dev = rctx->hcu_dev->dev;
- unsigned int remainder = 0;
- unsigned int total;
- size_t nents;
- size_t count;
- int rc;
- int i;
-
- /* This function should be called only when there is data to process. */
- total = kmb_get_total_data(rctx);
- if (!total)
- return -EINVAL;
-
- /*
- * If this is not a final DMA (terminated DMA), the data passed to the
- * HCU must be aligned to the block size; compute the remainder data to
- * be processed in the next request.
- */
- if (!(rctx->flags & REQ_FINAL))
- remainder = total % rctx->blk_sz;
-
- /* Determine the number of scatter gather list entries to process. */
- nents = sg_nents_for_len(req->src, rctx->sg_data_total - remainder);
-
- /* If there are entries to process, map them. */
- if (nents) {
- rctx->sg_dma_nents = dma_map_sg(dev, req->src, nents,
- DMA_TO_DEVICE);
- if (!rctx->sg_dma_nents) {
- dev_err(dev, "Failed to MAP SG\n");
- rc = -ENOMEM;
- goto cleanup;
- }
- /*
- * The value returned by dma_map_sg() can be < nents; so update
- * nents accordingly.
- */
- nents = rctx->sg_dma_nents;
- }
-
- /*
- * If context buffer is not empty, map it and add extra DMA entry for
- * it.
- */
- if (rctx->buf_cnt) {
- rctx->buf_dma_addr = dma_map_single(dev, rctx->buffer,
- rctx->buf_cnt,
- DMA_TO_DEVICE);
- if (dma_mapping_error(dev, rctx->buf_dma_addr)) {
- dev_err(dev, "Failed to map request context buffer\n");
- rc = -ENOMEM;
- goto cleanup;
- }
- rctx->buf_dma_count = rctx->buf_cnt;
- /* Increase number of dma entries. */
- nents++;
- }
-
- /* Allocate OCS HCU DMA list. */
- rctx->dma_list = ocs_hcu_dma_list_alloc(rctx->hcu_dev, nents);
- if (!rctx->dma_list) {
- rc = -ENOMEM;
- goto cleanup;
- }
-
- /* Add request context buffer (if previously DMA-mapped) */
- if (rctx->buf_dma_count) {
- rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev, rctx->dma_list,
- rctx->buf_dma_addr,
- rctx->buf_dma_count);
- if (rc)
- goto cleanup;
- }
-
- /* Add the SG nodes to be processed to the DMA linked list. */
- for_each_sg(req->src, rctx->sg, rctx->sg_dma_nents, i) {
- /*
- * The number of bytes to add to the list entry is the minimum
- * between:
- * - The DMA length of the SG entry.
- * - The data left to be processed.
- */
- count = min(rctx->sg_data_total - remainder,
- sg_dma_len(rctx->sg) - rctx->sg_data_offset);
- /*
- * Do not create a zero length DMA descriptor. Check in case of
- * zero length SG node.
- */
- if (count == 0)
- continue;
- /* Add sg to HCU DMA list. */
- rc = ocs_hcu_dma_list_add_tail(rctx->hcu_dev,
- rctx->dma_list,
- rctx->sg->dma_address,
- count);
- if (rc)
- goto cleanup;
-
- /* Update amount of data remaining in SG list. */
- rctx->sg_data_total -= count;
-
- /*
- * If remaining data is equal to remainder (note: 'less than'
- * case should never happen in practice), we are done: update
- * offset and exit the loop.
- */
- if (rctx->sg_data_total <= remainder) {
- WARN_ON(rctx->sg_data_total < remainder);
- rctx->sg_data_offset += count;
- break;
- }
-
- /*
- * If we get here is because we need to process the next sg in
- * the list; set offset within the sg to 0.
- */
- rctx->sg_data_offset = 0;
- }
-
- return 0;
-cleanup:
- dev_err(dev, "Failed to prepare DMA.\n");
- kmb_ocs_hcu_dma_cleanup(req, rctx);
-
- return rc;
-}
-
-static void kmb_ocs_hcu_secure_cleanup(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
-
- /* Clear buffer of any data. */
- memzero_explicit(rctx->buffer, sizeof(rctx->buffer));
-}
-
-static int kmb_ocs_hcu_handle_queue(struct ahash_request *req)
-{
- struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
-
- if (!hcu_dev)
- return -ENOENT;
-
- return crypto_transfer_hash_request_to_engine(hcu_dev->engine, req);
-}
-
-static int prepare_ipad(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
- int i;
-
- WARN(rctx->buf_cnt, "%s: Context buffer is not empty\n", __func__);
- WARN(!(rctx->flags & REQ_FLAGS_HMAC_SW),
- "%s: HMAC_SW flag is not set\n", __func__);
- /*
- * Key length must be equal to block size. If key is shorter,
- * we pad it with zero (note: key cannot be longer, since
- * longer keys are hashed by kmb_ocs_hcu_setkey()).
- */
- if (ctx->key_len > rctx->blk_sz) {
- WARN(1, "%s: Invalid key length in tfm context\n", __func__);
- return -EINVAL;
- }
- memzero_explicit(&ctx->key[ctx->key_len],
- rctx->blk_sz - ctx->key_len);
- ctx->key_len = rctx->blk_sz;
- /*
- * Prepare IPAD for HMAC. Only done for first block.
- * HMAC(k,m) = H(k ^ opad || H(k ^ ipad || m))
- * k ^ ipad will be first hashed block.
- * k ^ opad will be calculated in the final request.
- * Only needed if not using HW HMAC.
- */
- for (i = 0; i < rctx->blk_sz; i++)
- rctx->buffer[i] = ctx->key[i] ^ HMAC_IPAD_VALUE;
- rctx->buf_cnt = rctx->blk_sz;
-
- return 0;
-}
-
-static int kmb_ocs_hcu_do_one_request(struct crypto_engine *engine, void *areq)
-{
- struct ahash_request *req = container_of(areq, struct ahash_request,
- base);
- struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
- struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- struct ocs_hcu_ctx *tctx = crypto_ahash_ctx(tfm);
- int rc;
- int i;
-
- if (!hcu_dev) {
- rc = -ENOENT;
- goto error;
- }
-
- /*
- * If hardware HMAC flag is set, perform HMAC in hardware.
- *
- * NOTE: this flag implies REQ_FINAL && kmb_get_total_data(rctx)
- */
- if (rctx->flags & REQ_FLAGS_HMAC_HW) {
- /* Map input data into the HCU DMA linked list. */
- rc = kmb_ocs_dma_prepare(req);
- if (rc)
- goto error;
-
- rc = ocs_hcu_hmac(hcu_dev, rctx->algo, tctx->key, tctx->key_len,
- rctx->dma_list, req->result, rctx->dig_sz);
-
- /* Unmap data and free DMA list regardless of return code. */
- kmb_ocs_hcu_dma_cleanup(req, rctx);
-
- /* Process previous return code. */
- if (rc)
- goto error;
-
- goto done;
- }
-
- /* Handle update request case. */
- if (!(rctx->flags & REQ_FINAL)) {
- /* Update should always have input data. */
- if (!kmb_get_total_data(rctx))
- return -EINVAL;
-
- /* Map input data into the HCU DMA linked list. */
- rc = kmb_ocs_dma_prepare(req);
- if (rc)
- goto error;
-
- /* Do hashing step. */
- rc = ocs_hcu_hash_update(hcu_dev, &rctx->hash_ctx,
- rctx->dma_list);
-
- /* Unmap data and free DMA list regardless of return code. */
- kmb_ocs_hcu_dma_cleanup(req, rctx);
-
- /* Process previous return code. */
- if (rc)
- goto error;
-
- /*
- * Reset request buffer count (data in the buffer was just
- * processed).
- */
- rctx->buf_cnt = 0;
- /*
- * Move remaining sg data into the request buffer, so that it
- * will be processed during the next request.
- *
- * NOTE: we have remaining data if kmb_get_total_data() was not
- * a multiple of block size.
- */
- rc = flush_sg_to_ocs_buffer(rctx);
- if (rc)
- goto error;
-
- goto done;
- }
-
- /* If we get here, this is a final request. */
-
- /* If there is data to process, use finup. */
- if (kmb_get_total_data(rctx)) {
- /* Map input data into the HCU DMA linked list. */
- rc = kmb_ocs_dma_prepare(req);
- if (rc)
- goto error;
-
- /* Do hashing step. */
- rc = ocs_hcu_hash_finup(hcu_dev, &rctx->hash_ctx,
- rctx->dma_list,
- req->result, rctx->dig_sz);
- /* Free DMA list regardless of return code. */
- kmb_ocs_hcu_dma_cleanup(req, rctx);
-
- /* Process previous return code. */
- if (rc)
- goto error;
-
- } else { /* Otherwise (if we have no data), use final. */
- rc = ocs_hcu_hash_final(hcu_dev, &rctx->hash_ctx, req->result,
- rctx->dig_sz);
- if (rc)
- goto error;
- }
-
- /*
- * If we are finalizing a SW HMAC request, we just computed the result
- * of: H(k ^ ipad || m).
- *
- * We now need to complete the HMAC calculation with the OPAD step,
- * that is, we need to compute H(k ^ opad || digest), where digest is
- * the digest we just obtained, i.e., H(k ^ ipad || m).
- */
- if (rctx->flags & REQ_FLAGS_HMAC_SW) {
- /*
- * Compute k ^ opad and store it in the request buffer (which
- * is not used anymore at this point).
- * Note: key has been padded / hashed already (so keylen ==
- * blksz) .
- */
- WARN_ON(tctx->key_len != rctx->blk_sz);
- for (i = 0; i < rctx->blk_sz; i++)
- rctx->buffer[i] = tctx->key[i] ^ HMAC_OPAD_VALUE;
- /* Now append the digest to the rest of the buffer. */
- for (i = 0; (i < rctx->dig_sz); i++)
- rctx->buffer[rctx->blk_sz + i] = req->result[i];
-
- /* Now hash the buffer to obtain the final HMAC. */
- rc = ocs_hcu_digest(hcu_dev, rctx->algo, rctx->buffer,
- rctx->blk_sz + rctx->dig_sz, req->result,
- rctx->dig_sz);
- if (rc)
- goto error;
- }
-
- /* Perform secure clean-up. */
- kmb_ocs_hcu_secure_cleanup(req);
-done:
- crypto_finalize_hash_request(hcu_dev->engine, req, 0);
-
- return 0;
-
-error:
- kmb_ocs_hcu_secure_cleanup(req);
- return rc;
-}
-
-static int kmb_ocs_hcu_init(struct ahash_request *req)
-{
- struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
-
- if (!hcu_dev)
- return -ENOENT;
-
- /* Initialize entire request context to zero. */
- memset(rctx, 0, sizeof(*rctx));
-
- rctx->hcu_dev = hcu_dev;
- rctx->dig_sz = crypto_ahash_digestsize(tfm);
-
- switch (rctx->dig_sz) {
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
- case SHA224_DIGEST_SIZE:
- rctx->blk_sz = SHA224_BLOCK_SIZE;
- rctx->algo = OCS_HCU_ALGO_SHA224;
- break;
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
- case SHA256_DIGEST_SIZE:
- rctx->blk_sz = SHA256_BLOCK_SIZE;
- /*
- * SHA256 and SM3 have the same digest size: use info from tfm
- * context to find out which one we should use.
- */
- rctx->algo = ctx->is_sm3_tfm ? OCS_HCU_ALGO_SM3 :
- OCS_HCU_ALGO_SHA256;
- break;
- case SHA384_DIGEST_SIZE:
- rctx->blk_sz = SHA384_BLOCK_SIZE;
- rctx->algo = OCS_HCU_ALGO_SHA384;
- break;
- case SHA512_DIGEST_SIZE:
- rctx->blk_sz = SHA512_BLOCK_SIZE;
- rctx->algo = OCS_HCU_ALGO_SHA512;
- break;
- default:
- return -EINVAL;
- }
-
- /* Initialize intermediate data. */
- ocs_hcu_hash_init(&rctx->hash_ctx, rctx->algo);
-
- /* If this a HMAC request, set HMAC flag. */
- if (ctx->is_hmac_tfm)
- rctx->flags |= REQ_FLAGS_HMAC;
-
- return 0;
-}
-
-static int kmb_ocs_hcu_update(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- int rc;
-
- if (!req->nbytes)
- return 0;
-
- rctx->sg_data_total = req->nbytes;
- rctx->sg_data_offset = 0;
- rctx->sg = req->src;
-
- /*
- * If we are doing HMAC, then we must use SW-assisted HMAC, since HW
- * HMAC does not support context switching (there it can only be used
- * with finup() or digest()).
- */
- if (rctx->flags & REQ_FLAGS_HMAC &&
- !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
- rctx->flags |= REQ_FLAGS_HMAC_SW;
- rc = prepare_ipad(req);
- if (rc)
- return rc;
- }
-
- /*
- * If remaining sg_data fits into ctx buffer, just copy it there; we'll
- * process it at the next update() or final().
- */
- if (rctx->sg_data_total <= (sizeof(rctx->buffer) - rctx->buf_cnt))
- return flush_sg_to_ocs_buffer(rctx);
-
- return kmb_ocs_hcu_handle_queue(req);
-}
-
-/* Common logic for kmb_ocs_hcu_final() and kmb_ocs_hcu_finup(). */
-static int kmb_ocs_hcu_fin_common(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
- struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
- struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
- int rc;
-
- rctx->flags |= REQ_FINAL;
-
- /*
- * If this is a HMAC request and, so far, we didn't have to switch to
- * SW HMAC, check if we can use HW HMAC.
- */
- if (rctx->flags & REQ_FLAGS_HMAC &&
- !(rctx->flags & REQ_FLAGS_HMAC_SW)) {
- /*
- * If we are here, it means we never processed any data so far,
- * so we can use HW HMAC, but only if there is some data to
- * process (since OCS HW MAC does not support zero-length
- * messages) and the key length is supported by the hardware
- * (OCS HCU HW only supports length <= 64); if HW HMAC cannot
- * be used, fall back to SW-assisted HMAC.
- */
- if (kmb_get_total_data(rctx) &&
- ctx->key_len <= OCS_HCU_HW_KEY_LEN) {
- rctx->flags |= REQ_FLAGS_HMAC_HW;
- } else {
- rctx->flags |= REQ_FLAGS_HMAC_SW;
- rc = prepare_ipad(req);
- if (rc)
- return rc;
- }
- }
-
- return kmb_ocs_hcu_handle_queue(req);
-}
-
-static int kmb_ocs_hcu_final(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
-
- rctx->sg_data_total = 0;
- rctx->sg_data_offset = 0;
- rctx->sg = NULL;
-
- return kmb_ocs_hcu_fin_common(req);
-}
-
-static int kmb_ocs_hcu_finup(struct ahash_request *req)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
-
- rctx->sg_data_total = req->nbytes;
- rctx->sg_data_offset = 0;
- rctx->sg = req->src;
-
- return kmb_ocs_hcu_fin_common(req);
-}
-
-static int kmb_ocs_hcu_digest(struct ahash_request *req)
-{
- int rc = 0;
- struct ocs_hcu_dev *hcu_dev = kmb_ocs_hcu_find_dev(req);
-
- if (!hcu_dev)
- return -ENOENT;
-
- rc = kmb_ocs_hcu_init(req);
- if (rc)
- return rc;
-
- rc = kmb_ocs_hcu_finup(req);
-
- return rc;
-}
-
-static int kmb_ocs_hcu_export(struct ahash_request *req, void *out)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
-
- /* Intermediate data is always stored and applied per request. */
- memcpy(out, rctx, sizeof(*rctx));
-
- return 0;
-}
-
-static int kmb_ocs_hcu_import(struct ahash_request *req, const void *in)
-{
- struct ocs_hcu_rctx *rctx = ahash_request_ctx_dma(req);
-
- /* Intermediate data is always stored and applied per request. */
- memcpy(rctx, in, sizeof(*rctx));
-
- return 0;
-}
-
-static int kmb_ocs_hcu_setkey(struct crypto_ahash *tfm, const u8 *key,
- unsigned int keylen)
-{
- unsigned int digestsize = crypto_ahash_digestsize(tfm);
- struct ocs_hcu_ctx *ctx = crypto_ahash_ctx(tfm);
- size_t blk_sz = crypto_ahash_blocksize(tfm);
- struct crypto_ahash *ahash_tfm;
- struct ahash_request *req;
- struct crypto_wait wait;
- struct scatterlist sg;
- const char *alg_name;
- int rc;
-
- /*
- * Key length must be equal to block size:
- * - If key is shorter, we are done for now (the key will be padded
- * later on); this is to maximize the use of HW HMAC (which works
- * only for keys <= 64 bytes).
- * - If key is longer, we hash it.
- */
- if (keylen <= blk_sz) {
- memcpy(ctx->key, key, keylen);
- ctx->key_len = keylen;
- return 0;
- }
-
- switch (digestsize) {
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
- case SHA224_DIGEST_SIZE:
- alg_name = "sha224-keembay-ocs";
- break;
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
- case SHA256_DIGEST_SIZE:
- alg_name = ctx->is_sm3_tfm ? "sm3-keembay-ocs" :
- "sha256-keembay-ocs";
- break;
- case SHA384_DIGEST_SIZE:
- alg_name = "sha384-keembay-ocs";
- break;
- case SHA512_DIGEST_SIZE:
- alg_name = "sha512-keembay-ocs";
- break;
- default:
- return -EINVAL;
- }
-
- ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0);
- if (IS_ERR(ahash_tfm))
- return PTR_ERR(ahash_tfm);
-
- req = ahash_request_alloc(ahash_tfm, GFP_KERNEL);
- if (!req) {
- rc = -ENOMEM;
- goto err_free_ahash;
- }
-
- crypto_init_wait(&wait);
- ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
- crypto_req_done, &wait);
- crypto_ahash_clear_flags(ahash_tfm, ~0);
-
- sg_init_one(&sg, key, keylen);
- ahash_request_set_crypt(req, &sg, ctx->key, keylen);
-
- rc = crypto_wait_req(crypto_ahash_digest(req), &wait);
- if (rc == 0)
- ctx->key_len = digestsize;
-
- ahash_request_free(req);
-err_free_ahash:
- crypto_free_ahash(ahash_tfm);
-
- return rc;
-}
-
-/* Set request size and initialize tfm context. */
-static void __cra_init(struct crypto_tfm *tfm, struct ocs_hcu_ctx *ctx)
-{
- crypto_ahash_set_reqsize_dma(__crypto_ahash_cast(tfm),
- sizeof(struct ocs_hcu_rctx));
-
- /* Init context to 0. */
- memzero_explicit(ctx, sizeof(*ctx));
- /* Set engine ops. */
- ctx->engine_ctx.op.do_one_request = kmb_ocs_hcu_do_one_request;
-}
-
-static int kmb_ocs_hcu_sha_cra_init(struct crypto_tfm *tfm)
-{
- struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
-
- __cra_init(tfm, ctx);
-
- return 0;
-}
-
-static int kmb_ocs_hcu_sm3_cra_init(struct crypto_tfm *tfm)
-{
- struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
-
- __cra_init(tfm, ctx);
-
- ctx->is_sm3_tfm = true;
-
- return 0;
-}
-
-static int kmb_ocs_hcu_hmac_sm3_cra_init(struct crypto_tfm *tfm)
-{
- struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
-
- __cra_init(tfm, ctx);
-
- ctx->is_sm3_tfm = true;
- ctx->is_hmac_tfm = true;
-
- return 0;
-}
-
-static int kmb_ocs_hcu_hmac_cra_init(struct crypto_tfm *tfm)
-{
- struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
-
- __cra_init(tfm, ctx);
-
- ctx->is_hmac_tfm = true;
-
- return 0;
-}
-
-/* Function called when 'tfm' is de-initialized. */
-static void kmb_ocs_hcu_hmac_cra_exit(struct crypto_tfm *tfm)
-{
- struct ocs_hcu_ctx *ctx = crypto_tfm_ctx(tfm);
-
- /* Clear the key. */
- memzero_explicit(ctx->key, sizeof(ctx->key));
-}
-
-static struct ahash_alg ocs_hcu_algs[] = {
-#ifdef CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .halg = {
- .digestsize = SHA224_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "sha224",
- .cra_driver_name = "sha224-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA224_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_sha_cra_init,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .setkey = kmb_ocs_hcu_setkey,
- .halg = {
- .digestsize = SHA224_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "hmac(sha224)",
- .cra_driver_name = "hmac-sha224-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA224_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_hmac_cra_init,
- .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
- }
- }
-},
-#endif /* CONFIG_CRYPTO_DEV_KEEMBAY_OCS_HCU_HMAC_SHA224 */
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .halg = {
- .digestsize = SHA256_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "sha256",
- .cra_driver_name = "sha256-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_sha_cra_init,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .setkey = kmb_ocs_hcu_setkey,
- .halg = {
- .digestsize = SHA256_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "hmac(sha256)",
- .cra_driver_name = "hmac-sha256-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA256_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_hmac_cra_init,
- .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .halg = {
- .digestsize = SM3_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "sm3",
- .cra_driver_name = "sm3-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SM3_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_sm3_cra_init,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .setkey = kmb_ocs_hcu_setkey,
- .halg = {
- .digestsize = SM3_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "hmac(sm3)",
- .cra_driver_name = "hmac-sm3-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SM3_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_hmac_sm3_cra_init,
- .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .halg = {
- .digestsize = SHA384_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "sha384",
- .cra_driver_name = "sha384-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA384_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_sha_cra_init,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .setkey = kmb_ocs_hcu_setkey,
- .halg = {
- .digestsize = SHA384_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "hmac(sha384)",
- .cra_driver_name = "hmac-sha384-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA384_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_hmac_cra_init,
- .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .halg = {
- .digestsize = SHA512_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "sha512",
- .cra_driver_name = "sha512-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_sha_cra_init,
- }
- }
-},
-{
- .init = kmb_ocs_hcu_init,
- .update = kmb_ocs_hcu_update,
- .final = kmb_ocs_hcu_final,
- .finup = kmb_ocs_hcu_finup,
- .digest = kmb_ocs_hcu_digest,
- .export = kmb_ocs_hcu_export,
- .import = kmb_ocs_hcu_import,
- .setkey = kmb_ocs_hcu_setkey,
- .halg = {
- .digestsize = SHA512_DIGEST_SIZE,
- .statesize = sizeof(struct ocs_hcu_rctx),
- .base = {
- .cra_name = "hmac(sha512)",
- .cra_driver_name = "hmac-sha512-keembay-ocs",
- .cra_priority = 255,
- .cra_flags = CRYPTO_ALG_ASYNC,
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct ocs_hcu_ctx),
- .cra_alignmask = 0,
- .cra_module = THIS_MODULE,
- .cra_init = kmb_ocs_hcu_hmac_cra_init,
- .cra_exit = kmb_ocs_hcu_hmac_cra_exit,
- }
- }
-},
-};
-
-/* Device tree driver match. */
-static const struct of_device_id kmb_ocs_hcu_of_match[] = {
- {
- .compatible = "intel,keembay-ocs-hcu",
- },
- {}
-};
-
-static int kmb_ocs_hcu_remove(struct platform_device *pdev)
-{
- struct ocs_hcu_dev *hcu_dev;
- int rc;
-
- hcu_dev = platform_get_drvdata(pdev);
- if (!hcu_dev)
- return -ENODEV;
-
- crypto_unregister_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
-
- rc = crypto_engine_exit(hcu_dev->engine);
-
- spin_lock_bh(&ocs_hcu.lock);
- list_del(&hcu_dev->list);
- spin_unlock_bh(&ocs_hcu.lock);
-
- return rc;
-}
-
-static int kmb_ocs_hcu_probe(struct platform_device *pdev)
-{
- struct device *dev = &pdev->dev;
- struct ocs_hcu_dev *hcu_dev;
- struct resource *hcu_mem;
- int rc;
-
- hcu_dev = devm_kzalloc(dev, sizeof(*hcu_dev), GFP_KERNEL);
- if (!hcu_dev)
- return -ENOMEM;
-
- hcu_dev->dev = dev;
-
- platform_set_drvdata(pdev, hcu_dev);
- rc = dma_set_mask_and_coherent(&pdev->dev, OCS_HCU_DMA_BIT_MASK);
- if (rc)
- return rc;
-
- /* Get the memory address and remap. */
- hcu_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- if (!hcu_mem) {
- dev_err(dev, "Could not retrieve io mem resource.\n");
- return -ENODEV;
- }
-
- hcu_dev->io_base = devm_ioremap_resource(dev, hcu_mem);
- if (IS_ERR(hcu_dev->io_base))
- return PTR_ERR(hcu_dev->io_base);
-
- init_completion(&hcu_dev->irq_done);
-
- /* Get and request IRQ. */
- hcu_dev->irq = platform_get_irq(pdev, 0);
- if (hcu_dev->irq < 0)
- return hcu_dev->irq;
-
- rc = devm_request_threaded_irq(&pdev->dev, hcu_dev->irq,
- ocs_hcu_irq_handler, NULL, 0,
- "keembay-ocs-hcu", hcu_dev);
- if (rc < 0) {
- dev_err(dev, "Could not request IRQ.\n");
- return rc;
- }
-
- INIT_LIST_HEAD(&hcu_dev->list);
-
- spin_lock_bh(&ocs_hcu.lock);
- list_add_tail(&hcu_dev->list, &ocs_hcu.dev_list);
- spin_unlock_bh(&ocs_hcu.lock);
-
- /* Initialize crypto engine */
- hcu_dev->engine = crypto_engine_alloc_init(dev, 1);
- if (!hcu_dev->engine) {
- rc = -ENOMEM;
- goto list_del;
- }
-
- rc = crypto_engine_start(hcu_dev->engine);
- if (rc) {
- dev_err(dev, "Could not start engine.\n");
- goto cleanup;
- }
-
- /* Security infrastructure guarantees OCS clock is enabled. */
-
- rc = crypto_register_ahashes(ocs_hcu_algs, ARRAY_SIZE(ocs_hcu_algs));
- if (rc) {
- dev_err(dev, "Could not register algorithms.\n");
- goto cleanup;
- }
-
- return 0;
-
-cleanup:
- crypto_engine_exit(hcu_dev->engine);
-list_del:
- spin_lock_bh(&ocs_hcu.lock);
- list_del(&hcu_dev->list);
- spin_unlock_bh(&ocs_hcu.lock);
-
- return rc;
-}
-
-/* The OCS driver is a platform device. */
-static struct platform_driver kmb_ocs_hcu_driver = {
- .probe = kmb_ocs_hcu_probe,
- .remove = kmb_ocs_hcu_remove,
- .driver = {
- .name = DRV_NAME,
- .of_match_table = kmb_ocs_hcu_of_match,
- },
-};
-
-module_platform_driver(kmb_ocs_hcu_driver);
-
-MODULE_LICENSE("GPL");
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Intel Keem Bay OCS AES Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/swab.h>
-
-#include <asm/byteorder.h>
-#include <asm/errno.h>
-
-#include <crypto/aes.h>
-#include <crypto/gcm.h>
-
-#include "ocs-aes.h"
-
-#define AES_COMMAND_OFFSET 0x0000
-#define AES_KEY_0_OFFSET 0x0004
-#define AES_KEY_1_OFFSET 0x0008
-#define AES_KEY_2_OFFSET 0x000C
-#define AES_KEY_3_OFFSET 0x0010
-#define AES_KEY_4_OFFSET 0x0014
-#define AES_KEY_5_OFFSET 0x0018
-#define AES_KEY_6_OFFSET 0x001C
-#define AES_KEY_7_OFFSET 0x0020
-#define AES_IV_0_OFFSET 0x0024
-#define AES_IV_1_OFFSET 0x0028
-#define AES_IV_2_OFFSET 0x002C
-#define AES_IV_3_OFFSET 0x0030
-#define AES_ACTIVE_OFFSET 0x0034
-#define AES_STATUS_OFFSET 0x0038
-#define AES_KEY_SIZE_OFFSET 0x0044
-#define AES_IER_OFFSET 0x0048
-#define AES_ISR_OFFSET 0x005C
-#define AES_MULTIPURPOSE1_0_OFFSET 0x0200
-#define AES_MULTIPURPOSE1_1_OFFSET 0x0204
-#define AES_MULTIPURPOSE1_2_OFFSET 0x0208
-#define AES_MULTIPURPOSE1_3_OFFSET 0x020C
-#define AES_MULTIPURPOSE2_0_OFFSET 0x0220
-#define AES_MULTIPURPOSE2_1_OFFSET 0x0224
-#define AES_MULTIPURPOSE2_2_OFFSET 0x0228
-#define AES_MULTIPURPOSE2_3_OFFSET 0x022C
-#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0
-#define AES_TLEN_OFFSET 0x0300
-#define AES_T_MAC_0_OFFSET 0x0304
-#define AES_T_MAC_1_OFFSET 0x0308
-#define AES_T_MAC_2_OFFSET 0x030C
-#define AES_T_MAC_3_OFFSET 0x0310
-#define AES_PLEN_OFFSET 0x0314
-#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400
-#define AES_A_DMA_DST_ADDR_OFFSET 0x0404
-#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408
-#define AES_A_DMA_DST_SIZE_OFFSET 0x040C
-#define AES_A_DMA_DMA_MODE_OFFSET 0x0410
-#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418
-#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C
-#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420
-#define AES_A_DMA_LOG_OFFSET 0x0424
-#define AES_A_DMA_STATUS_OFFSET 0x0428
-#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C
-#define AES_A_DMA_MSI_ISR_OFFSET 0x0480
-#define AES_A_DMA_MSI_IER_OFFSET 0x0484
-#define AES_A_DMA_MSI_MASK_OFFSET 0x0488
-#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600
-#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700
-
-/*
- * AES_A_DMA_DMA_MODE register.
- * Default: 0x00000000.
- * bit[31] ACTIVE
- * This bit activates the DMA. When the DMA finishes, it resets
- * this bit to zero.
- * bit[30:26] Unused by this driver.
- * bit[25] SRC_LINK_LIST_EN
- * Source link list enable bit. When the linked list is terminated
- * this bit is reset by the DMA.
- * bit[24] DST_LINK_LIST_EN
- * Destination link list enable bit. When the linked list is
- * terminated this bit is reset by the DMA.
- * bit[23:0] Unused by this driver.
- */
-#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31)
-#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25)
-#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24)
-
-/*
- * AES_ACTIVE register
- * default 0x00000000
- * bit[31:10] Reserved
- * bit[9] LAST_ADATA
- * bit[8] LAST_GCX
- * bit[7:2] Reserved
- * bit[1] TERMINATION
- * bit[0] TRIGGER
- */
-#define AES_ACTIVE_LAST_ADATA BIT(9)
-#define AES_ACTIVE_LAST_CCM_GCM BIT(8)
-#define AES_ACTIVE_TERMINATION BIT(1)
-#define AES_ACTIVE_TRIGGER BIT(0)
-
-#define AES_DISABLE_INT 0x00000000
-#define AES_DMA_CPD_ERR_INT BIT(8)
-#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7)
-#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6)
-#define AES_DMA_INBUF_RD_ERR_INT BIT(5)
-#define AES_DMA_INBUF_WR_ERR_INT BIT(4)
-#define AES_DMA_BAD_COMP_INT BIT(3)
-#define AES_DMA_SAI_INT BIT(2)
-#define AES_DMA_SRC_DONE_INT BIT(0)
-#define AES_COMPLETE_INT BIT(1)
-
-#define AES_DMA_MSI_MASK_CLEAR BIT(0)
-
-#define AES_128_BIT_KEY 0x00000000
-#define AES_256_BIT_KEY BIT(0)
-
-#define AES_DEACTIVATE_PERF_CNTR 0x00000000
-#define AES_ACTIVATE_PERF_CNTR BIT(0)
-
-#define AES_MAX_TAG_SIZE_U32 4
-
-#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
-
-/*
- * There is an inconsistency in the documentation. This is documented as a
- * 11-bit value, but it is actually 10-bits.
- */
-#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF
-
-/*
- * During CCM decrypt, the OCS block needs to finish processing the ciphertext
- * before the tag is written. For 128-bit mode this required delay is 28 OCS
- * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
- */
-#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL
-
-/*
- * During CCM decrypt there must be a delay of at least 42 OCS clock cycles
- * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM
- * bit in the same register (as stated in the OCS databook)
- */
-#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL
-
-/* See RFC3610 section 2.2 */
-#define L_PRIME_MIN (1)
-#define L_PRIME_MAX (7)
-/*
- * CCM IV format from RFC 3610 section 2.3
- *
- * Octet Number Contents
- * ------------ ---------
- * 0 Flags
- * 1 ... 15-L Nonce N
- * 16-L ... 15 Counter i
- *
- * Flags = L' = L - 1
- */
-#define L_PRIME_IDX 0
-#define COUNTER_START(lprime) (16 - ((lprime) + 1))
-#define COUNTER_LEN(lprime) ((lprime) + 1)
-
-enum aes_counter_mode {
- AES_CTR_M_NO_INC = 0,
- AES_CTR_M_32_INC = 1,
- AES_CTR_M_64_INC = 2,
- AES_CTR_M_128_INC = 3,
-};
-
-/**
- * struct ocs_dma_linked_list - OCS DMA linked list entry.
- * @src_addr: Source address of the data.
- * @src_len: Length of data to be fetched.
- * @next: Next dma_list to fetch.
- * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
- */
-struct ocs_dma_linked_list {
- u32 src_addr;
- u32 src_len;
- u32 next;
- u32 ll_flags;
-} __packed;
-
-/*
- * Set endianness of inputs and outputs
- * AES_BYTE_ORDER_CFG
- * default 0x00000000
- * bit [10] - KEY_HI_LO_SWAP
- * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
- * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
- * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
- * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
- * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
- * bit [4] - IV_SWAP_BYTES_IN_DWORD
- * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
- * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
- * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
- * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
- */
-static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
-}
-
-/* Trigger AES process start. */
-static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
-}
-
-/* Indicate last bulk of data. */
-static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_ACTIVE_TERMINATION,
- aes_dev->base_reg + AES_ACTIVE_OFFSET);
-}
-
-/*
- * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits.
- *
- * Called when DMA is programmed to fetch the last batch of data.
- * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
- * data.
- * - For AES-GCM, it is called for the last batch of Plaintext data and
- * Ciphertext data.
- */
-static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_ACTIVE_LAST_CCM_GCM,
- aes_dev->base_reg + AES_ACTIVE_OFFSET);
-}
-
-/* Wait for LAST_CCM_GCM bit to be unset. */
-static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
-{
- u32 aes_active_reg;
-
- do {
- aes_active_reg = ioread32(aes_dev->base_reg +
- AES_ACTIVE_OFFSET);
- } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
-}
-
-/* Wait for 10 bits of input occupancy. */
-static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
-{
- u32 reg;
-
- do {
- reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
- } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
-}
-
- /*
- * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all
- * other bits).
- *
- * Called when DMA is programmed to fetch the last batch of Associated Data
- * (CCM case) or Additional Authenticated Data (GCM case).
- */
-static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
- aes_dev->base_reg + AES_ACTIVE_OFFSET);
-}
-
-/* Set DMA src and dst transfer size to 0 */
-static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
- iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
-}
-
-/* Activate DMA for zero-byte transfer case. */
-static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
- aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
-}
-
-/* Activate DMA and enable src linked list */
-static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
- AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
- aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
-}
-
-/* Activate DMA and enable dst linked list */
-static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
- AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
- aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
-}
-
-/* Activate DMA and enable src and dst linked lists */
-static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
- AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
- AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
- aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
-}
-
-/* Reset PERF_CNTR to 0 and activate it */
-static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
-{
- iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
- iowrite32(AES_ACTIVATE_PERF_CNTR,
- aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
-}
-
-/* Wait until PERF_CNTR is > delay, then deactivate it */
-static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
- int delay)
-{
- while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
- ;
- iowrite32(AES_DEACTIVATE_PERF_CNTR,
- aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
-}
-
-/* Disable AES and DMA IRQ. */
-static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
-{
- u32 isr_val = 0;
-
- /* Disable interrupts */
- iowrite32(AES_DISABLE_INT,
- aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
- iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
-
- /* Clear any pending interrupt */
- isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
- if (isr_val)
- iowrite32(isr_val,
- aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
-
- isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
- if (isr_val)
- iowrite32(isr_val,
- aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
-
- isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
- if (isr_val)
- iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
-}
-
-/* Enable AES or DMA IRQ. IRQ is disabled once fired. */
-static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
-{
- if (irq == AES_COMPLETE_INT) {
- /* Ensure DMA error interrupts are enabled */
- iowrite32(AES_DMA_CPD_ERR_INT |
- AES_DMA_OUTBUF_RD_ERR_INT |
- AES_DMA_OUTBUF_WR_ERR_INT |
- AES_DMA_INBUF_RD_ERR_INT |
- AES_DMA_INBUF_WR_ERR_INT |
- AES_DMA_BAD_COMP_INT |
- AES_DMA_SAI_INT,
- aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
- /*
- * AES_IER
- * default 0x00000000
- * bits [31:3] - reserved
- * bit [2] - EN_SKS_ERR
- * bit [1] - EN_AES_COMPLETE
- * bit [0] - reserved
- */
- iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
- return;
- }
- if (irq == AES_DMA_SRC_DONE_INT) {
- /* Ensure AES interrupts are disabled */
- iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
- /*
- * DMA_MSI_IER
- * default 0x00000000
- * bits [31:9] - reserved
- * bit [8] - CPD_ERR_INT_EN
- * bit [7] - OUTBUF_RD_ERR_INT_EN
- * bit [6] - OUTBUF_WR_ERR_INT_EN
- * bit [5] - INBUF_RD_ERR_INT_EN
- * bit [4] - INBUF_WR_ERR_INT_EN
- * bit [3] - BAD_COMP_INT_EN
- * bit [2] - SAI_INT_EN
- * bit [1] - DST_DONE_INT_EN
- * bit [0] - SRC_DONE_INT_EN
- */
- iowrite32(AES_DMA_CPD_ERR_INT |
- AES_DMA_OUTBUF_RD_ERR_INT |
- AES_DMA_OUTBUF_WR_ERR_INT |
- AES_DMA_INBUF_RD_ERR_INT |
- AES_DMA_INBUF_WR_ERR_INT |
- AES_DMA_BAD_COMP_INT |
- AES_DMA_SAI_INT |
- AES_DMA_SRC_DONE_INT,
- aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
- }
-}
-
-/* Enable and wait for IRQ (either from OCS AES engine or DMA) */
-static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
-{
- int rc;
-
- reinit_completion(&aes_dev->irq_completion);
- aes_irq_enable(aes_dev, irq);
- rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
- if (rc)
- return rc;
-
- return aes_dev->dma_err_mask ? -EIO : 0;
-}
-
-/* Configure DMA to OCS, linked list mode */
-static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
- dma_addr_t dma_list)
-{
- iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
- iowrite32(dma_list,
- aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
-}
-
-/* Configure DMA from OCS, linked list mode */
-static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
- dma_addr_t dma_list)
-{
- iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
- iowrite32(dma_list,
- aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
-}
-
-irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
-{
- struct ocs_aes_dev *aes_dev = dev_id;
- u32 aes_dma_isr;
-
- /* Read DMA ISR status. */
- aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
-
- /* Disable and clear interrupts. */
- aes_irq_disable(aes_dev);
-
- /* Save DMA error status. */
- aes_dev->dma_err_mask = aes_dma_isr &
- (AES_DMA_CPD_ERR_INT |
- AES_DMA_OUTBUF_RD_ERR_INT |
- AES_DMA_OUTBUF_WR_ERR_INT |
- AES_DMA_INBUF_RD_ERR_INT |
- AES_DMA_INBUF_WR_ERR_INT |
- AES_DMA_BAD_COMP_INT |
- AES_DMA_SAI_INT);
-
- /* Signal IRQ completion. */
- complete(&aes_dev->irq_completion);
-
- return IRQ_HANDLED;
-}
-
-/**
- * ocs_aes_set_key() - Write key into OCS AES hardware.
- * @aes_dev: The OCS AES device to write the key to.
- * @key_size: The size of the key (in bytes).
- * @key: The key to write.
- * @cipher: The cipher the key is for.
- *
- * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
- enum ocs_cipher cipher)
-{
- const u32 *key_u32;
- u32 val;
- int i;
-
- /* OCS AES supports 128-bit and 256-bit keys only. */
- if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
- dev_err(aes_dev->dev,
- "%d-bit keys not supported by AES cipher\n",
- key_size * 8);
- return -EINVAL;
- }
- /* OCS SM4 supports 128-bit keys only. */
- if (cipher == OCS_SM4 && key_size != 16) {
- dev_err(aes_dev->dev,
- "%d-bit keys not supported for SM4 cipher\n",
- key_size * 8);
- return -EINVAL;
- }
-
- if (!key)
- return -EINVAL;
-
- key_u32 = (const u32 *)key;
-
- /* Write key to AES_KEY[0-7] registers */
- for (i = 0; i < (key_size / sizeof(u32)); i++) {
- iowrite32(key_u32[i],
- aes_dev->base_reg + AES_KEY_0_OFFSET +
- (i * sizeof(u32)));
- }
- /*
- * Write key size
- * bits [31:1] - reserved
- * bit [0] - AES_KEY_SIZE
- * 0 - 128 bit key
- * 1 - 256 bit key
- */
- val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
- iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
-
- return 0;
-}
-
-/* Write AES_COMMAND */
-static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
- enum ocs_cipher cipher,
- enum ocs_mode mode,
- enum ocs_instruction instruction)
-{
- u32 val;
-
- /* AES_COMMAND
- * default 0x000000CC
- * bit [14] - CIPHER_SELECT
- * 0 - AES
- * 1 - SM4
- * bits [11:8] - OCS_AES_MODE
- * 0000 - ECB
- * 0001 - CBC
- * 0010 - CTR
- * 0110 - CCM
- * 0111 - GCM
- * 1001 - CTS
- * bits [7:6] - AES_INSTRUCTION
- * 00 - ENCRYPT
- * 01 - DECRYPT
- * 10 - EXPAND
- * 11 - BYPASS
- * bits [3:2] - CTR_M_BITS
- * 00 - No increment
- * 01 - Least significant 32 bits are incremented
- * 10 - Least significant 64 bits are incremented
- * 11 - Full 128 bits are incremented
- */
- val = (cipher << 14) | (mode << 8) | (instruction << 6) |
- (AES_CTR_M_128_INC << 2);
- iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
-}
-
-static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
- enum ocs_mode mode,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction)
-{
- /* Ensure interrupts are disabled and pending interrupts cleared. */
- aes_irq_disable(aes_dev);
-
- /* Set endianness recommended by data-sheet. */
- aes_a_set_endianness(aes_dev);
-
- /* Set AES_COMMAND register. */
- set_ocs_aes_command(aes_dev, cipher, mode, instruction);
-}
-
-/*
- * Write the byte length of the last AES/SM4 block of Payload data (without
- * zero padding and without the length of the MAC) in register AES_PLEN.
- */
-static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
- u32 size)
-{
- u32 val;
-
- if (size == 0) {
- val = 0;
- goto exit;
- }
-
- val = size % AES_BLOCK_SIZE;
- if (val == 0)
- val = AES_BLOCK_SIZE;
-
-exit:
- iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
-}
-
-/*
- * Validate inputs according to mode.
- * If OK return 0; else return -EINVAL.
- */
-static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
- const u8 *iv, u32 iv_size,
- dma_addr_t aad_dma_list, u32 aad_size,
- const u8 *tag, u32 tag_size,
- enum ocs_cipher cipher, enum ocs_mode mode,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list)
-{
- /* Ensure cipher, mode and instruction are valid. */
- if (!(cipher == OCS_AES || cipher == OCS_SM4))
- return -EINVAL;
-
- if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
- mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
- mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
- return -EINVAL;
-
- if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
- instruction != OCS_EXPAND && instruction != OCS_BYPASS)
- return -EINVAL;
-
- /*
- * When instruction is OCS_BYPASS, OCS simply copies data from source
- * to destination using DMA.
- *
- * AES mode is irrelevant, but both source and destination DMA
- * linked-list must be defined.
- */
- if (instruction == OCS_BYPASS) {
- if (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- return 0;
- }
-
- /*
- * For performance reasons switch based on mode to limit unnecessary
- * conditionals for each mode
- */
- switch (mode) {
- case OCS_MODE_ECB:
- /* Ensure input length is multiple of block size */
- if (src_size % AES_BLOCK_SIZE != 0)
- return -EINVAL;
-
- /* Ensure source and destination linked lists are created */
- if (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CBC:
- /* Ensure input length is multiple of block size */
- if (src_size % AES_BLOCK_SIZE != 0)
- return -EINVAL;
-
- /* Ensure source and destination linked lists are created */
- if (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* Ensure IV is present and block size in length */
- if (!iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CTR:
- /* Ensure input length of 1 byte or greater */
- if (src_size == 0)
- return -EINVAL;
-
- /* Ensure source and destination linked lists are created */
- if (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* Ensure IV is present and block size in length */
- if (!iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CTS:
- /* Ensure input length >= block size */
- if (src_size < AES_BLOCK_SIZE)
- return -EINVAL;
-
- /* Ensure source and destination linked lists are created */
- if (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* Ensure IV is present and block size in length */
- if (!iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_GCM:
- /* Ensure IV is present and GCM_AES_IV_SIZE in length */
- if (!iv || iv_size != GCM_AES_IV_SIZE)
- return -EINVAL;
-
- /*
- * If input data present ensure source and destination linked
- * lists are created
- */
- if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR))
- return -EINVAL;
-
- /* If aad present ensure aad linked list is created */
- if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* Ensure tag destination is set */
- if (!tag)
- return -EINVAL;
-
- /* Just ensure that tag_size doesn't cause overflows. */
- if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
- return -EINVAL;
-
- return 0;
-
- case OCS_MODE_CCM:
- /* Ensure IV is present and block size in length */
- if (!iv || iv_size != AES_BLOCK_SIZE)
- return -EINVAL;
-
- /* 2 <= L <= 8, so 1 <= L' <= 7 */
- if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
- iv[L_PRIME_IDX] > L_PRIME_MAX)
- return -EINVAL;
-
- /* If aad present ensure aad linked list is created */
- if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* Just ensure that tag_size doesn't cause overflows. */
- if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
- return -EINVAL;
-
- if (instruction == OCS_DECRYPT) {
- /*
- * If input data present ensure source and destination
- * linked lists are created
- */
- if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
- dst_dma_list == DMA_MAPPING_ERROR))
- return -EINVAL;
-
- /* Ensure input tag is present */
- if (!tag)
- return -EINVAL;
-
- return 0;
- }
-
- /* Instruction == OCS_ENCRYPT */
-
- /*
- * Destination linked list always required (for tag even if no
- * input data)
- */
- if (dst_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- /* If input data present ensure src linked list is created */
- if (src_size && src_dma_list == DMA_MAPPING_ERROR)
- return -EINVAL;
-
- return 0;
-
- default:
- return -EINVAL;
- }
-}
-
-/**
- * ocs_aes_op() - Perform AES/SM4 operation.
- * @aes_dev: The OCS AES device to use.
- * @mode: The mode to use (ECB, CBC, CTR, or CTS).
- * @cipher: The cipher to use (AES or SM4).
- * @instruction: The instruction to perform (encrypt or decrypt).
- * @dst_dma_list: The OCS DMA list mapping output memory.
- * @src_dma_list: The OCS DMA list mapping input payload data.
- * @src_size: The amount of data mapped by @src_dma_list.
- * @iv: The IV vector.
- * @iv_size: The size (in bytes) of @iv.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_aes_op(struct ocs_aes_dev *aes_dev,
- enum ocs_mode mode,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- u8 *iv,
- u32 iv_size)
-{
- u32 *iv32;
- int rc;
-
- rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
- NULL, 0, cipher, mode, instruction,
- dst_dma_list);
- if (rc)
- return rc;
- /*
- * ocs_aes_validate_inputs() is a generic check, now ensure mode is not
- * GCM or CCM.
- */
- if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
- return -EINVAL;
-
- /* Cast IV to u32 array. */
- iv32 = (u32 *)iv;
-
- ocs_aes_init(aes_dev, mode, cipher, instruction);
-
- if (mode == OCS_MODE_CTS) {
- /* Write the byte length of the last data block to engine. */
- ocs_aes_write_last_data_blk_len(aes_dev, src_size);
- }
-
- /* ECB is the only mode that doesn't use IV. */
- if (mode != OCS_MODE_ECB) {
- iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
- iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
- iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
- iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
- }
-
- /* Set AES_ACTIVE.TRIGGER to start the operation. */
- aes_a_op_trigger(aes_dev);
-
- /* Configure and activate input / output DMA. */
- dma_to_ocs_aes_ll(aes_dev, src_dma_list);
- dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
- aes_a_dma_active_src_dst_ll_en(aes_dev);
-
- if (mode == OCS_MODE_CTS) {
- /*
- * For CTS mode, instruct engine to activate ciphertext
- * stealing if last block of data is incomplete.
- */
- aes_a_set_last_gcx(aes_dev);
- } else {
- /* For all other modes, just write the 'termination' bit. */
- aes_a_op_termination(aes_dev);
- }
-
- /* Wait for engine to complete processing. */
- rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
- if (rc)
- return rc;
-
- if (mode == OCS_MODE_CTR) {
- /* Read back IV for streaming mode */
- iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
- iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
- iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
- iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
- }
-
- return 0;
-}
-
-/* Compute and write J0 to engine registers. */
-static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
- const u8 *iv)
-{
- const u32 *j0 = (u32 *)iv;
-
- /*
- * IV must be 12 bytes; Other sizes not supported as Linux crypto API
- * does only expects/allows 12 byte IV for GCM
- */
- iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
- iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
- iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
- iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
-}
-
-/* Read GCM tag from engine registers. */
-static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
- u8 *tag, u32 tag_size)
-{
- u32 tag_u32[AES_MAX_TAG_SIZE_U32];
-
- /*
- * The Authentication Tag T is stored in Little Endian order in the
- * registers with the most significant bytes stored from AES_T_MAC[3]
- * downward.
- */
- tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
- tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
- tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
- tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
-
- memcpy(tag, tag_u32, tag_size);
-}
-
-/**
- * ocs_aes_gcm_op() - Perform GCM operation.
- * @aes_dev: The OCS AES device to use.
- * @cipher: The Cipher to use (AES or SM4).
- * @instruction: The instruction to perform (encrypt or decrypt).
- * @dst_dma_list: The OCS DMA list mapping output memory.
- * @src_dma_list: The OCS DMA list mapping input payload data.
- * @src_size: The amount of data mapped by @src_dma_list.
- * @iv: The input IV vector.
- * @aad_dma_list: The OCS DMA list mapping input AAD data.
- * @aad_size: The amount of data mapped by @aad_dma_list.
- * @out_tag: Where to store computed tag.
- * @tag_size: The size (in bytes) of @out_tag.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- const u8 *iv,
- dma_addr_t aad_dma_list,
- u32 aad_size,
- u8 *out_tag,
- u32 tag_size)
-{
- u64 bit_len;
- u32 val;
- int rc;
-
- rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
- GCM_AES_IV_SIZE, aad_dma_list,
- aad_size, out_tag, tag_size, cipher,
- OCS_MODE_GCM, instruction,
- dst_dma_list);
- if (rc)
- return rc;
-
- ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
-
- /* Compute and write J0 to OCS HW. */
- ocs_aes_gcm_write_j0(aes_dev, iv);
-
- /* Write out_tag byte length */
- iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
-
- /* Write the byte length of the last plaintext / ciphertext block. */
- ocs_aes_write_last_data_blk_len(aes_dev, src_size);
-
- /* Write ciphertext bit length */
- bit_len = (u64)src_size * 8;
- val = bit_len & 0xFFFFFFFF;
- iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
- val = bit_len >> 32;
- iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
-
- /* Write aad bit length */
- bit_len = (u64)aad_size * 8;
- val = bit_len & 0xFFFFFFFF;
- iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
- val = bit_len >> 32;
- iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
-
- /* Set AES_ACTIVE.TRIGGER to start the operation. */
- aes_a_op_trigger(aes_dev);
-
- /* Process AAD. */
- if (aad_size) {
- /* If aad present, configure DMA to feed it to the engine. */
- dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
- aes_a_dma_active_src_ll_en(aes_dev);
-
- /* Instructs engine to pad last block of aad, if needed. */
- aes_a_set_last_gcx_and_adata(aes_dev);
-
- /* Wait for DMA transfer to complete. */
- rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
- if (rc)
- return rc;
- } else {
- aes_a_set_last_gcx_and_adata(aes_dev);
- }
-
- /* Wait until adata (if present) has been processed. */
- aes_a_wait_last_gcx(aes_dev);
- aes_a_dma_wait_input_buffer_occupancy(aes_dev);
-
- /* Now process payload. */
- if (src_size) {
- /* Configure and activate DMA for both input and output data. */
- dma_to_ocs_aes_ll(aes_dev, src_dma_list);
- dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
- aes_a_dma_active_src_dst_ll_en(aes_dev);
- } else {
- aes_a_dma_set_xfer_size_zero(aes_dev);
- aes_a_dma_active(aes_dev);
- }
-
- /* Instruct AES/SMA4 engine payload processing is over. */
- aes_a_set_last_gcx(aes_dev);
-
- /* Wait for OCS AES engine to complete processing. */
- rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
- if (rc)
- return rc;
-
- ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
-
- return 0;
-}
-
-/* Write encrypted tag to AES/SM4 engine. */
-static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
- const u8 *in_tag, u32 tag_size)
-{
- int i;
-
- /* Ensure DMA input buffer is empty */
- aes_a_dma_wait_input_buffer_occupancy(aes_dev);
-
- /*
- * During CCM decrypt, the OCS block needs to finish processing the
- * ciphertext before the tag is written. So delay needed after DMA has
- * completed writing the ciphertext
- */
- aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
- aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
- CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
-
- /* Write encrypted tag to AES/SM4 engine. */
- for (i = 0; i < tag_size; i++) {
- iowrite8(in_tag[i], aes_dev->base_reg +
- AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
- }
-}
-
-/*
- * Write B0 CCM block to OCS AES HW.
- *
- * Note: B0 format is documented in NIST Special Publication 800-38C
- * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
- * (see Section A.2.1)
- */
-static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
- const u8 *iv, u32 adata_size, u32 tag_size,
- u32 cryptlen)
-{
- u8 b0[16]; /* CCM B0 block is 16 bytes long. */
- int i, q;
-
- /* Initialize B0 to 0. */
- memset(b0, 0, sizeof(b0));
-
- /*
- * B0[0] is the 'Flags Octet' and has the following structure:
- * bit 7: Reserved
- * bit 6: Adata flag
- * bit 5-3: t value encoded as (t-2)/2
- * bit 2-0: q value encoded as q - 1
- */
- /* If there is AAD data, set the Adata flag. */
- if (adata_size)
- b0[0] |= BIT(6);
- /*
- * t denotes the octet length of T.
- * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is
- * encoded as (t - 2) / 2
- */
- b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
- /*
- * q is the octet length of Q.
- * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as
- * q - 1 == iv[0] & 0x7;
- */
- b0[0] |= iv[0] & 0x7;
- /*
- * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
- * and must be copied to b0[1]..b0[15-q].
- * q == (iv[0] & 0x7) + 1
- */
- q = (iv[0] & 0x7) + 1;
- for (i = 1; i <= 15 - q; i++)
- b0[i] = iv[i];
- /*
- * The rest of B0 must contain Q, i.e., the message length.
- * Q is encoded in q octets, in big-endian order, so to write it, we
- * start from the end of B0 and we move backward.
- */
- i = sizeof(b0) - 1;
- while (q) {
- b0[i] = cryptlen & 0xff;
- cryptlen >>= 8;
- i--;
- q--;
- }
- /*
- * If cryptlen is not zero at this point, it means that its original
- * value was too big.
- */
- if (cryptlen)
- return -EOVERFLOW;
- /* Now write B0 to OCS AES input buffer. */
- for (i = 0; i < sizeof(b0); i++)
- iowrite8(b0[i], aes_dev->base_reg +
- AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
- return 0;
-}
-
-/*
- * Write adata length to OCS AES HW.
- *
- * Note: adata len encoding is documented in NIST Special Publication 800-38C
- * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
- * (see Section A.2.2)
- */
-static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
- u64 adata_len)
-{
- u8 enc_a[10]; /* Maximum encoded size: 10 octets. */
- int i, len;
-
- /*
- * adata_len ('a') is encoded as follows:
- * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets
- * (big endian).
- * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
- * i.e., six octets (big endian).
- * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64,
- * i.e., ten octets (big endian).
- */
- if (adata_len < 65280) {
- len = 2;
- *(__be16 *)enc_a = cpu_to_be16(adata_len);
- } else if (adata_len <= 0xFFFFFFFF) {
- len = 6;
- *(__be16 *)enc_a = cpu_to_be16(0xfffe);
- *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
- } else { /* adata_len >= 2^32 */
- len = 10;
- *(__be16 *)enc_a = cpu_to_be16(0xffff);
- *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
- }
- for (i = 0; i < len; i++)
- iowrite8(enc_a[i],
- aes_dev->base_reg +
- AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
-}
-
-static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
- dma_addr_t adata_dma_list, u32 adata_size)
-{
- int rc;
-
- if (!adata_size) {
- /* Since no aad the LAST_GCX bit can be set now */
- aes_a_set_last_gcx_and_adata(aes_dev);
- goto exit;
- }
-
- /* Adata case. */
-
- /*
- * Form the encoding of the Associated data length and write it
- * to the AES/SM4 input buffer.
- */
- ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
-
- /* Configure the AES/SM4 DMA to fetch the Associated Data */
- dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
-
- /* Activate DMA to fetch Associated data. */
- aes_a_dma_active_src_ll_en(aes_dev);
-
- /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */
- aes_a_set_last_gcx_and_adata(aes_dev);
-
- /* Wait for DMA transfer to complete. */
- rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
- if (rc)
- return rc;
-
-exit:
- /* Wait until adata (if present) has been processed. */
- aes_a_wait_last_gcx(aes_dev);
- aes_a_dma_wait_input_buffer_occupancy(aes_dev);
-
- return 0;
-}
-
-static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size)
-{
- if (src_size) {
- /*
- * Configure and activate DMA for both input and output
- * data.
- */
- dma_to_ocs_aes_ll(aes_dev, src_dma_list);
- dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
- aes_a_dma_active_src_dst_ll_en(aes_dev);
- } else {
- /* Configure and activate DMA for output data only. */
- dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
- aes_a_dma_active_dst_ll_en(aes_dev);
- }
-
- /*
- * Set the LAST GCX bit in AES_ACTIVE Register to instruct
- * AES/SM4 engine to pad the last block of data.
- */
- aes_a_set_last_gcx(aes_dev);
-
- /* We are done, wait for IRQ and return. */
- return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
-}
-
-static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size)
-{
- if (!src_size) {
- /* Let engine process 0-length input. */
- aes_a_dma_set_xfer_size_zero(aes_dev);
- aes_a_dma_active(aes_dev);
- aes_a_set_last_gcx(aes_dev);
-
- return 0;
- }
-
- /*
- * Configure and activate DMA for both input and output
- * data.
- */
- dma_to_ocs_aes_ll(aes_dev, src_dma_list);
- dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
- aes_a_dma_active_src_dst_ll_en(aes_dev);
- /*
- * Set the LAST GCX bit in AES_ACTIVE Register; this allows the
- * AES/SM4 engine to differentiate between encrypted data and
- * encrypted MAC.
- */
- aes_a_set_last_gcx(aes_dev);
- /*
- * Enable DMA DONE interrupt; once DMA transfer is over,
- * interrupt handler will process the MAC/tag.
- */
- return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
-}
-
-/*
- * Compare Tag to Yr.
- *
- * Only used at the end of CCM decrypt. If tag == yr, message authentication
- * has succeeded.
- */
-static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
- u8 tag_size_bytes)
-{
- u32 tag[AES_MAX_TAG_SIZE_U32];
- u32 yr[AES_MAX_TAG_SIZE_U32];
- u8 i;
-
- /* Read Tag and Yr from AES registers. */
- for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
- tag[i] = ioread32(aes_dev->base_reg +
- AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
- yr[i] = ioread32(aes_dev->base_reg +
- AES_MULTIPURPOSE2_0_OFFSET +
- (i * sizeof(u32)));
- }
-
- return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
-}
-
-/**
- * ocs_aes_ccm_op() - Perform CCM operation.
- * @aes_dev: The OCS AES device to use.
- * @cipher: The Cipher to use (AES or SM4).
- * @instruction: The instruction to perform (encrypt or decrypt).
- * @dst_dma_list: The OCS DMA list mapping output memory.
- * @src_dma_list: The OCS DMA list mapping input payload data.
- * @src_size: The amount of data mapped by @src_dma_list.
- * @iv: The input IV vector.
- * @adata_dma_list: The OCS DMA list mapping input A-data.
- * @adata_size: The amount of data mapped by @adata_dma_list.
- * @in_tag: Input tag.
- * @tag_size: The size (in bytes) of @in_tag.
- *
- * Note: for encrypt the tag is appended to the ciphertext (in the memory
- * mapped by @dst_dma_list).
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- u8 *iv,
- dma_addr_t adata_dma_list,
- u32 adata_size,
- u8 *in_tag,
- u32 tag_size)
-{
- u32 *iv_32;
- u8 lprime;
- int rc;
-
- rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
- AES_BLOCK_SIZE, adata_dma_list, adata_size,
- in_tag, tag_size, cipher, OCS_MODE_CCM,
- instruction, dst_dma_list);
- if (rc)
- return rc;
-
- ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
-
- /*
- * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
- * auth tag so ensure this is the case
- */
- lprime = iv[L_PRIME_IDX];
- memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
-
- /*
- * Nonce is already converted to ctr0 before being passed into this
- * function as iv.
- */
- iv_32 = (u32 *)iv;
- iowrite32(__swab32(iv_32[0]),
- aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
- iowrite32(__swab32(iv_32[1]),
- aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
- iowrite32(__swab32(iv_32[2]),
- aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
- iowrite32(__swab32(iv_32[3]),
- aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
-
- /* Write MAC/tag length in register AES_TLEN */
- iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
- /*
- * Write the byte length of the last AES/SM4 block of Payload data
- * (without zero padding and without the length of the MAC) in register
- * AES_PLEN.
- */
- ocs_aes_write_last_data_blk_len(aes_dev, src_size);
-
- /* Set AES_ACTIVE.TRIGGER to start the operation. */
- aes_a_op_trigger(aes_dev);
-
- aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
-
- /* Form block B0 and write it to the AES/SM4 input buffer. */
- rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
- if (rc)
- return rc;
- /*
- * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT
- * clock cycles since TRIGGER bit was set
- */
- aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
- CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
-
- /* Process Adata. */
- ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
-
- /* For Encrypt case we just process the payload and return. */
- if (instruction == OCS_ENCRYPT) {
- return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
- src_dma_list, src_size);
- }
- /* For Decypt we need to process the payload and then the tag. */
- rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
- src_dma_list, src_size);
- if (rc)
- return rc;
-
- /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
- ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
- rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
- if (rc)
- return rc;
-
- return ccm_compare_tag_to_yr(aes_dev, tag_size);
-}
-
-/**
- * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
- * @aes_dev: The OCS AES device the list will be created for.
- * @sg: The SG list OCS DMA linked list will be created from. When
- * passed to this function, @sg must have been already mapped
- * with dma_map_sg().
- * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
- * value returned by dma_map_sg() when @sg was mapped.
- * @dll_desc: The OCS DMA dma_list to use to store information about the
- * created linked list.
- * @data_size: The size of the data (from the SG list) to be mapped into the
- * OCS DMA linked list.
- * @data_offset: The offset (within the SG list) of the data to be mapped.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
- struct scatterlist *sg,
- int sg_dma_count,
- struct ocs_dll_desc *dll_desc,
- size_t data_size, size_t data_offset)
-{
- struct ocs_dma_linked_list *ll = NULL;
- struct scatterlist *sg_tmp;
- unsigned int tmp;
- int dma_nents;
- int i;
-
- if (!dll_desc || !sg || !aes_dev)
- return -EINVAL;
-
- /* Default values for when no ddl_desc is created. */
- dll_desc->vaddr = NULL;
- dll_desc->dma_addr = DMA_MAPPING_ERROR;
- dll_desc->size = 0;
-
- if (data_size == 0)
- return 0;
-
- /* Loop over sg_list until we reach entry at specified offset. */
- while (data_offset >= sg_dma_len(sg)) {
- data_offset -= sg_dma_len(sg);
- sg_dma_count--;
- sg = sg_next(sg);
- /* If we reach the end of the list, offset was invalid. */
- if (!sg || sg_dma_count == 0)
- return -EINVAL;
- }
-
- /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
- dma_nents = 0;
- tmp = 0;
- sg_tmp = sg;
- while (tmp < data_offset + data_size) {
- /* If we reach the end of the list, data_size was invalid. */
- if (!sg_tmp)
- return -EINVAL;
- tmp += sg_dma_len(sg_tmp);
- dma_nents++;
- sg_tmp = sg_next(sg_tmp);
- }
- if (dma_nents > sg_dma_count)
- return -EINVAL;
-
- /* Allocate the DMA list, one entry for each SG entry. */
- dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
- dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
- &dll_desc->dma_addr, GFP_KERNEL);
- if (!dll_desc->vaddr)
- return -ENOMEM;
-
- /* Populate DMA linked list entries. */
- ll = dll_desc->vaddr;
- for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
- ll[i].src_addr = sg_dma_address(sg) + data_offset;
- ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
- (sg_dma_len(sg) - data_offset) : data_size;
- data_offset = 0;
- data_size -= ll[i].src_len;
- /* Current element points to the DMA address of the next one. */
- ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
- ll[i].ll_flags = 0;
- }
- /* Terminate last element. */
- ll[i - 1].next = 0;
- ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
-
- return 0;
-}
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Intel Keem Bay OCS AES Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#ifndef _CRYPTO_OCS_AES_H
-#define _CRYPTO_OCS_AES_H
-
-#include <linux/dma-mapping.h>
-
-enum ocs_cipher {
- OCS_AES = 0,
- OCS_SM4 = 1,
-};
-
-enum ocs_mode {
- OCS_MODE_ECB = 0,
- OCS_MODE_CBC = 1,
- OCS_MODE_CTR = 2,
- OCS_MODE_CCM = 6,
- OCS_MODE_GCM = 7,
- OCS_MODE_CTS = 9,
-};
-
-enum ocs_instruction {
- OCS_ENCRYPT = 0,
- OCS_DECRYPT = 1,
- OCS_EXPAND = 2,
- OCS_BYPASS = 3,
-};
-
-/**
- * struct ocs_aes_dev - AES device context.
- * @list: List head for insertion into device list hold
- * by driver.
- * @dev: OCS AES device.
- * @irq: IRQ number.
- * @base_reg: IO base address of OCS AES.
- * @irq_copy_completion: Completion to indicate IRQ has been triggered.
- * @dma_err_mask: Error reported by OCS DMA interrupts.
- * @engine: Crypto engine for the device.
- */
-struct ocs_aes_dev {
- struct list_head list;
- struct device *dev;
- int irq;
- void __iomem *base_reg;
- struct completion irq_completion;
- u32 dma_err_mask;
- struct crypto_engine *engine;
-};
-
-/**
- * struct ocs_dll_desc - Descriptor of an OCS DMA Linked List.
- * @vaddr: Virtual address of the linked list head.
- * @dma_addr: DMA address of the linked list head.
- * @size: Size (in bytes) of the linked list.
- */
-struct ocs_dll_desc {
- void *vaddr;
- dma_addr_t dma_addr;
- size_t size;
-};
-
-int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, const u32 key_size,
- const u8 *key, const enum ocs_cipher cipher);
-
-int ocs_aes_op(struct ocs_aes_dev *aes_dev,
- enum ocs_mode mode,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- u8 *iv,
- u32 iv_size);
-
-/**
- * ocs_aes_bypass_op() - Use OCS DMA to copy data.
- * @aes_dev: The OCS AES device to use.
- * @dst_dma_list: The OCS DMA list mapping the memory where input data
- * will be copied to.
- * @src_dma_list: The OCS DMA list mapping input data.
- * @src_size: The amount of data to copy.
- */
-static inline int ocs_aes_bypass_op(struct ocs_aes_dev *aes_dev,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list, u32 src_size)
-{
- return ocs_aes_op(aes_dev, OCS_MODE_ECB, OCS_AES, OCS_BYPASS,
- dst_dma_list, src_dma_list, src_size, NULL, 0);
-}
-
-int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- const u8 *iv,
- dma_addr_t aad_dma_list,
- u32 aad_size,
- u8 *out_tag,
- u32 tag_size);
-
-int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
- enum ocs_cipher cipher,
- enum ocs_instruction instruction,
- dma_addr_t dst_dma_list,
- dma_addr_t src_dma_list,
- u32 src_size,
- u8 *iv,
- dma_addr_t adata_dma_list,
- u32 adata_size,
- u8 *in_tag,
- u32 tag_size);
-
-int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
- struct scatterlist *sg,
- int sg_dma_count,
- struct ocs_dll_desc *dll_desc,
- size_t data_size,
- size_t data_offset);
-
-irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id);
-
-#endif
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Intel Keem Bay OCS HCU Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/iopoll.h>
-#include <linux/irq.h>
-#include <linux/module.h>
-
-#include <crypto/sha2.h>
-
-#include "ocs-hcu.h"
-
-/* Registers. */
-#define OCS_HCU_MODE 0x00
-#define OCS_HCU_CHAIN 0x04
-#define OCS_HCU_OPERATION 0x08
-#define OCS_HCU_KEY_0 0x0C
-#define OCS_HCU_ISR 0x50
-#define OCS_HCU_IER 0x54
-#define OCS_HCU_STATUS 0x58
-#define OCS_HCU_MSG_LEN_LO 0x60
-#define OCS_HCU_MSG_LEN_HI 0x64
-#define OCS_HCU_KEY_BYTE_ORDER_CFG 0x80
-#define OCS_HCU_DMA_SRC_ADDR 0x400
-#define OCS_HCU_DMA_SRC_SIZE 0x408
-#define OCS_HCU_DMA_DST_SIZE 0x40C
-#define OCS_HCU_DMA_DMA_MODE 0x410
-#define OCS_HCU_DMA_NEXT_SRC_DESCR 0x418
-#define OCS_HCU_DMA_MSI_ISR 0x480
-#define OCS_HCU_DMA_MSI_IER 0x484
-#define OCS_HCU_DMA_MSI_MASK 0x488
-
-/* Register bit definitions. */
-#define HCU_MODE_ALGO_SHIFT 16
-#define HCU_MODE_HMAC_SHIFT 22
-
-#define HCU_STATUS_BUSY BIT(0)
-
-#define HCU_BYTE_ORDER_SWAP BIT(0)
-
-#define HCU_IRQ_HASH_DONE BIT(2)
-#define HCU_IRQ_HASH_ERR_MASK (BIT(3) | BIT(1) | BIT(0))
-
-#define HCU_DMA_IRQ_SRC_DONE BIT(0)
-#define HCU_DMA_IRQ_SAI_ERR BIT(2)
-#define HCU_DMA_IRQ_BAD_COMP_ERR BIT(3)
-#define HCU_DMA_IRQ_INBUF_RD_ERR BIT(4)
-#define HCU_DMA_IRQ_INBUF_WD_ERR BIT(5)
-#define HCU_DMA_IRQ_OUTBUF_WR_ERR BIT(6)
-#define HCU_DMA_IRQ_OUTBUF_RD_ERR BIT(7)
-#define HCU_DMA_IRQ_CRD_ERR BIT(8)
-#define HCU_DMA_IRQ_ERR_MASK (HCU_DMA_IRQ_SAI_ERR | \
- HCU_DMA_IRQ_BAD_COMP_ERR | \
- HCU_DMA_IRQ_INBUF_RD_ERR | \
- HCU_DMA_IRQ_INBUF_WD_ERR | \
- HCU_DMA_IRQ_OUTBUF_WR_ERR | \
- HCU_DMA_IRQ_OUTBUF_RD_ERR | \
- HCU_DMA_IRQ_CRD_ERR)
-
-#define HCU_DMA_SNOOP_MASK (0x7 << 28)
-#define HCU_DMA_SRC_LL_EN BIT(25)
-#define HCU_DMA_EN BIT(31)
-
-#define OCS_HCU_ENDIANNESS_VALUE 0x2A
-
-#define HCU_DMA_MSI_UNMASK BIT(0)
-#define HCU_DMA_MSI_DISABLE 0
-#define HCU_IRQ_DISABLE 0
-
-#define OCS_HCU_START BIT(0)
-#define OCS_HCU_TERMINATE BIT(1)
-
-#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
-
-#define OCS_HCU_HW_KEY_LEN_U32 (OCS_HCU_HW_KEY_LEN / sizeof(u32))
-
-#define HCU_DATA_WRITE_ENDIANNESS_OFFSET 26
-
-#define OCS_HCU_NUM_CHAINS_SHA256_224_SM3 (SHA256_DIGEST_SIZE / sizeof(u32))
-#define OCS_HCU_NUM_CHAINS_SHA384_512 (SHA512_DIGEST_SIZE / sizeof(u32))
-
-/*
- * While polling on a busy HCU, wait maximum 200us between one check and the
- * other.
- */
-#define OCS_HCU_WAIT_BUSY_RETRY_DELAY_US 200
-/* Wait on a busy HCU for maximum 1 second. */
-#define OCS_HCU_WAIT_BUSY_TIMEOUT_US 1000000
-
-/**
- * struct ocs_hcu_dma_entry - An entry in an OCS DMA linked list.
- * @src_addr: Source address of the data.
- * @src_len: Length of data to be fetched.
- * @nxt_desc: Next descriptor to fetch.
- * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
- */
-struct ocs_hcu_dma_entry {
- u32 src_addr;
- u32 src_len;
- u32 nxt_desc;
- u32 ll_flags;
-};
-
-/**
- * struct ocs_hcu_dma_list - OCS-specific DMA linked list.
- * @head: The head of the list (points to the array backing the list).
- * @tail: The current tail of the list; NULL if the list is empty.
- * @dma_addr: The DMA address of @head (i.e., the DMA address of the backing
- * array).
- * @max_nents: Maximum number of entries in the list (i.e., number of elements
- * in the backing array).
- *
- * The OCS DMA list is an array-backed list of OCS DMA descriptors. The array
- * backing the list is allocated with dma_alloc_coherent() and pointed by
- * @head.
- */
-struct ocs_hcu_dma_list {
- struct ocs_hcu_dma_entry *head;
- struct ocs_hcu_dma_entry *tail;
- dma_addr_t dma_addr;
- size_t max_nents;
-};
-
-static inline u32 ocs_hcu_num_chains(enum ocs_hcu_algo algo)
-{
- switch (algo) {
- case OCS_HCU_ALGO_SHA224:
- case OCS_HCU_ALGO_SHA256:
- case OCS_HCU_ALGO_SM3:
- return OCS_HCU_NUM_CHAINS_SHA256_224_SM3;
- case OCS_HCU_ALGO_SHA384:
- case OCS_HCU_ALGO_SHA512:
- return OCS_HCU_NUM_CHAINS_SHA384_512;
- default:
- return 0;
- };
-}
-
-static inline u32 ocs_hcu_digest_size(enum ocs_hcu_algo algo)
-{
- switch (algo) {
- case OCS_HCU_ALGO_SHA224:
- return SHA224_DIGEST_SIZE;
- case OCS_HCU_ALGO_SHA256:
- case OCS_HCU_ALGO_SM3:
- /* SM3 shares the same block size. */
- return SHA256_DIGEST_SIZE;
- case OCS_HCU_ALGO_SHA384:
- return SHA384_DIGEST_SIZE;
- case OCS_HCU_ALGO_SHA512:
- return SHA512_DIGEST_SIZE;
- default:
- return 0;
- }
-}
-
-/**
- * ocs_hcu_wait_busy() - Wait for HCU OCS hardware to became usable.
- * @hcu_dev: OCS HCU device to wait for.
- *
- * Return: 0 if device free, -ETIMEOUT if device busy and internal timeout has
- * expired.
- */
-static int ocs_hcu_wait_busy(struct ocs_hcu_dev *hcu_dev)
-{
- long val;
-
- return readl_poll_timeout(hcu_dev->io_base + OCS_HCU_STATUS, val,
- !(val & HCU_STATUS_BUSY),
- OCS_HCU_WAIT_BUSY_RETRY_DELAY_US,
- OCS_HCU_WAIT_BUSY_TIMEOUT_US);
-}
-
-static void ocs_hcu_done_irq_en(struct ocs_hcu_dev *hcu_dev)
-{
- /* Clear any pending interrupts. */
- writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_ISR);
- hcu_dev->irq_err = false;
- /* Enable error and HCU done interrupts. */
- writel(HCU_IRQ_HASH_DONE | HCU_IRQ_HASH_ERR_MASK,
- hcu_dev->io_base + OCS_HCU_IER);
-}
-
-static void ocs_hcu_dma_irq_en(struct ocs_hcu_dev *hcu_dev)
-{
- /* Clear any pending interrupts. */
- writel(0xFFFFFFFF, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
- hcu_dev->irq_err = false;
- /* Only operating on DMA source completion and error interrupts. */
- writel(HCU_DMA_IRQ_ERR_MASK | HCU_DMA_IRQ_SRC_DONE,
- hcu_dev->io_base + OCS_HCU_DMA_MSI_IER);
- /* Unmask */
- writel(HCU_DMA_MSI_UNMASK, hcu_dev->io_base + OCS_HCU_DMA_MSI_MASK);
-}
-
-static void ocs_hcu_irq_dis(struct ocs_hcu_dev *hcu_dev)
-{
- writel(HCU_IRQ_DISABLE, hcu_dev->io_base + OCS_HCU_IER);
- writel(HCU_DMA_MSI_DISABLE, hcu_dev->io_base + OCS_HCU_DMA_MSI_IER);
-}
-
-static int ocs_hcu_wait_and_disable_irq(struct ocs_hcu_dev *hcu_dev)
-{
- int rc;
-
- rc = wait_for_completion_interruptible(&hcu_dev->irq_done);
- if (rc)
- goto exit;
-
- if (hcu_dev->irq_err) {
- /* Unset flag and return error. */
- hcu_dev->irq_err = false;
- rc = -EIO;
- goto exit;
- }
-
-exit:
- ocs_hcu_irq_dis(hcu_dev);
-
- return rc;
-}
-
-/**
- * ocs_hcu_get_intermediate_data() - Get intermediate data.
- * @hcu_dev: The target HCU device.
- * @data: Where to store the intermediate.
- * @algo: The algorithm being used.
- *
- * This function is used to save the current hashing process state in order to
- * continue it in the future.
- *
- * Note: once all data has been processed, the intermediate data actually
- * contains the hashing result. So this function is also used to retrieve the
- * final result of a hashing process.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int ocs_hcu_get_intermediate_data(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_idata *data,
- enum ocs_hcu_algo algo)
-{
- const int n = ocs_hcu_num_chains(algo);
- u32 *chain;
- int rc;
- int i;
-
- /* Data not requested. */
- if (!data)
- return -EINVAL;
-
- chain = (u32 *)data->digest;
-
- /* Ensure that the OCS is no longer busy before reading the chains. */
- rc = ocs_hcu_wait_busy(hcu_dev);
- if (rc)
- return rc;
-
- /*
- * This loops is safe because data->digest is an array of
- * SHA512_DIGEST_SIZE bytes and the maximum value returned by
- * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal
- * to SHA512_DIGEST_SIZE / sizeof(u32).
- */
- for (i = 0; i < n; i++)
- chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN);
-
- data->msg_len_lo = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_LO);
- data->msg_len_hi = readl(hcu_dev->io_base + OCS_HCU_MSG_LEN_HI);
-
- return 0;
-}
-
-/**
- * ocs_hcu_set_intermediate_data() - Set intermediate data.
- * @hcu_dev: The target HCU device.
- * @data: The intermediate data to be set.
- * @algo: The algorithm being used.
- *
- * This function is used to continue a previous hashing process.
- */
-static void ocs_hcu_set_intermediate_data(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_idata *data,
- enum ocs_hcu_algo algo)
-{
- const int n = ocs_hcu_num_chains(algo);
- u32 *chain = (u32 *)data->digest;
- int i;
-
- /*
- * This loops is safe because data->digest is an array of
- * SHA512_DIGEST_SIZE bytes and the maximum value returned by
- * ocs_hcu_num_chains() is OCS_HCU_NUM_CHAINS_SHA384_512 which is equal
- * to SHA512_DIGEST_SIZE / sizeof(u32).
- */
- for (i = 0; i < n; i++)
- writel(chain[i], hcu_dev->io_base + OCS_HCU_CHAIN);
-
- writel(data->msg_len_lo, hcu_dev->io_base + OCS_HCU_MSG_LEN_LO);
- writel(data->msg_len_hi, hcu_dev->io_base + OCS_HCU_MSG_LEN_HI);
-}
-
-static int ocs_hcu_get_digest(struct ocs_hcu_dev *hcu_dev,
- enum ocs_hcu_algo algo, u8 *dgst, size_t dgst_len)
-{
- u32 *chain;
- int rc;
- int i;
-
- if (!dgst)
- return -EINVAL;
-
- /* Length of the output buffer must match the algo digest size. */
- if (dgst_len != ocs_hcu_digest_size(algo))
- return -EINVAL;
-
- /* Ensure that the OCS is no longer busy before reading the chains. */
- rc = ocs_hcu_wait_busy(hcu_dev);
- if (rc)
- return rc;
-
- chain = (u32 *)dgst;
- for (i = 0; i < dgst_len / sizeof(u32); i++)
- chain[i] = readl(hcu_dev->io_base + OCS_HCU_CHAIN);
-
- return 0;
-}
-
-/**
- * ocs_hcu_hw_cfg() - Configure the HCU hardware.
- * @hcu_dev: The HCU device to configure.
- * @algo: The algorithm to be used by the HCU device.
- * @use_hmac: Whether or not HW HMAC should be used.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int ocs_hcu_hw_cfg(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
- bool use_hmac)
-{
- u32 cfg;
- int rc;
-
- if (algo != OCS_HCU_ALGO_SHA256 && algo != OCS_HCU_ALGO_SHA224 &&
- algo != OCS_HCU_ALGO_SHA384 && algo != OCS_HCU_ALGO_SHA512 &&
- algo != OCS_HCU_ALGO_SM3)
- return -EINVAL;
-
- rc = ocs_hcu_wait_busy(hcu_dev);
- if (rc)
- return rc;
-
- /* Ensure interrupts are disabled. */
- ocs_hcu_irq_dis(hcu_dev);
-
- /* Configure endianness, hashing algorithm and HW HMAC (if needed) */
- cfg = OCS_HCU_ENDIANNESS_VALUE << HCU_DATA_WRITE_ENDIANNESS_OFFSET;
- cfg |= algo << HCU_MODE_ALGO_SHIFT;
- if (use_hmac)
- cfg |= BIT(HCU_MODE_HMAC_SHIFT);
-
- writel(cfg, hcu_dev->io_base + OCS_HCU_MODE);
-
- return 0;
-}
-
-/**
- * ocs_hcu_clear_key() - Clear key stored in OCS HMAC KEY registers.
- * @hcu_dev: The OCS HCU device whose key registers should be cleared.
- */
-static void ocs_hcu_clear_key(struct ocs_hcu_dev *hcu_dev)
-{
- int reg_off;
-
- /* Clear OCS_HCU_KEY_[0..15] */
- for (reg_off = 0; reg_off < OCS_HCU_HW_KEY_LEN; reg_off += sizeof(u32))
- writel(0, hcu_dev->io_base + OCS_HCU_KEY_0 + reg_off);
-}
-
-/**
- * ocs_hcu_write_key() - Write key to OCS HMAC KEY registers.
- * @hcu_dev: The OCS HCU device the key should be written to.
- * @key: The key to be written.
- * @len: The size of the key to write. It must be OCS_HCU_HW_KEY_LEN.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int ocs_hcu_write_key(struct ocs_hcu_dev *hcu_dev, const u8 *key, size_t len)
-{
- u32 key_u32[OCS_HCU_HW_KEY_LEN_U32];
- int i;
-
- if (len > OCS_HCU_HW_KEY_LEN)
- return -EINVAL;
-
- /* Copy key into temporary u32 array. */
- memcpy(key_u32, key, len);
-
- /*
- * Hardware requires all the bytes of the HW Key vector to be
- * written. So pad with zero until we reach OCS_HCU_HW_KEY_LEN.
- */
- memzero_explicit((u8 *)key_u32 + len, OCS_HCU_HW_KEY_LEN - len);
-
- /*
- * OCS hardware expects the MSB of the key to be written at the highest
- * address of the HCU Key vector; in other word, the key must be
- * written in reverse order.
- *
- * Therefore, we first enable byte swapping for the HCU key vector;
- * so that bytes of 32-bit word written to OCS_HCU_KEY_[0..15] will be
- * swapped:
- * 3 <---> 0, 2 <---> 1.
- */
- writel(HCU_BYTE_ORDER_SWAP,
- hcu_dev->io_base + OCS_HCU_KEY_BYTE_ORDER_CFG);
- /*
- * And then we write the 32-bit words composing the key starting from
- * the end of the key.
- */
- for (i = 0; i < OCS_HCU_HW_KEY_LEN_U32; i++)
- writel(key_u32[OCS_HCU_HW_KEY_LEN_U32 - 1 - i],
- hcu_dev->io_base + OCS_HCU_KEY_0 + (sizeof(u32) * i));
-
- memzero_explicit(key_u32, OCS_HCU_HW_KEY_LEN);
-
- return 0;
-}
-
-/**
- * ocs_hcu_ll_dma_start() - Start OCS HCU hashing via DMA
- * @hcu_dev: The OCS HCU device to use.
- * @dma_list: The OCS DMA list mapping the data to hash.
- * @finalize: Whether or not this is the last hashing operation and therefore
- * the final hash should be compute even if data is not
- * block-aligned.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-static int ocs_hcu_ll_dma_start(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_dma_list *dma_list,
- bool finalize)
-{
- u32 cfg = HCU_DMA_SNOOP_MASK | HCU_DMA_SRC_LL_EN | HCU_DMA_EN;
- int rc;
-
- if (!dma_list)
- return -EINVAL;
-
- /*
- * For final requests we use HCU_DONE IRQ to be notified when all input
- * data has been processed by the HCU; however, we cannot do so for
- * non-final requests, because we don't get a HCU_DONE IRQ when we
- * don't terminate the operation.
- *
- * Therefore, for non-final requests, we use the DMA IRQ, which
- * triggers when DMA has finishing feeding all the input data to the
- * HCU, but the HCU may still be processing it. This is fine, since we
- * will wait for the HCU processing to be completed when we try to read
- * intermediate results, in ocs_hcu_get_intermediate_data().
- */
- if (finalize)
- ocs_hcu_done_irq_en(hcu_dev);
- else
- ocs_hcu_dma_irq_en(hcu_dev);
-
- reinit_completion(&hcu_dev->irq_done);
- writel(dma_list->dma_addr, hcu_dev->io_base + OCS_HCU_DMA_NEXT_SRC_DESCR);
- writel(0, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE);
- writel(0, hcu_dev->io_base + OCS_HCU_DMA_DST_SIZE);
-
- writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION);
-
- writel(cfg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE);
-
- if (finalize)
- writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
-
- rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
- if (rc)
- return rc;
-
- return 0;
-}
-
-struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev,
- int max_nents)
-{
- struct ocs_hcu_dma_list *dma_list;
-
- dma_list = kmalloc(sizeof(*dma_list), GFP_KERNEL);
- if (!dma_list)
- return NULL;
-
- /* Total size of the DMA list to allocate. */
- dma_list->head = dma_alloc_coherent(hcu_dev->dev,
- sizeof(*dma_list->head) * max_nents,
- &dma_list->dma_addr, GFP_KERNEL);
- if (!dma_list->head) {
- kfree(dma_list);
- return NULL;
- }
- dma_list->max_nents = max_nents;
- dma_list->tail = NULL;
-
- return dma_list;
-}
-
-void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_dma_list *dma_list)
-{
- if (!dma_list)
- return;
-
- dma_free_coherent(hcu_dev->dev,
- sizeof(*dma_list->head) * dma_list->max_nents,
- dma_list->head, dma_list->dma_addr);
-
- kfree(dma_list);
-}
-
-/* Add a new DMA entry at the end of the OCS DMA list. */
-int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_dma_list *dma_list,
- dma_addr_t addr, u32 len)
-{
- struct device *dev = hcu_dev->dev;
- struct ocs_hcu_dma_entry *old_tail;
- struct ocs_hcu_dma_entry *new_tail;
-
- if (!len)
- return 0;
-
- if (!dma_list)
- return -EINVAL;
-
- if (addr & ~OCS_HCU_DMA_BIT_MASK) {
- dev_err(dev,
- "Unexpected error: Invalid DMA address for OCS HCU\n");
- return -EINVAL;
- }
-
- old_tail = dma_list->tail;
- new_tail = old_tail ? old_tail + 1 : dma_list->head;
-
- /* Check if list is full. */
- if (new_tail - dma_list->head >= dma_list->max_nents)
- return -ENOMEM;
-
- /*
- * If there was an old tail (i.e., this is not the first element we are
- * adding), un-terminate the old tail and make it point to the new one.
- */
- if (old_tail) {
- old_tail->ll_flags &= ~OCS_LL_DMA_FLAG_TERMINATE;
- /*
- * The old tail 'nxt_desc' must point to the DMA address of the
- * new tail.
- */
- old_tail->nxt_desc = dma_list->dma_addr +
- sizeof(*dma_list->tail) * (new_tail -
- dma_list->head);
- }
-
- new_tail->src_addr = (u32)addr;
- new_tail->src_len = (u32)len;
- new_tail->ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
- new_tail->nxt_desc = 0;
-
- /* Update list tail with new tail. */
- dma_list->tail = new_tail;
-
- return 0;
-}
-
-/**
- * ocs_hcu_hash_init() - Initialize hash operation context.
- * @ctx: The context to initialize.
- * @algo: The hashing algorithm to use.
- *
- * Return: 0 on success, negative error code otherwise.
- */
-int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo)
-{
- if (!ctx)
- return -EINVAL;
-
- ctx->algo = algo;
- ctx->idata.msg_len_lo = 0;
- ctx->idata.msg_len_hi = 0;
- /* No need to set idata.digest to 0. */
-
- return 0;
-}
-
-/**
- * ocs_hcu_hash_update() - Perform a hashing iteration.
- * @hcu_dev: The OCS HCU device to use.
- * @ctx: The OCS HCU hashing context.
- * @dma_list: The OCS DMA list mapping the input data to process.
- *
- * Return: 0 on success; negative error code otherwise.
- */
-int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_hash_ctx *ctx,
- const struct ocs_hcu_dma_list *dma_list)
-{
- int rc;
-
- if (!hcu_dev || !ctx)
- return -EINVAL;
-
- /* Configure the hardware for the current request. */
- rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
- if (rc)
- return rc;
-
- /* If we already processed some data, idata needs to be set. */
- if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
- ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
-
- /* Start linked-list DMA hashing. */
- rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, false);
- if (rc)
- return rc;
-
- /* Update idata and return. */
- return ocs_hcu_get_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
-}
-
-/**
- * ocs_hcu_hash_finup() - Update and finalize hash computation.
- * @hcu_dev: The OCS HCU device to use.
- * @ctx: The OCS HCU hashing context.
- * @dma_list: The OCS DMA list mapping the input data to process.
- * @dgst: The buffer where to save the computed digest.
- * @dgst_len: The length of @dgst.
- *
- * Return: 0 on success; negative error code otherwise.
- */
-int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_hash_ctx *ctx,
- const struct ocs_hcu_dma_list *dma_list,
- u8 *dgst, size_t dgst_len)
-{
- int rc;
-
- if (!hcu_dev || !ctx)
- return -EINVAL;
-
- /* Configure the hardware for the current request. */
- rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
- if (rc)
- return rc;
-
- /* If we already processed some data, idata needs to be set. */
- if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
- ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
-
- /* Start linked-list DMA hashing. */
- rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true);
- if (rc)
- return rc;
-
- /* Get digest and return. */
- return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len);
-}
-
-/**
- * ocs_hcu_hash_final() - Finalize hash computation.
- * @hcu_dev: The OCS HCU device to use.
- * @ctx: The OCS HCU hashing context.
- * @dgst: The buffer where to save the computed digest.
- * @dgst_len: The length of @dgst.
- *
- * Return: 0 on success; negative error code otherwise.
- */
-int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_hash_ctx *ctx, u8 *dgst,
- size_t dgst_len)
-{
- int rc;
-
- if (!hcu_dev || !ctx)
- return -EINVAL;
-
- /* Configure the hardware for the current request. */
- rc = ocs_hcu_hw_cfg(hcu_dev, ctx->algo, false);
- if (rc)
- return rc;
-
- /* If we already processed some data, idata needs to be set. */
- if (ctx->idata.msg_len_lo || ctx->idata.msg_len_hi)
- ocs_hcu_set_intermediate_data(hcu_dev, &ctx->idata, ctx->algo);
-
- /*
- * Enable HCU interrupts, so that HCU_DONE will be triggered once the
- * final hash is computed.
- */
- ocs_hcu_done_irq_en(hcu_dev);
- reinit_completion(&hcu_dev->irq_done);
- writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
-
- rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
- if (rc)
- return rc;
-
- /* Get digest and return. */
- return ocs_hcu_get_digest(hcu_dev, ctx->algo, dgst, dgst_len);
-}
-
-/**
- * ocs_hcu_digest() - Compute hash digest.
- * @hcu_dev: The OCS HCU device to use.
- * @algo: The hash algorithm to use.
- * @data: The input data to process.
- * @data_len: The length of @data.
- * @dgst: The buffer where to save the computed digest.
- * @dgst_len: The length of @dgst.
- *
- * Return: 0 on success; negative error code otherwise.
- */
-int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
- void *data, size_t data_len, u8 *dgst, size_t dgst_len)
-{
- struct device *dev = hcu_dev->dev;
- dma_addr_t dma_handle;
- u32 reg;
- int rc;
-
- /* Configure the hardware for the current request. */
- rc = ocs_hcu_hw_cfg(hcu_dev, algo, false);
- if (rc)
- return rc;
-
- dma_handle = dma_map_single(dev, data, data_len, DMA_TO_DEVICE);
- if (dma_mapping_error(dev, dma_handle))
- return -EIO;
-
- reg = HCU_DMA_SNOOP_MASK | HCU_DMA_EN;
-
- ocs_hcu_done_irq_en(hcu_dev);
-
- reinit_completion(&hcu_dev->irq_done);
-
- writel(dma_handle, hcu_dev->io_base + OCS_HCU_DMA_SRC_ADDR);
- writel(data_len, hcu_dev->io_base + OCS_HCU_DMA_SRC_SIZE);
- writel(OCS_HCU_START, hcu_dev->io_base + OCS_HCU_OPERATION);
- writel(reg, hcu_dev->io_base + OCS_HCU_DMA_DMA_MODE);
-
- writel(OCS_HCU_TERMINATE, hcu_dev->io_base + OCS_HCU_OPERATION);
-
- rc = ocs_hcu_wait_and_disable_irq(hcu_dev);
- if (rc)
- return rc;
-
- dma_unmap_single(dev, dma_handle, data_len, DMA_TO_DEVICE);
-
- return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len);
-}
-
-/**
- * ocs_hcu_hmac() - Compute HMAC.
- * @hcu_dev: The OCS HCU device to use.
- * @algo: The hash algorithm to use with HMAC.
- * @key: The key to use.
- * @dma_list: The OCS DMA list mapping the input data to process.
- * @key_len: The length of @key.
- * @dgst: The buffer where to save the computed HMAC.
- * @dgst_len: The length of @dgst.
- *
- * Return: 0 on success; negative error code otherwise.
- */
-int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
- const u8 *key, size_t key_len,
- const struct ocs_hcu_dma_list *dma_list,
- u8 *dgst, size_t dgst_len)
-{
- int rc;
-
- /* Ensure 'key' is not NULL. */
- if (!key || key_len == 0)
- return -EINVAL;
-
- /* Configure the hardware for the current request. */
- rc = ocs_hcu_hw_cfg(hcu_dev, algo, true);
- if (rc)
- return rc;
-
- rc = ocs_hcu_write_key(hcu_dev, key, key_len);
- if (rc)
- return rc;
-
- rc = ocs_hcu_ll_dma_start(hcu_dev, dma_list, true);
-
- /* Clear HW key before processing return code. */
- ocs_hcu_clear_key(hcu_dev);
-
- if (rc)
- return rc;
-
- return ocs_hcu_get_digest(hcu_dev, algo, dgst, dgst_len);
-}
-
-irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id)
-{
- struct ocs_hcu_dev *hcu_dev = dev_id;
- u32 hcu_irq;
- u32 dma_irq;
-
- /* Read and clear the HCU interrupt. */
- hcu_irq = readl(hcu_dev->io_base + OCS_HCU_ISR);
- writel(hcu_irq, hcu_dev->io_base + OCS_HCU_ISR);
-
- /* Read and clear the HCU DMA interrupt. */
- dma_irq = readl(hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
- writel(dma_irq, hcu_dev->io_base + OCS_HCU_DMA_MSI_ISR);
-
- /* Check for errors. */
- if (hcu_irq & HCU_IRQ_HASH_ERR_MASK || dma_irq & HCU_DMA_IRQ_ERR_MASK) {
- hcu_dev->irq_err = true;
- goto complete;
- }
-
- /* Check for DONE IRQs. */
- if (hcu_irq & HCU_IRQ_HASH_DONE || dma_irq & HCU_DMA_IRQ_SRC_DONE)
- goto complete;
-
- return IRQ_NONE;
-
-complete:
- complete(&hcu_dev->irq_done);
-
- return IRQ_HANDLED;
-}
-
-MODULE_LICENSE("GPL");
+++ /dev/null
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Intel Keem Bay OCS HCU Crypto Driver.
- *
- * Copyright (C) 2018-2020 Intel Corporation
- */
-
-#include <linux/dma-mapping.h>
-
-#ifndef _CRYPTO_OCS_HCU_H
-#define _CRYPTO_OCS_HCU_H
-
-#define OCS_HCU_DMA_BIT_MASK DMA_BIT_MASK(32)
-
-#define OCS_HCU_HW_KEY_LEN 64
-
-struct ocs_hcu_dma_list;
-
-enum ocs_hcu_algo {
- OCS_HCU_ALGO_SHA256 = 2,
- OCS_HCU_ALGO_SHA224 = 3,
- OCS_HCU_ALGO_SHA384 = 4,
- OCS_HCU_ALGO_SHA512 = 5,
- OCS_HCU_ALGO_SM3 = 6,
-};
-
-/**
- * struct ocs_hcu_dev - OCS HCU device context.
- * @list: List of device contexts.
- * @dev: OCS HCU device.
- * @io_base: Base address of OCS HCU registers.
- * @engine: Crypto engine for the device.
- * @irq: IRQ number.
- * @irq_done: Completion for IRQ.
- * @irq_err: Flag indicating an IRQ error has happened.
- */
-struct ocs_hcu_dev {
- struct list_head list;
- struct device *dev;
- void __iomem *io_base;
- struct crypto_engine *engine;
- int irq;
- struct completion irq_done;
- bool irq_err;
-};
-
-/**
- * struct ocs_hcu_idata - Intermediate data generated by the HCU.
- * @msg_len_lo: Length of data the HCU has operated on in bits, low 32b.
- * @msg_len_hi: Length of data the HCU has operated on in bits, high 32b.
- * @digest: The digest read from the HCU. If the HCU is terminated, it will
- * contain the actual hash digest. Otherwise it is the intermediate
- * state.
- */
-struct ocs_hcu_idata {
- u32 msg_len_lo;
- u32 msg_len_hi;
- u8 digest[SHA512_DIGEST_SIZE];
-};
-
-/**
- * struct ocs_hcu_hash_ctx - Context for OCS HCU hashing operation.
- * @algo: The hashing algorithm being used.
- * @idata: The current intermediate data.
- */
-struct ocs_hcu_hash_ctx {
- enum ocs_hcu_algo algo;
- struct ocs_hcu_idata idata;
-};
-
-irqreturn_t ocs_hcu_irq_handler(int irq, void *dev_id);
-
-struct ocs_hcu_dma_list *ocs_hcu_dma_list_alloc(struct ocs_hcu_dev *hcu_dev,
- int max_nents);
-
-void ocs_hcu_dma_list_free(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_dma_list *dma_list);
-
-int ocs_hcu_dma_list_add_tail(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_dma_list *dma_list,
- dma_addr_t addr, u32 len);
-
-int ocs_hcu_hash_init(struct ocs_hcu_hash_ctx *ctx, enum ocs_hcu_algo algo);
-
-int ocs_hcu_hash_update(struct ocs_hcu_dev *hcu_dev,
- struct ocs_hcu_hash_ctx *ctx,
- const struct ocs_hcu_dma_list *dma_list);
-
-int ocs_hcu_hash_finup(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_hash_ctx *ctx,
- const struct ocs_hcu_dma_list *dma_list,
- u8 *dgst, size_t dgst_len);
-
-int ocs_hcu_hash_final(struct ocs_hcu_dev *hcu_dev,
- const struct ocs_hcu_hash_ctx *ctx, u8 *dgst,
- size_t dgst_len);
-
-int ocs_hcu_digest(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
- void *data, size_t data_len, u8 *dgst, size_t dgst_len);
-
-int ocs_hcu_hmac(struct ocs_hcu_dev *hcu_dev, enum ocs_hcu_algo algo,
- const u8 *key, size_t key_len,
- const struct ocs_hcu_dma_list *dma_list,
- u8 *dgst, size_t dgst_len);
-
-#endif /* _CRYPTO_OCS_HCU_H */
projects / linux.git / commitdiff