crypto: qat - add backlog mechanism

[ Upstream commit 38682383973280e5be2802ba8a8d4a636d36cb19 ]

The implementations of the crypto algorithms (aead, skcipher, etc) in
the QAT driver do not properly support requests with the
CRYPTO_TFM_REQ_MAY_BACKLOG flag set. If the HW queue is full, the driver
returns -EBUSY but does not enqueue the request. This can result in
applications like dm-crypt waiting indefinitely for the completion of a
request that was never submitted to the hardware.

Fix this by adding a software backlog queue: if the ring buffer is more
than eighty percent full, then the request is enqueued to a backlog
list and the error code -EBUSY is returned back to the caller.
Requests in the backlog queue are resubmitted at a later time, in the
context of the callback of a previously submitted request.
The request for which -EBUSY is returned is then marked as -EINPROGRESS
once submitted to the HW queues.

The submission loop inside the function qat_alg_send_message() has been
modified to decide which submission policy to use based on the request
flags. If the request does not have the CRYPTO_TFM_REQ_MAY_BACKLOG set,
the previous behaviour has been preserved.

Based on a patch by
Vishnu Das Ramachandran <vishnu.dasx.ramachandran@intel.com>

Cc: stable@vger.kernel.org
Fixes: d370cec32194 ("crypto: qat - Intel(R) QAT crypto interface")
Reported-by: Mikulas Patocka <mpatocka@redhat.com>
Reported-by: Kyle Sanderson <kyle.leet@gmail.com>
Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
Reviewed-by: Marco Chiappero <marco.chiappero@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c
index 8ba28409fb74b7ad9da74f6a9ec659a31b4124c2..630d0483c4e0a1a4134f9f73f0def8091e9e4a55 100644 (file)
--- a/drivers/crypto/qat/qat_common/adf_transport.c
+++ b/drivers/crypto/qat/qat_common/adf_transport.c
@@ -8,6 +8,9 @@
 #include "adf_cfg.h"
 #include "adf_common_drv.h"
 
+#define ADF_MAX_RING_THRESHOLD         80
+#define ADF_PERCENT(tot, percent)      (((tot) * (percent)) / 100)
+
 static inline u32 adf_modulo(u32 data, u32 shift)
 {
        u32 div = data >> shift;
@@ -77,6 +80,11 @@ static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
                                      bank->irq_mask);
 }
 
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)
+{
+       return atomic_read(ring->inflights) > ring->threshold;
+}
+
 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
 {
        struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
@@ -217,6 +225,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
        struct adf_etr_bank_data *bank;
        struct adf_etr_ring_data *ring;
        char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
+       int max_inflights;
        u32 ring_num;
        int ret;
 
@@ -263,6 +272,8 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
        ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);
        ring->head = 0;
        ring->tail = 0;
+       max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
+       ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);
        atomic_set(ring->inflights, 0);
        ret = adf_init_ring(ring);
        if (ret)

diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/qat/qat_common/adf_transport.h
index 2c95f1697c76fe2d3589318e8e641c17b84aa04d..e6ef6f9b769133a3b713cf71aaa600ae741edfed 100644 (file)
--- a/drivers/crypto/qat/qat_common/adf_transport.h
+++ b/drivers/crypto/qat/qat_common/adf_transport.h
@@ -14,6 +14,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
                    const char *ring_name, adf_callback_fn callback,
                    int poll_mode, struct adf_etr_ring_data **ring_ptr);
 
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring);
 int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg);
 void adf_remove_ring(struct adf_etr_ring_data *ring);
 #endif

diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h
index 501bcf0f1809afb34be0633cf0edd4c63d7ee608..8b2c92ba7ca1f029ccd99664f350230db83b1612 100644 (file)
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h
@@ -22,6 +22,7 @@ struct adf_etr_ring_data {
        spinlock_t lock;        /* protects ring data struct */
        u16 head;
        u16 tail;
+       u32 threshold;
        u8 ring_number;
        u8 ring_size;
        u8 msg_size;

diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 6017ae82c7133d493d782c02677d68eba13e3fb4..873533dc43a741c10d3af6c8253dcd9a9dcce2f0 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -935,19 +935,25 @@ void qat_alg_callback(void *resp)
        struct icp_qat_fw_la_resp *qat_resp = resp;
        struct qat_crypto_request *qat_req =
                                (void *)(__force long)qat_resp->opaque_data;
+       struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
 
        qat_req->cb(qat_resp, qat_req);
+
+       qat_alg_send_backlog(backlog);
 }
 
 static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
-                                   struct qat_crypto_instance *inst)
+                                   struct qat_crypto_instance *inst,
+                                   struct crypto_async_request *base)
 {
-       struct qat_alg_req req;
+       struct qat_alg_req *alg_req = &qat_req->alg_req;
 
-       req.fw_req = (u32 *)&qat_req->req;
-       req.tx_ring = inst->sym_tx;
+       alg_req->fw_req = (u32 *)&qat_req->req;
+       alg_req->tx_ring = inst->sym_tx;
+       alg_req->base = base;
+       alg_req->backlog = &inst->backlog;
 
-       return qat_alg_send_message(&req);
+       return qat_alg_send_message(alg_req);
 }
 
 static int qat_alg_aead_dec(struct aead_request *areq)
@@ -987,7 +993,7 @@ static int qat_alg_aead_dec(struct aead_request *areq)
        auth_param->auth_off = 0;
        auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
 
-       ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
        if (ret == -ENOSPC)
                qat_alg_free_bufl(ctx->inst, qat_req);
 
@@ -1031,7 +1037,7 @@ static int qat_alg_aead_enc(struct aead_request *areq)
        auth_param->auth_off = 0;
        auth_param->auth_len = areq->assoclen + areq->cryptlen;
 
-       ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
        if (ret == -ENOSPC)
                qat_alg_free_bufl(ctx->inst, qat_req);
 
@@ -1212,7 +1218,7 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
 
        qat_alg_set_req_iv(qat_req);
 
-       ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
        if (ret == -ENOSPC)
                qat_alg_free_bufl(ctx->inst, qat_req);
 
@@ -1278,7 +1284,7 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
        qat_alg_set_req_iv(qat_req);
        qat_alg_update_iv(qat_req);
 
-       ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
        if (ret == -ENOSPC)
                qat_alg_free_bufl(ctx->inst, qat_req);
 

diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.c b/drivers/crypto/qat/qat_common/qat_algs_send.c
index 78f1bb8c26c0817c12e692c04387d9f0c908c83d..ff5b4347f7831d8489d481c6f8c1d13099cbbb77 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_algs_send.c
+++ b/drivers/crypto/qat/qat_common/qat_algs_send.c
@@ -6,7 +6,7 @@
 
 #define ADF_MAX_RETRIES                20
 
-int qat_alg_send_message(struct qat_alg_req *req)
+static int qat_alg_send_message_retry(struct qat_alg_req *req)
 {
        int ret = 0, ctr = 0;
 
@@ -19,3 +19,68 @@ int qat_alg_send_message(struct qat_alg_req *req)
 
        return -EINPROGRESS;
 }
+
+void qat_alg_send_backlog(struct qat_instance_backlog *backlog)
+{
+       struct qat_alg_req *req, *tmp;
+
+       spin_lock_bh(&backlog->lock);
+       list_for_each_entry_safe(req, tmp, &backlog->list, list) {
+               if (adf_send_message(req->tx_ring, req->fw_req)) {
+                       /* The HW ring is full. Do nothing.
+                        * qat_alg_send_backlog() will be invoked again by
+                        * another callback.
+                        */
+                       break;
+               }
+               list_del(&req->list);
+               req->base->complete(req->base, -EINPROGRESS);
+       }
+       spin_unlock_bh(&backlog->lock);
+}
+
+static void qat_alg_backlog_req(struct qat_alg_req *req,
+                               struct qat_instance_backlog *backlog)
+{
+       INIT_LIST_HEAD(&req->list);
+
+       spin_lock_bh(&backlog->lock);
+       list_add_tail(&req->list, &backlog->list);
+       spin_unlock_bh(&backlog->lock);
+}
+
+static int qat_alg_send_message_maybacklog(struct qat_alg_req *req)
+{
+       struct qat_instance_backlog *backlog = req->backlog;
+       struct adf_etr_ring_data *tx_ring = req->tx_ring;
+       u32 *fw_req = req->fw_req;
+
+       /* If any request is already backlogged, then add to backlog list */
+       if (!list_empty(&backlog->list))
+               goto enqueue;
+
+       /* If ring is nearly full, then add to backlog list */
+       if (adf_ring_nearly_full(tx_ring))
+               goto enqueue;
+
+       /* If adding request to HW ring fails, then add to backlog list */
+       if (adf_send_message(tx_ring, fw_req))
+               goto enqueue;
+
+       return -EINPROGRESS;
+
+enqueue:
+       qat_alg_backlog_req(req, backlog);
+
+       return -EBUSY;
+}
+
+int qat_alg_send_message(struct qat_alg_req *req)
+{
+       u32 flags = req->base->flags;
+
+       if (flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+               return qat_alg_send_message_maybacklog(req);
+       else
+               return qat_alg_send_message_retry(req);
+}

diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.h b/drivers/crypto/qat/qat_common/qat_algs_send.h
index 3fa685d0c293aa54aa8711fafa042b74549d6fdb..5ce9f4f69d8ff8339e9b64c215a19d57da4a13cd 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_algs_send.h
+++ b/drivers/crypto/qat/qat_common/qat_algs_send.h
@@ -6,5 +6,6 @@
 #include "qat_crypto.h"
 
 int qat_alg_send_message(struct qat_alg_req *req);
+void qat_alg_send_backlog(struct qat_instance_backlog *backlog);
 
 #endif

diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 08b8d83e070a0ecb7c65c033d60118d459755c8d..ff7249c093c9b9fed478f94d12f55ffda83a2c9d 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -136,17 +136,21 @@ struct qat_asym_request {
        } areq;
        int err;
        void (*cb)(struct icp_qat_fw_pke_resp *resp);
+       struct qat_alg_req alg_req;
 } __aligned(64);
 
 static int qat_alg_send_asym_message(struct qat_asym_request *qat_req,
-                                    struct qat_crypto_instance *inst)
+                                    struct qat_crypto_instance *inst,
+                                    struct crypto_async_request *base)
 {
-       struct qat_alg_req req;
+       struct qat_alg_req *alg_req = &qat_req->alg_req;
 
-       req.fw_req = (u32 *)&qat_req->req;
-       req.tx_ring = inst->pke_tx;
+       alg_req->fw_req = (u32 *)&qat_req->req;
+       alg_req->tx_ring = inst->pke_tx;
+       alg_req->base = base;
+       alg_req->backlog = &inst->backlog;
 
-       return qat_alg_send_message(&req);
+       return qat_alg_send_message(alg_req);
 }
 
 static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp)
@@ -350,7 +354,7 @@ static int qat_dh_compute_value(struct kpp_request *req)
        msg->input_param_count = n_input_params;
        msg->output_param_count = 1;
 
-       ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
        if (ret == -ENOSPC)
                goto unmap_all;
 
@@ -557,8 +561,11 @@ void qat_alg_asym_callback(void *_resp)
 {
        struct icp_qat_fw_pke_resp *resp = _resp;
        struct qat_asym_request *areq = (void *)(__force long)resp->opaque;
+       struct qat_instance_backlog *backlog = areq->alg_req.backlog;
 
        areq->cb(resp);
+
+       qat_alg_send_backlog(backlog);
 }
 
 #define PKE_RSA_EP_512 0x1c161b21
@@ -748,7 +755,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
        msg->input_param_count = 3;
        msg->output_param_count = 1;
 
-       ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
        if (ret == -ENOSPC)
                goto unmap_all;
 
@@ -901,7 +908,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
 
        msg->output_param_count = 1;
 
-       ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+       ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
        if (ret == -ENOSPC)
                goto unmap_all;
 

diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
index 3efbb388360100b22856032e8f032f6524498c7d..59e122afa4343d84f702049f6152dcf3e9355b04 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/qat/qat_common/qat_crypto.c
@@ -328,6 +328,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
                                      &inst->pke_rx);
                if (ret)
                        goto err;
+
+               INIT_LIST_HEAD(&inst->backlog.list);
+               spin_lock_init(&inst->backlog.lock);
        }
        return 0;
 err:

diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index 0dcba6fc358c92f6c02ede5c2166d668c0a3a752..245b6d9a365078145af80125a987793663fd35b2 100644 (file)
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -9,9 +9,17 @@
 #include "adf_accel_devices.h"
 #include "icp_qat_fw_la.h"
 
+struct qat_instance_backlog {
+       struct list_head list;
+       spinlock_t lock; /* protects backlog list */
+};
+
 struct qat_alg_req {
        u32 *fw_req;
        struct adf_etr_ring_data *tx_ring;
+       struct crypto_async_request *base;
+       struct list_head list;
+       struct qat_instance_backlog *backlog;
 };
 
 struct qat_crypto_instance {
@@ -24,6 +32,7 @@ struct qat_crypto_instance {
        unsigned long state;
        int id;
        atomic_t refctr;
+       struct qat_instance_backlog backlog;
 };
 
 #define QAT_MAX_BUFF_DESC      4
@@ -82,6 +91,7 @@ struct qat_crypto_request {
                u8 iv[AES_BLOCK_SIZE];
        };
        bool encryption;
+       struct qat_alg_req alg_req;
 };
 
 static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev)