return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen);
}
+static int alloc_src_sglist(struct skcipher_request *skreq, int ivsize)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ int nents = sg_nents(skreq->src) + 1;
+ struct se_crypto_request *creq = &nkreq->creq;
+ char *iv;
+ struct scatterlist *sg;
+
+ /* Allocate buffer to hold IV and input scatterlist array */
+ nkreq->src = alloc_req_buf(nents, ivsize, creq->gfp);
+ if (!nkreq->src)
+ return -ENOMEM;
+
+ /* copy iv */
+ iv = nkreq->src;
+ memcpy(iv, skreq->iv, ivsize);
+
+ sg = (struct scatterlist *)(iv + ivsize);
+ creq->src = sg;
+ sg_init_table(sg, nents);
+
+ /* Input format:
+ * +----+----------------+
+ * | IV | SRC sg entries |
+ * +----+----------------+
+ */
+
+ /* IV */
+ sg = create_single_sg(sg, iv, ivsize);
+ /* SRC entries */
+ create_multi_sg(sg, skreq->src);
+
+ return 0;
+}
+
+static int alloc_dst_sglist(struct skcipher_request *skreq, int ivsize)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+ int nents = sg_nents(skreq->dst) + 3;
+ int extralen = ORH_HLEN + COMP_HLEN;
+ struct se_crypto_request *creq = &nkreq->creq;
+ struct scatterlist *sg;
+ char *iv = nkreq->src;
+
+ /* Allocate buffer to hold ORH, COMPLETION and output scatterlist
+ * array
+ */
+ nkreq->dst = alloc_req_buf(nents, extralen, creq->gfp);
+ if (!nkreq->dst)
+ return -ENOMEM;
+
+ creq->orh = (u64 *)(nkreq->dst);
+ set_orh_value(creq->orh);
+
+ creq->comp = (u64 *)(nkreq->dst + ORH_HLEN);
+ set_comp_value(creq->comp);
+
+ sg = (struct scatterlist *)(nkreq->dst + ORH_HLEN + COMP_HLEN);
+ creq->dst = sg;
+ sg_init_table(sg, nents);
+
+ /* Output format:
+ * +-----+----+----------------+-----------------+
+ * | ORH | IV | DST sg entries | COMPLETION Bytes|
+ * +-----+----+----------------+-----------------+
+ */
+
+ /* ORH */
+ sg = create_single_sg(sg, creq->orh, ORH_HLEN);
+ /* IV */
+ sg = create_single_sg(sg, iv, ivsize);
+ /* DST entries */
+ sg = create_multi_sg(sg, skreq->dst);
+ /* COMPLETION Bytes */
+ create_single_sg(sg, creq->comp, COMP_HLEN);
+
+ return 0;
+}
+
+static void free_src_sglist(struct skcipher_request *skreq)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+
+ kfree(nkreq->src);
+}
+
+static void free_dst_sglist(struct skcipher_request *skreq)
+{
+ struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
+
+ kfree(nkreq->dst);
+}
+
static void nitrox_skcipher_callback(struct skcipher_request *skreq,
int err)
{
+ free_src_sglist(skreq);
+ free_dst_sglist(skreq);
if (err) {
pr_err_ratelimited("request failed status 0x%0x\n", err);
err = -EINVAL;
}
+
skcipher_request_complete(skreq, err);
}
struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq);
int ivsize = crypto_skcipher_ivsize(cipher);
struct se_crypto_request *creq;
+ int ret;
creq = &nkreq->creq;
creq->flags = skreq->base.flags;
creq->ctx_handle = nctx->u.ctx_handle;
creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
- /* copy the iv */
- memcpy(creq->iv, skreq->iv, ivsize);
- creq->ivsize = ivsize;
- creq->src = skreq->src;
- creq->dst = skreq->dst;
+ ret = alloc_src_sglist(skreq, ivsize);
+ if (ret)
+ return ret;
+
+ ret = alloc_dst_sglist(skreq, ivsize);
+ if (ret) {
+ free_src_sglist(skreq);
+ return ret;
+ }
nkreq->nctx = nctx;
nkreq->skreq = skreq;
#include "nitrox_dev.h"
+#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
+
/**
* struct gphdr - General purpose Header
* @param0: first parameter.
} s;
};
-struct nitrox_sglist {
- u16 len;
- u16 raz0;
- u32 raz1;
- dma_addr_t dma;
-};
-
#define MAX_IV_LEN 16
/**
* @ctx_handle: Crypto context handle.
* @gph: GP Header
* @ctrl: Request Information.
- * @in: Input sglist
- * @out: Output sglist
+ * @orh: ORH address
+ * @comp: completion address
+ * @src: Input sglist
+ * @dst: Output sglist
*/
struct se_crypto_request {
u8 opcode;
struct gphdr gph;
union se_req_ctrl ctrl;
-
- u8 iv[MAX_IV_LEN];
- u16 ivsize;
+ u64 *orh;
+ u64 *comp;
struct scatterlist *src;
struct scatterlist *dst;
struct se_crypto_request creq;
struct nitrox_crypto_ctx *nctx;
struct skcipher_request *skreq;
+ u8 *src;
+ u8 *dst;
};
/**
/*
* strutct nitrox_sgtable - SG list information
- * @map_cnt: Number of buffers mapped
- * @nr_comp: Number of sglist components
+ * @sgmap_cnt: Number of buffers mapped
* @total_bytes: Total bytes in sglist.
- * @len: Total sglist components length.
- * @dma: DMA address of sglist component.
- * @dir: DMA direction.
- * @buf: crypto request buffer.
- * @sglist: SG list of input/output buffers.
+ * @sgcomp_len: Total sglist components length.
+ * @sgcomp_dma: DMA address of sglist component.
+ * @sg: crypto request buffer.
* @sgcomp: sglist component for NITROX.
*/
struct nitrox_sgtable {
- u8 map_bufs_cnt;
- u8 nr_sgcomp;
+ u8 sgmap_cnt;
u16 total_bytes;
- u32 len;
- dma_addr_t dma;
- enum dma_data_direction dir;
-
- struct scatterlist *buf;
- struct nitrox_sglist *sglist;
+ u32 sgcomp_len;
+ dma_addr_t sgcomp_dma;
+ struct scatterlist *sg;
struct nitrox_sgcomp *sgcomp;
};
#define COMP_HLEN 8
struct resp_hdr {
- u64 orh;
- dma_addr_t orh_dma;
- u64 completion;
- dma_addr_t completion_dma;
+ u64 *orh;
+ u64 *completion;
};
typedef void (*completion_t)(struct skcipher_request *skreq, int err);
u32 flags;
gfp_t gfp;
atomic_t status;
- bool inplace;
struct nitrox_device *ndev;
struct nitrox_cmdq *cmdq;
struct skcipher_request *skreq;
};
+static inline void *alloc_req_buf(int nents, int extralen, gfp_t gfp)
+{
+ size_t size;
+
+ size = sizeof(struct scatterlist) * nents;
+ size += extralen;
+
+ return kzalloc(size, gfp);
+}
+
+static inline struct scatterlist *create_single_sg(struct scatterlist *sg,
+ void *buf, int buflen)
+{
+ sg_set_buf(sg, buf, buflen);
+ sg++;
+ return sg;
+}
+
+static inline struct scatterlist *create_multi_sg(struct scatterlist *to_sg,
+ struct scatterlist *from_sg)
+{
+ struct scatterlist *sg;
+ int i;
+
+ for_each_sg(from_sg, sg, sg_nents(from_sg), i) {
+ sg_set_buf(to_sg, sg_virt(sg), sg->length);
+ to_sg++;
+ }
+
+ return to_sg;
+}
+
+static inline void set_orh_value(u64 *orh)
+{
+ WRITE_ONCE(*orh, PENDING_SIG);
+}
+
+static inline void set_comp_value(u64 *comp)
+{
+ WRITE_ONCE(*comp, PENDING_SIG);
+}
+
#endif /* __NITROX_REQ_H */
#define FDATA_SIZE 32
/* Base destination port for the solicited requests */
#define SOLICIT_BASE_DPORT 256
-#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL
#define REQ_NOT_POSTED 1
#define REQ_BACKLOG 2
return index;
}
-/**
- * dma_free_sglist - unmap and free the sg lists.
- * @ndev: N5 device
- * @sgtbl: SG table
- */
static void softreq_unmap_sgbufs(struct nitrox_softreq *sr)
{
struct nitrox_device *ndev = sr->ndev;
struct device *dev = DEV(ndev);
- struct nitrox_sglist *sglist;
-
- /* unmap in sgbuf */
- sglist = sr->in.sglist;
- if (!sglist)
- goto out_unmap;
-
- /* unmap iv */
- dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL);
- /* unmpa src sglist */
- dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir);
- /* unamp gather component */
- dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE);
- kfree(sr->in.sglist);
+
+
+ dma_unmap_sg(dev, sr->in.sg, sr->in.sgmap_cnt, DMA_BIDIRECTIONAL);
+ dma_unmap_single(dev, sr->in.sgcomp_dma, sr->in.sgcomp_len,
+ DMA_TO_DEVICE);
kfree(sr->in.sgcomp);
- sr->in.sglist = NULL;
- sr->in.buf = NULL;
- sr->in.map_bufs_cnt = 0;
-
-out_unmap:
- /* unmap out sgbuf */
- sglist = sr->out.sglist;
- if (!sglist)
- return;
-
- /* unmap orh */
- dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
-
- /* unmap dst sglist */
- if (!sr->inplace) {
- dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3),
- sr->out.dir);
- }
- /* unmap completion */
- dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
+ sr->in.sg = NULL;
+ sr->in.sgmap_cnt = 0;
- /* unmap scatter component */
- dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE);
- kfree(sr->out.sglist);
+ dma_unmap_sg(dev, sr->out.sg, sr->out.sgmap_cnt,
+ DMA_BIDIRECTIONAL);
+ dma_unmap_single(dev, sr->out.sgcomp_dma, sr->out.sgcomp_len,
+ DMA_TO_DEVICE);
kfree(sr->out.sgcomp);
- sr->out.sglist = NULL;
- sr->out.buf = NULL;
- sr->out.map_bufs_cnt = 0;
+ sr->out.sg = NULL;
+ sr->out.sgmap_cnt = 0;
}
static void softreq_destroy(struct nitrox_softreq *sr)
* create_sg_component - create SG componets for N5 device.
* @sr: Request structure
* @sgtbl: SG table
- * @nr_comp: total number of components required
+ * @map_nents: number of dma mapped entries
*
* Component structure
*
{
struct nitrox_device *ndev = sr->ndev;
struct nitrox_sgcomp *sgcomp;
- struct nitrox_sglist *sglist;
+ struct scatterlist *sg;
dma_addr_t dma;
size_t sz_comp;
int i, j, nr_sgcomp;
return -ENOMEM;
sgtbl->sgcomp = sgcomp;
- sgtbl->nr_sgcomp = nr_sgcomp;
- sglist = sgtbl->sglist;
+ sg = sgtbl->sg;
/* populate device sg component */
for (i = 0; i < nr_sgcomp; i++) {
- for (j = 0; j < 4; j++) {
- sgcomp->len[j] = cpu_to_be16(sglist->len);
- sgcomp->dma[j] = cpu_to_be64(sglist->dma);
- sglist++;
+ for (j = 0; j < 4 && sg; j++) {
+ sgcomp[i].len[j] = cpu_to_be16(sg_dma_len(sg));
+ sgcomp[i].dma[j] = cpu_to_be64(sg_dma_address(sg));
+ sg = sg_next(sg);
}
- sgcomp++;
}
/* map the device sg component */
dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE);
return -ENOMEM;
}
- sgtbl->dma = dma;
- sgtbl->len = sz_comp;
+ sgtbl->sgcomp_dma = dma;
+ sgtbl->sgcomp_len = sz_comp;
return 0;
}
{
struct device *dev = DEV(sr->ndev);
struct scatterlist *sg = req->src;
- struct nitrox_sglist *glist;
int i, nents, ret = 0;
- dma_addr_t dma;
- size_t sz;
- nents = sg_nents(req->src);
+ nents = dma_map_sg(dev, req->src, sg_nents(req->src),
+ DMA_BIDIRECTIONAL);
+ if (!nents)
+ return -EINVAL;
- /* creater gather list IV and src entries */
- sz = roundup((1 + nents), 4) * sizeof(*glist);
- glist = kzalloc(sz, sr->gfp);
- if (!glist)
- return -ENOMEM;
-
- sr->in.sglist = glist;
- /* map IV */
- dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL);
- if (dma_mapping_error(dev, dma)) {
- ret = -EINVAL;
- goto iv_map_err;
- }
-
- sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE;
- /* map src entries */
- nents = dma_map_sg(dev, req->src, nents, sr->in.dir);
- if (!nents) {
- ret = -EINVAL;
- goto src_map_err;
- }
- sr->in.buf = req->src;
-
- /* store the mappings */
- glist->len = req->ivsize;
- glist->dma = dma;
- glist++;
- sr->in.total_bytes += req->ivsize;
-
- for_each_sg(req->src, sg, nents, i) {
- glist->len = sg_dma_len(sg);
- glist->dma = sg_dma_address(sg);
- sr->in.total_bytes += glist->len;
- glist++;
- }
- /* roundup map count to align with entires in sg component */
- sr->in.map_bufs_cnt = (1 + nents);
+ for_each_sg(req->src, sg, nents, i)
+ sr->in.total_bytes += sg_dma_len(sg);
- /* create NITROX gather component */
- ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt);
+ sr->in.sg = req->src;
+ sr->in.sgmap_cnt = nents;
+ ret = create_sg_component(sr, &sr->in, sr->in.sgmap_cnt);
if (ret)
goto incomp_err;
return 0;
incomp_err:
- dma_unmap_sg(dev, req->src, nents, sr->in.dir);
- sr->in.map_bufs_cnt = 0;
-src_map_err:
- dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL);
-iv_map_err:
- kfree(sr->in.sglist);
- sr->in.sglist = NULL;
+ dma_unmap_sg(dev, req->src, nents, DMA_BIDIRECTIONAL);
+ sr->in.sgmap_cnt = 0;
return ret;
}
struct se_crypto_request *req)
{
struct device *dev = DEV(sr->ndev);
- struct nitrox_sglist *glist = sr->in.sglist;
- struct nitrox_sglist *slist;
- struct scatterlist *sg;
- int i, nents, map_bufs_cnt, ret = 0;
- size_t sz;
+ int nents, ret = 0;
- nents = sg_nents(req->dst);
+ nents = dma_map_sg(dev, req->dst, sg_nents(req->dst),
+ DMA_BIDIRECTIONAL);
+ if (!nents)
+ return -EINVAL;
- /* create scatter list ORH, IV, dst entries and Completion header */
- sz = roundup((3 + nents), 4) * sizeof(*slist);
- slist = kzalloc(sz, sr->gfp);
- if (!slist)
- return -ENOMEM;
-
- sr->out.sglist = slist;
- sr->out.dir = DMA_BIDIRECTIONAL;
- /* map ORH */
- sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN,
- sr->out.dir);
- if (dma_mapping_error(dev, sr->resp.orh_dma)) {
- ret = -EINVAL;
- goto orh_map_err;
- }
-
- /* map completion */
- sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion,
- COMP_HLEN, sr->out.dir);
- if (dma_mapping_error(dev, sr->resp.completion_dma)) {
- ret = -EINVAL;
- goto compl_map_err;
- }
-
- sr->inplace = (req->src == req->dst) ? true : false;
- /* out place */
- if (!sr->inplace) {
- nents = dma_map_sg(dev, req->dst, nents, sr->out.dir);
- if (!nents) {
- ret = -EINVAL;
- goto dst_map_err;
- }
- }
- sr->out.buf = req->dst;
-
- /* store the mappings */
- /* orh */
- slist->len = ORH_HLEN;
- slist->dma = sr->resp.orh_dma;
- slist++;
-
- /* copy the glist mappings */
- if (sr->inplace) {
- nents = sr->in.map_bufs_cnt - 1;
- map_bufs_cnt = sr->in.map_bufs_cnt;
- while (map_bufs_cnt--) {
- slist->len = glist->len;
- slist->dma = glist->dma;
- slist++;
- glist++;
- }
- } else {
- /* copy iv mapping */
- slist->len = glist->len;
- slist->dma = glist->dma;
- slist++;
- /* copy remaining maps */
- for_each_sg(req->dst, sg, nents, i) {
- slist->len = sg_dma_len(sg);
- slist->dma = sg_dma_address(sg);
- slist++;
- }
- }
-
- /* completion */
- slist->len = COMP_HLEN;
- slist->dma = sr->resp.completion_dma;
-
- sr->out.map_bufs_cnt = (3 + nents);
-
- ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt);
+ sr->out.sg = req->dst;
+ sr->out.sgmap_cnt = nents;
+ ret = create_sg_component(sr, &sr->out, sr->out.sgmap_cnt);
if (ret)
goto outcomp_map_err;
return 0;
outcomp_map_err:
- if (!sr->inplace)
- dma_unmap_sg(dev, req->dst, nents, sr->out.dir);
- sr->out.map_bufs_cnt = 0;
- sr->out.buf = NULL;
-dst_map_err:
- dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir);
- sr->resp.completion_dma = 0;
-compl_map_err:
- dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir);
- sr->resp.orh_dma = 0;
-orh_map_err:
- kfree(sr->out.sglist);
- sr->out.sglist = NULL;
+ dma_unmap_sg(dev, req->dst, nents, DMA_BIDIRECTIONAL);
+ sr->out.sgmap_cnt = 0;
+ sr->out.sg = NULL;
return ret;
}
atomic_set(&sr->status, REQ_NOT_POSTED);
- WRITE_ONCE(sr->resp.orh, PENDING_SIG);
- WRITE_ONCE(sr->resp.completion, PENDING_SIG);
+ sr->resp.orh = req->orh;
+ sr->resp.completion = req->comp;
ret = softreq_map_iobuf(sr, req);
if (ret) {
/* fill the packet instruction */
/* word 0 */
- sr->instr.dptr0 = cpu_to_be64(sr->in.dma);
+ sr->instr.dptr0 = cpu_to_be64(sr->in.sgcomp_dma);
/* word 1 */
sr->instr.ih.value = 0;
sr->instr.ih.s.g = 1;
- sr->instr.ih.s.gsz = sr->in.map_bufs_cnt;
- sr->instr.ih.s.ssz = sr->out.map_bufs_cnt;
+ sr->instr.ih.s.gsz = sr->in.sgmap_cnt;
+ sr->instr.ih.s.ssz = sr->out.sgmap_cnt;
sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr);
sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes;
sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value);
/* word 4 */
sr->instr.slc.value[0] = 0;
- sr->instr.slc.s.ssz = sr->out.map_bufs_cnt;
+ sr->instr.slc.s.ssz = sr->out.sgmap_cnt;
sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]);
/* word 5 */
- sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma);
+ sr->instr.slc.s.rptr = cpu_to_be64(sr->out.sgcomp_dma);
/*
* No conversion for front data,
post_backlog_cmds(cmdq);
}
+static bool sr_completed(struct nitrox_softreq *sr)
+{
+ return (READ_ONCE(*sr->resp.orh) != READ_ONCE(*sr->resp.completion));
+}
+
/**
* process_request_list - process completed requests
* @ndev: N5 device
break;
/* check orh and completion bytes updates */
- if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) {
+ if (!sr_completed(sr)) {
/* request not completed, check for timeout */
if (!cmd_timeout(sr->tstamp, ndev->timeout))
break;
dev_err_ratelimited(DEV(ndev),
"Request timeout, orh 0x%016llx\n",
- READ_ONCE(sr->resp.orh));
+ READ_ONCE(*sr->resp.orh));
}
atomic_dec(&cmdq->pending_count);
atomic64_inc(&ndev->stats.completed);
skreq = sr->skreq;
/* ORH error code */
- err = READ_ONCE(sr->resp.orh) & 0xff;
+ err = READ_ONCE(*sr->resp.orh) & 0xff;
softreq_destroy(sr);
if (callback)
projects / linux.git / commitdiff