crypto: nx - fix concurrency issue

The NX driver uses the transformation context to store several fields
containing data related to the state of the operations in progress.
Since a single tfm can be used by different kernel threads at the same
time, we need to protect the data stored into the context.

This patch makes use of spin locks to protect the data where a race
condition can happen.

Reviewed-by: Fionnuala Gunter <[e-mail]>
Reviewed-by: Joy Latten <[e-mail]>
Signed-off-by: Marcelo Cerri <[e-mail]>
Signed-off-by: Herbert Xu <[e-mail]>

drivers/crypto/nx/nx-aes-cbc.c

···
 {
 	struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
 	int rc;
 
-	if (nbytes > nx_ctx->ap->databytelen)
-		return -EINVAL;
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes > nx_ctx->ap->databytelen) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	if (enc)
 		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
···
 	atomic64_add(csbcpb->csb.processed_byte_count,
 		     &(nx_ctx->stats->aes_bytes));
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-aes-ccm.c

···
 	unsigned int nbytes = req->cryptlen;
 	unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
 	struct nx_ccm_priv *priv = &nx_ctx->priv.ccm;
+	unsigned long irq_flags;
 	int rc = -1;
 
-	if (nbytes > nx_ctx->ap->databytelen)
-		return -EINVAL;
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes > nx_ctx->ap->databytelen) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	nbytes -= authsize;
 
···
 	rc = memcmp(csbcpb->cpb.aes_ccm.out_pat_or_mac, priv->oauth_tag,
 		    authsize) ? -EBADMSG : 0;
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 	unsigned int nbytes = req->cryptlen;
 	unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
+	unsigned long irq_flags;
 	int rc = -1;
 
-	if (nbytes > nx_ctx->ap->databytelen)
-		return -EINVAL;
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes > nx_ctx->ap->databytelen) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	rc = generate_pat(desc->info, req, nx_ctx, authsize, nbytes,
 			  csbcpb->cpb.aes_ccm.in_pat_or_b0);
···
 				 req->dst, nbytes, authsize,
 				 SCATTERWALK_TO_SG);
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-aes-ctr.c

···
 {
 	struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
 	int rc;
 
-	if (nbytes > nx_ctx->ap->databytelen)
-		return -EINVAL;
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes > nx_ctx->ap->databytelen) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	rc = nx_build_sg_lists(nx_ctx, desc, dst, src, nbytes,
 			       csbcpb->cpb.aes_ctr.iv);
···
 	atomic64_add(csbcpb->csb.processed_byte_count,
 		     &(nx_ctx->stats->aes_bytes));
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-aes-ecb.c

···
 {
 	struct nx_crypto_ctx *nx_ctx = crypto_blkcipher_ctx(desc->tfm);
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
+	unsigned long irq_flags;
 	int rc;
 
-	if (nbytes > nx_ctx->ap->databytelen)
-		return -EINVAL;
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
+
+	if (nbytes > nx_ctx->ap->databytelen) {
+		rc = -EINVAL;
+		goto out;
+	}
 
 	if (enc)
 		NX_CPB_FDM(csbcpb) |= NX_FDM_ENDE_ENCRYPT;
···
 	atomic64_add(csbcpb->csb.processed_byte_count,
 		     &(nx_ctx->stats->aes_bytes));
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-aes-gcm.c

···
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 	struct blkcipher_desc desc;
 	unsigned int nbytes = req->cryptlen;
+	unsigned long irq_flags;
 	int rc = -EINVAL;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	if (nbytes > nx_ctx->ap->databytelen)
 		goto out;
···
 		     -EBADMSG : 0;
 	}
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-aes-xcbc.c

···
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 	struct nx_sg *in_sg;
 	u32 to_process, leftover;
+	unsigned long irq_flags;
 	int rc = 0;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
 		/* we've hit the nx chip previously and we're updating again,
···
 	/* everything after the first update is continuation */
 	NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION;
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
 	struct nx_csbcpb *csbcpb = nx_ctx->csbcpb;
 	struct nx_sg *in_sg, *out_sg;
+	unsigned long irq_flags;
 	int rc = 0;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) {
 		/* we've hit the nx chip previously, now we're finalizing,
···
 
 	memcpy(out, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE);
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 

drivers/crypto/nx/nx-sha256.c

···
 	struct nx_sg *in_sg;
 	u64 to_process, leftover, total;
 	u32 max_sg_len;
+	unsigned long irq_flags;
 	int rc = 0;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	/* 2 cases for total data len:
 	 *  1: < SHA256_BLOCK_SIZE: copy into state, return 0
···
 		memcpy(sctx->buf, data, leftover);
 	sctx->count = leftover;
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
 	struct nx_sg *in_sg, *out_sg;
 	u32 max_sg_len;
+	unsigned long irq_flags;
 	int rc;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
 
···
 		     &(nx_ctx->stats->sha256_bytes));
 	memcpy(out, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE);
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
 	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
 	struct sha256_state *octx = out;
+	unsigned long irq_flags;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	octx->count = sctx->count +
 		      (csbcpb->cpb.sha256.message_bit_length / 8);
···
 		octx->state[7] = SHA256_H7;
 	}
 
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return 0;
 }
 
···
 	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
 	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
 	const struct sha256_state *ictx = in;
+	unsigned long irq_flags;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
 
···
 		NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
 	}
 
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return 0;
 }
 

drivers/crypto/nx/nx-sha512.c

···
 	struct nx_sg *in_sg;
 	u64 to_process, leftover, total, spbc_bits;
 	u32 max_sg_len;
+	unsigned long irq_flags;
 	int rc = 0;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	/* 2 cases for total data len:
 	 *  1: < SHA512_BLOCK_SIZE: copy into state, return 0
···
 		memcpy(sctx->buf, data, leftover);
 	sctx->count[0] = leftover;
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_sg *in_sg, *out_sg;
 	u32 max_sg_len;
 	u64 count0;
+	unsigned long irq_flags;
 	int rc;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	max_sg_len = min_t(u32, nx_driver.of.max_sg_len, nx_ctx->ap->sglen);
 
···
 
 	memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE);
 out:
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return rc;
 }
 
···
 	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
 	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
 	struct sha512_state *octx = out;
+	unsigned long irq_flags;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	/* move message_bit_length (128 bits) into count and convert its value
 	 * to bytes */
···
 		octx->state[7] = SHA512_H7;
 	}
 
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return 0;
 }
 
···
 	struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base);
 	struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb;
 	const struct sha512_state *ictx = in;
+	unsigned long irq_flags;
+
+	spin_lock_irqsave(&nx_ctx->lock, irq_flags);
 
 	memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf));
 	sctx->count[0] = ictx->count[0] & 0x3f;
···
 		NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE;
 	}
 
+	spin_unlock_irqrestore(&nx_ctx->lock, irq_flags);
 	return 0;
 }
 

drivers/crypto/nx/nx.c

···
 
 	do {
 		rc = vio_h_cop_sync(viodev, op);
-	} while ((rc == -EBUSY && !may_sleep && retries--) ||
-	         (rc == -EBUSY && may_sleep && cond_resched()));
+	} while (rc == -EBUSY && !may_sleep && retries--);
 
 	if (rc) {
 		dev_dbg(&viodev->dev, "vio_h_cop_sync failed: rc: %d "
···
  */
 void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function)
 {
+	spin_lock_init(&nx_ctx->lock);
 	memset(nx_ctx->kmem, 0, nx_ctx->kmem_len);
 	nx_ctx->csbcpb->csb.valid |= NX_CSB_VALID_BIT;
 

drivers/crypto/nx/nx.h

···
 };
 
 struct nx_crypto_ctx {
+	spinlock_t lock;	  /* synchronize access to the context */
 	void *kmem;		  /* unaligned, kmalloc'd buffer */
 	size_t kmem_len;	  /* length of kmem */
 	struct nx_csbcpb *csbcpb; /* aligned page given to phyp @ hcall time */