crypto: nx - Use API partial block handling

+40 -88

drivers/crypto/nx/nx-aes-xcbc.c

··· 7 7 * Author: Kent Yoder <yoder1@us.ibm.com> 8 8 */ 9 9 10 - #include <crypto/internal/hash.h> 11 10 #include <crypto/aes.h> 12 - #include <crypto/algapi.h> 11 + #include <crypto/internal/hash.h> 12 + #include <linux/atomic.h> 13 + #include <linux/errno.h> 14 + #include <linux/kernel.h> 13 15 #include <linux/module.h> 14 - #include <linux/types.h> 15 - #include <linux/crypto.h> 16 - #include <asm/vio.h> 16 + #include <linux/spinlock.h> 17 + #include <linux/string.h> 17 18 18 19 #include "nx_csbcpb.h" 19 20 #include "nx.h" ··· 22 21 23 22 struct xcbc_state { 24 23 u8 state[AES_BLOCK_SIZE]; 25 - unsigned int count; 26 - u8 buffer[AES_BLOCK_SIZE]; 27 24 }; 28 25 29 26 static int nx_xcbc_set_key(struct crypto_shash *desc, ··· 57 58 */ 58 59 static int nx_xcbc_empty(struct shash_desc *desc, u8 *out) 59 60 { 60 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 61 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 61 62 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 62 63 struct nx_sg *in_sg, *out_sg; 63 64 u8 keys[2][AES_BLOCK_SIZE]; ··· 134 135 return rc; 135 136 } 136 137 137 - static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_tfm *tfm) 138 + static int nx_crypto_ctx_aes_xcbc_init2(struct crypto_shash *tfm) 138 139 { 139 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); 140 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); 140 141 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 141 142 int err; 142 143 ··· 165 166 const u8 *data, 166 167 unsigned int len) 167 168 { 169 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 168 170 struct xcbc_state *sctx = shash_desc_ctx(desc); 169 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 170 171 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 171 172 struct nx_sg *in_sg; 172 173 struct nx_sg *out_sg; 173 - u32 to_process = 0, leftover, total; 174 174 unsigned int max_sg_len; 175 175 unsigned long irq_flags; 176 + u32 to_process, total; 176 177 int rc = 0; 177 178 int data_len; 178 179 179 180 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 180 181 182 + memcpy(csbcpb->cpb.aes_xcbc.out_cv_mac, sctx->state, AES_BLOCK_SIZE); 183 + NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 184 + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 181 185 182 - total = sctx->count + len; 183 - 184 - /* 2 cases for total data len: 185 - * 1: <= AES_BLOCK_SIZE: copy into state, return 0 186 - * 2: > AES_BLOCK_SIZE: process X blocks, copy in leftover 187 - */ 188 - if (total <= AES_BLOCK_SIZE) { 189 - memcpy(sctx->buffer + sctx->count, data, len); 190 - sctx->count += len; 191 - goto out; 192 - } 186 + total = len; 193 187 194 188 in_sg = nx_ctx->in_sg; 195 189 max_sg_len = min_t(u64, nx_driver.of.max_sg_len/sizeof(struct nx_sg), ··· 192 200 193 201 data_len = AES_BLOCK_SIZE; 194 202 out_sg = nx_build_sg_list(nx_ctx->out_sg, (u8 *)sctx->state, 195 - &len, nx_ctx->ap->sglen); 203 + &data_len, nx_ctx->ap->sglen); 196 204 197 205 if (data_len != AES_BLOCK_SIZE) { 198 206 rc = -EINVAL; ··· 202 210 nx_ctx->op.outlen = (nx_ctx->out_sg - out_sg) * sizeof(struct nx_sg); 203 211 204 212 do { 205 - to_process = total - to_process; 206 - to_process = to_process & ~(AES_BLOCK_SIZE - 1); 213 + to_process = total & ~(AES_BLOCK_SIZE - 1); 207 214 208 - leftover = total - to_process; 209 - 210 - /* the hardware will not accept a 0 byte operation for this 211 - * algorithm and the operation MUST be finalized to be correct. 212 - * So if we happen to get an update that falls on a block sized 213 - * boundary, we must save off the last block to finalize with 214 - * later. */ 215 - if (!leftover) { 216 - to_process -= AES_BLOCK_SIZE; 217 - leftover = AES_BLOCK_SIZE; 218 - } 219 - 220 - if (sctx->count) { 221 - data_len = sctx->count; 222 - in_sg = nx_build_sg_list(nx_ctx->in_sg, 223 - (u8 *) sctx->buffer, 224 - &data_len, 225 - max_sg_len); 226 - if (data_len != sctx->count) { 227 - rc = -EINVAL; 228 - goto out; 229 - } 230 - } 231 - 232 - data_len = to_process - sctx->count; 233 215 in_sg = nx_build_sg_list(in_sg, 234 216 (u8 *) data, 235 - &data_len, 217 + &to_process, 236 218 max_sg_len); 237 - 238 - if (data_len != to_process - sctx->count) { 239 - rc = -EINVAL; 240 - goto out; 241 - } 242 219 243 220 nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * 244 221 sizeof(struct nx_sg); 245 222 246 223 /* we've hit the nx chip previously and we're updating again, 247 224 * so copy over the partial digest */ 248 - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { 249 - memcpy(csbcpb->cpb.aes_xcbc.cv, 250 - csbcpb->cpb.aes_xcbc.out_cv_mac, 251 - AES_BLOCK_SIZE); 252 - } 225 + memcpy(csbcpb->cpb.aes_xcbc.cv, 226 + csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); 253 227 254 - NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; 255 228 if (!nx_ctx->op.inlen || !nx_ctx->op.outlen) { 256 229 rc = -EINVAL; 257 230 goto out; ··· 228 271 229 272 atomic_inc(&(nx_ctx->stats->aes_ops)); 230 273 231 - /* everything after the first update is continuation */ 232 - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 233 - 234 274 total -= to_process; 235 - data += to_process - sctx->count; 236 - sctx->count = 0; 275 + data += to_process; 237 276 in_sg = nx_ctx->in_sg; 238 - } while (leftover > AES_BLOCK_SIZE); 277 + } while (total >= AES_BLOCK_SIZE); 239 278 240 - /* copy the leftover back into the state struct */ 241 - memcpy(sctx->buffer, data, leftover); 242 - sctx->count = leftover; 279 + rc = total; 280 + memcpy(sctx->state, csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); 243 281 244 282 out: 245 283 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 246 284 return rc; 247 285 } 248 286 249 - static int nx_xcbc_final(struct shash_desc *desc, u8 *out) 287 + static int nx_xcbc_finup(struct shash_desc *desc, const u8 *src, 288 + unsigned int nbytes, u8 *out) 250 289 { 290 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 251 291 struct xcbc_state *sctx = shash_desc_ctx(desc); 252 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 253 292 struct nx_csbcpb *csbcpb = nx_ctx->csbcpb; 254 293 struct nx_sg *in_sg, *out_sg; 255 294 unsigned long irq_flags; ··· 254 301 255 302 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 256 303 257 - if (NX_CPB_FDM(csbcpb) & NX_FDM_CONTINUATION) { 258 - /* we've hit the nx chip previously, now we're finalizing, 259 - * so copy over the partial digest */ 260 - memcpy(csbcpb->cpb.aes_xcbc.cv, 261 - csbcpb->cpb.aes_xcbc.out_cv_mac, AES_BLOCK_SIZE); 262 - } else if (sctx->count == 0) { 304 + if (nbytes) { 305 + /* non-zero final, so copy over the partial digest */ 306 + memcpy(csbcpb->cpb.aes_xcbc.cv, sctx->state, AES_BLOCK_SIZE); 307 + } else { 263 308 /* 264 309 * we've never seen an update, so this is a 0 byte op. The 265 310 * hardware cannot handle a 0 byte op, so just ECB to ··· 271 320 * this is not an intermediate operation */ 272 321 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 273 322 274 - len = sctx->count; 275 - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)sctx->buffer, 276 - &len, nx_ctx->ap->sglen); 323 + len = nbytes; 324 + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, 325 + nx_ctx->ap->sglen); 277 326 278 - if (len != sctx->count) { 327 + if (len != nbytes) { 279 328 rc = -EINVAL; 280 329 goto out; 281 330 } ··· 313 362 .digestsize = AES_BLOCK_SIZE, 314 363 .init = nx_xcbc_init, 315 364 .update = nx_xcbc_update, 316 - .final = nx_xcbc_final, 365 + .finup = nx_xcbc_finup, 317 366 .setkey = nx_xcbc_set_key, 318 367 .descsize = sizeof(struct xcbc_state), 319 - .statesize = sizeof(struct xcbc_state), 368 + .init_tfm = nx_crypto_ctx_aes_xcbc_init2, 369 + .exit_tfm = nx_crypto_ctx_shash_exit, 320 370 .base = { 321 371 .cra_name = "xcbc(aes)", 322 372 .cra_driver_name = "xcbc-aes-nx", 323 373 .cra_priority = 300, 374 + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | 375 + CRYPTO_AHASH_ALG_FINAL_NONZERO, 324 376 .cra_blocksize = AES_BLOCK_SIZE, 325 377 .cra_module = THIS_MODULE, 326 378 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 327 - .cra_init = nx_crypto_ctx_aes_xcbc_init2, 328 - .cra_exit = nx_crypto_ctx_exit, 329 379 } 330 380 };

+51 -79

drivers/crypto/nx/nx-sha256.c

··· 9 9 10 10 #include <crypto/internal/hash.h> 11 11 #include <crypto/sha2.h> 12 + #include <linux/errno.h> 13 + #include <linux/kernel.h> 12 14 #include <linux/module.h> 13 - #include <asm/vio.h> 14 - #include <asm/byteorder.h> 15 + #include <linux/spinlock.h> 16 + #include <linux/string.h> 17 + #include <linux/unaligned.h> 15 18 16 19 #include "nx_csbcpb.h" 17 20 #include "nx.h" ··· 22 19 struct sha256_state_be { 23 20 __be32 state[SHA256_DIGEST_SIZE / 4]; 24 21 u64 count; 25 - u8 buf[SHA256_BLOCK_SIZE]; 26 22 }; 27 23 28 - static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) 24 + static int nx_crypto_ctx_sha256_init(struct crypto_shash *tfm) 29 25 { 30 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); 26 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); 31 27 int err; 32 28 33 29 err = nx_crypto_ctx_sha_init(tfm); ··· 42 40 return 0; 43 41 } 44 42 45 - static int nx_sha256_init(struct shash_desc *desc) { 43 + static int nx_sha256_init(struct shash_desc *desc) 44 + { 46 45 struct sha256_state_be *sctx = shash_desc_ctx(desc); 47 - 48 - memset(sctx, 0, sizeof *sctx); 49 46 50 47 sctx->state[0] = __cpu_to_be32(SHA256_H0); 51 48 sctx->state[1] = __cpu_to_be32(SHA256_H1); ··· 62 61 static int nx_sha256_update(struct shash_desc *desc, const u8 *data, 63 62 unsigned int len) 64 63 { 64 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 65 65 struct sha256_state_be *sctx = shash_desc_ctx(desc); 66 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 67 66 struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; 67 + u64 to_process, leftover, total = len; 68 68 struct nx_sg *out_sg; 69 - u64 to_process = 0, leftover, total; 70 69 unsigned long irq_flags; 71 70 int rc = 0; 72 71 int data_len; 73 72 u32 max_sg_len; 74 - u64 buf_len = (sctx->count % SHA256_BLOCK_SIZE); 75 73 76 74 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 77 - 78 - /* 2 cases for total data len: 79 - * 1: < SHA256_BLOCK_SIZE: copy into state, return 0 80 - * 2: >= SHA256_BLOCK_SIZE: process X blocks, copy in leftover 81 - */ 82 - total = (sctx->count % SHA256_BLOCK_SIZE) + len; 83 - if (total < SHA256_BLOCK_SIZE) { 84 - memcpy(sctx->buf + buf_len, data, len); 85 - sctx->count += len; 86 - goto out; 87 - } 88 75 89 76 memcpy(csbcpb->cpb.sha256.message_digest, sctx->state, SHA256_DIGEST_SIZE); 90 77 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; ··· 94 105 } 95 106 96 107 do { 97 - int used_sgs = 0; 98 108 struct nx_sg *in_sg = nx_ctx->in_sg; 99 109 100 - if (buf_len) { 101 - data_len = buf_len; 102 - in_sg = nx_build_sg_list(in_sg, 103 - (u8 *) sctx->buf, 104 - &data_len, 105 - max_sg_len); 110 + to_process = total & ~(SHA256_BLOCK_SIZE - 1); 106 111 107 - if (data_len != buf_len) { 108 - rc = -EINVAL; 109 - goto out; 110 - } 111 - used_sgs = in_sg - nx_ctx->in_sg; 112 - } 113 - 114 - /* to_process: SHA256_BLOCK_SIZE aligned chunk to be 115 - * processed in this iteration. This value is restricted 116 - * by sg list limits and number of sgs we already used 117 - * for leftover data. (see above) 118 - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, 119 - * but because data may not be aligned, we need to account 120 - * for that too. */ 121 - to_process = min_t(u64, total, 122 - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); 123 - to_process = to_process & ~(SHA256_BLOCK_SIZE - 1); 124 - 125 - data_len = to_process - buf_len; 112 + data_len = to_process; 126 113 in_sg = nx_build_sg_list(in_sg, (u8 *) data, 127 114 &data_len, max_sg_len); 128 115 129 116 nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); 130 117 131 - to_process = data_len + buf_len; 118 + to_process = data_len; 132 119 leftover = total - to_process; 133 120 134 121 /* ··· 127 162 atomic_inc(&(nx_ctx->stats->sha256_ops)); 128 163 129 164 total -= to_process; 130 - data += to_process - buf_len; 131 - buf_len = 0; 132 - 165 + data += to_process; 166 + sctx->count += to_process; 133 167 } while (leftover >= SHA256_BLOCK_SIZE); 134 168 135 - /* copy the leftover back into the state struct */ 136 - if (leftover) 137 - memcpy(sctx->buf, data, leftover); 138 - 139 - sctx->count += len; 169 + rc = leftover; 140 170 memcpy(sctx->state, csbcpb->cpb.sha256.message_digest, SHA256_DIGEST_SIZE); 141 171 out: 142 172 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 143 173 return rc; 144 174 } 145 175 146 - static int nx_sha256_final(struct shash_desc *desc, u8 *out) 176 + static int nx_sha256_finup(struct shash_desc *desc, const u8 *src, 177 + unsigned int nbytes, u8 *out) 147 178 { 179 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 148 180 struct sha256_state_be *sctx = shash_desc_ctx(desc); 149 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 150 181 struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; 151 182 struct nx_sg *in_sg, *out_sg; 152 183 unsigned long irq_flags; ··· 158 197 nx_ctx->ap->databytelen/NX_PAGE_SIZE); 159 198 160 199 /* final is represented by continuing the operation and indicating that 161 - * this is not an intermediate operation */ 162 - if (sctx->count >= SHA256_BLOCK_SIZE) { 163 - /* we've hit the nx chip previously, now we're finalizing, 164 - * so copy over the partial digest */ 165 - memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); 166 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 167 - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 168 - } else { 169 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 170 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; 171 - } 200 + * this is not an intermediate operation 201 + * copy over the partial digest */ 202 + memcpy(csbcpb->cpb.sha256.input_partial_digest, sctx->state, SHA256_DIGEST_SIZE); 203 + NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 204 + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 172 205 206 + sctx->count += nbytes; 173 207 csbcpb->cpb.sha256.message_bit_length = (u64) (sctx->count * 8); 174 208 175 - len = sctx->count & (SHA256_BLOCK_SIZE - 1); 176 - in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *) sctx->buf, 177 - &len, max_sg_len); 209 + len = nbytes; 210 + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); 178 211 179 - if (len != (sctx->count & (SHA256_BLOCK_SIZE - 1))) { 212 + if (len != nbytes) { 180 213 rc = -EINVAL; 181 214 goto out; 182 215 } ··· 206 251 static int nx_sha256_export(struct shash_desc *desc, void *out) 207 252 { 208 253 struct sha256_state_be *sctx = shash_desc_ctx(desc); 254 + union { 255 + u8 *u8; 256 + u32 *u32; 257 + u64 *u64; 258 + } p = { .u8 = out }; 259 + int i; 209 260 210 - memcpy(out, sctx, sizeof(*sctx)); 261 + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) 262 + put_unaligned(be32_to_cpu(sctx->state[i]), p.u32++); 211 263 264 + put_unaligned(sctx->count, p.u64++); 212 265 return 0; 213 266 } 214 267 215 268 static int nx_sha256_import(struct shash_desc *desc, const void *in) 216 269 { 217 270 struct sha256_state_be *sctx = shash_desc_ctx(desc); 271 + union { 272 + const u8 *u8; 273 + const u32 *u32; 274 + const u64 *u64; 275 + } p = { .u8 = in }; 276 + int i; 218 277 219 - memcpy(sctx, in, sizeof(*sctx)); 278 + for (i = 0; i < SHA256_DIGEST_SIZE / sizeof(*p.u32); i++) 279 + sctx->state[i] = cpu_to_be32(get_unaligned(p.u32++)); 220 280 281 + sctx->count = get_unaligned(p.u64++); 221 282 return 0; 222 283 } 223 284 ··· 241 270 .digestsize = SHA256_DIGEST_SIZE, 242 271 .init = nx_sha256_init, 243 272 .update = nx_sha256_update, 244 - .final = nx_sha256_final, 273 + .finup = nx_sha256_finup, 245 274 .export = nx_sha256_export, 246 275 .import = nx_sha256_import, 276 + .init_tfm = nx_crypto_ctx_sha256_init, 277 + .exit_tfm = nx_crypto_ctx_shash_exit, 247 278 .descsize = sizeof(struct sha256_state_be), 248 279 .statesize = sizeof(struct sha256_state_be), 249 280 .base = { 250 281 .cra_name = "sha256", 251 282 .cra_driver_name = "sha256-nx", 252 283 .cra_priority = 300, 284 + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 253 285 .cra_blocksize = SHA256_BLOCK_SIZE, 254 286 .cra_module = THIS_MODULE, 255 287 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 256 - .cra_init = nx_crypto_ctx_sha256_init, 257 - .cra_exit = nx_crypto_ctx_exit, 258 288 } 259 289 };

+57 -86

drivers/crypto/nx/nx-sha512.c

··· 9 9 10 10 #include <crypto/internal/hash.h> 11 11 #include <crypto/sha2.h> 12 + #include <linux/errno.h> 13 + #include <linux/kernel.h> 12 14 #include <linux/module.h> 13 - #include <asm/vio.h> 15 + #include <linux/spinlock.h> 16 + #include <linux/string.h> 17 + #include <linux/unaligned.h> 14 18 15 19 #include "nx_csbcpb.h" 16 20 #include "nx.h" ··· 22 18 struct sha512_state_be { 23 19 __be64 state[SHA512_DIGEST_SIZE / 8]; 24 20 u64 count[2]; 25 - u8 buf[SHA512_BLOCK_SIZE]; 26 21 }; 27 22 28 - static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) 23 + static int nx_crypto_ctx_sha512_init(struct crypto_shash *tfm) 29 24 { 30 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(tfm); 25 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(tfm); 31 26 int err; 32 27 33 28 err = nx_crypto_ctx_sha_init(tfm); ··· 46 43 { 47 44 struct sha512_state_be *sctx = shash_desc_ctx(desc); 48 45 49 - memset(sctx, 0, sizeof *sctx); 50 - 51 46 sctx->state[0] = __cpu_to_be64(SHA512_H0); 52 47 sctx->state[1] = __cpu_to_be64(SHA512_H1); 53 48 sctx->state[2] = __cpu_to_be64(SHA512_H2); ··· 55 54 sctx->state[6] = __cpu_to_be64(SHA512_H6); 56 55 sctx->state[7] = __cpu_to_be64(SHA512_H7); 57 56 sctx->count[0] = 0; 57 + sctx->count[1] = 0; 58 58 59 59 return 0; 60 60 } ··· 63 61 static int nx_sha512_update(struct shash_desc *desc, const u8 *data, 64 62 unsigned int len) 65 63 { 64 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 66 65 struct sha512_state_be *sctx = shash_desc_ctx(desc); 67 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 68 66 struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; 67 + u64 to_process, leftover, total = len; 69 68 struct nx_sg *out_sg; 70 - u64 to_process, leftover = 0, total; 71 69 unsigned long irq_flags; 72 70 int rc = 0; 73 71 int data_len; 74 72 u32 max_sg_len; 75 - u64 buf_len = (sctx->count[0] % SHA512_BLOCK_SIZE); 76 73 77 74 spin_lock_irqsave(&nx_ctx->lock, irq_flags); 78 - 79 - /* 2 cases for total data len: 80 - * 1: < SHA512_BLOCK_SIZE: copy into state, return 0 81 - * 2: >= SHA512_BLOCK_SIZE: process X blocks, copy in leftover 82 - */ 83 - total = (sctx->count[0] % SHA512_BLOCK_SIZE) + len; 84 - if (total < SHA512_BLOCK_SIZE) { 85 - memcpy(sctx->buf + buf_len, data, len); 86 - sctx->count[0] += len; 87 - goto out; 88 - } 89 75 90 76 memcpy(csbcpb->cpb.sha512.message_digest, sctx->state, SHA512_DIGEST_SIZE); 91 77 NX_CPB_FDM(csbcpb) |= NX_FDM_INTERMEDIATE; ··· 95 105 } 96 106 97 107 do { 98 - int used_sgs = 0; 99 108 struct nx_sg *in_sg = nx_ctx->in_sg; 100 109 101 - if (buf_len) { 102 - data_len = buf_len; 103 - in_sg = nx_build_sg_list(in_sg, 104 - (u8 *) sctx->buf, 105 - &data_len, max_sg_len); 110 + to_process = total & ~(SHA512_BLOCK_SIZE - 1); 106 111 107 - if (data_len != buf_len) { 108 - rc = -EINVAL; 109 - goto out; 110 - } 111 - used_sgs = in_sg - nx_ctx->in_sg; 112 - } 113 - 114 - /* to_process: SHA512_BLOCK_SIZE aligned chunk to be 115 - * processed in this iteration. This value is restricted 116 - * by sg list limits and number of sgs we already used 117 - * for leftover data. (see above) 118 - * In ideal case, we could allow NX_PAGE_SIZE * max_sg_len, 119 - * but because data may not be aligned, we need to account 120 - * for that too. */ 121 - to_process = min_t(u64, total, 122 - (max_sg_len - 1 - used_sgs) * NX_PAGE_SIZE); 123 - to_process = to_process & ~(SHA512_BLOCK_SIZE - 1); 124 - 125 - data_len = to_process - buf_len; 112 + data_len = to_process; 126 113 in_sg = nx_build_sg_list(in_sg, (u8 *) data, 127 114 &data_len, max_sg_len); 128 115 129 116 nx_ctx->op.inlen = (nx_ctx->in_sg - in_sg) * sizeof(struct nx_sg); 130 117 131 - if (data_len != (to_process - buf_len)) { 132 - rc = -EINVAL; 133 - goto out; 134 - } 135 - 136 - to_process = data_len + buf_len; 118 + to_process = data_len; 137 119 leftover = total - to_process; 138 120 139 121 /* ··· 128 166 atomic_inc(&(nx_ctx->stats->sha512_ops)); 129 167 130 168 total -= to_process; 131 - data += to_process - buf_len; 132 - buf_len = 0; 133 - 169 + data += to_process; 170 + sctx->count[0] += to_process; 171 + if (sctx->count[0] < to_process) 172 + sctx->count[1]++; 134 173 } while (leftover >= SHA512_BLOCK_SIZE); 135 174 136 - /* copy the leftover back into the state struct */ 137 - if (leftover) 138 - memcpy(sctx->buf, data, leftover); 139 - sctx->count[0] += len; 175 + rc = leftover; 140 176 memcpy(sctx->state, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); 141 177 out: 142 178 spin_unlock_irqrestore(&nx_ctx->lock, irq_flags); 143 179 return rc; 144 180 } 145 181 146 - static int nx_sha512_final(struct shash_desc *desc, u8 *out) 182 + static int nx_sha512_finup(struct shash_desc *desc, const u8 *src, 183 + unsigned int nbytes, u8 *out) 147 184 { 148 185 struct sha512_state_be *sctx = shash_desc_ctx(desc); 149 - struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); 186 + struct nx_crypto_ctx *nx_ctx = crypto_shash_ctx(desc->tfm); 150 187 struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; 151 188 struct nx_sg *in_sg, *out_sg; 152 189 u32 max_sg_len; 153 - u64 count0; 154 190 unsigned long irq_flags; 191 + u64 count0, count1; 155 192 int rc = 0; 156 193 int len; 157 194 ··· 162 201 nx_ctx->ap->databytelen/NX_PAGE_SIZE); 163 202 164 203 /* final is represented by continuing the operation and indicating that 165 - * this is not an intermediate operation */ 166 - if (sctx->count[0] >= SHA512_BLOCK_SIZE) { 167 - /* we've hit the nx chip previously, now we're finalizing, 168 - * so copy over the partial digest */ 169 - memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, 170 - SHA512_DIGEST_SIZE); 171 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 172 - NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 173 - } else { 174 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 175 - NX_CPB_FDM(csbcpb) &= ~NX_FDM_CONTINUATION; 176 - } 177 - 204 + * this is not an intermediate operation 205 + * copy over the partial digest */ 206 + memcpy(csbcpb->cpb.sha512.input_partial_digest, sctx->state, SHA512_DIGEST_SIZE); 178 207 NX_CPB_FDM(csbcpb) &= ~NX_FDM_INTERMEDIATE; 208 + NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; 179 209 180 - count0 = sctx->count[0] * 8; 210 + count0 = sctx->count[0] + nbytes; 211 + count1 = sctx->count[1]; 181 212 182 - csbcpb->cpb.sha512.message_bit_length_lo = count0; 213 + csbcpb->cpb.sha512.message_bit_length_lo = count0 << 3; 214 + csbcpb->cpb.sha512.message_bit_length_hi = (count1 << 3) | 215 + (count0 >> 61); 183 216 184 - len = sctx->count[0] & (SHA512_BLOCK_SIZE - 1); 185 - in_sg = nx_build_sg_list(nx_ctx->in_sg, sctx->buf, &len, 186 - max_sg_len); 217 + len = nbytes; 218 + in_sg = nx_build_sg_list(nx_ctx->in_sg, (u8 *)src, &len, max_sg_len); 187 219 188 - if (len != (sctx->count[0] & (SHA512_BLOCK_SIZE - 1))) { 220 + if (len != nbytes) { 189 221 rc = -EINVAL; 190 222 goto out; 191 223 } ··· 200 246 goto out; 201 247 202 248 atomic_inc(&(nx_ctx->stats->sha512_ops)); 203 - atomic64_add(sctx->count[0], &(nx_ctx->stats->sha512_bytes)); 249 + atomic64_add(count0, &(nx_ctx->stats->sha512_bytes)); 204 250 205 251 memcpy(out, csbcpb->cpb.sha512.message_digest, SHA512_DIGEST_SIZE); 206 252 out: ··· 211 257 static int nx_sha512_export(struct shash_desc *desc, void *out) 212 258 { 213 259 struct sha512_state_be *sctx = shash_desc_ctx(desc); 260 + union { 261 + u8 *u8; 262 + u64 *u64; 263 + } p = { .u8 = out }; 264 + int i; 214 265 215 - memcpy(out, sctx, sizeof(*sctx)); 266 + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) 267 + put_unaligned(be64_to_cpu(sctx->state[i]), p.u64++); 216 268 269 + put_unaligned(sctx->count[0], p.u64++); 270 + put_unaligned(sctx->count[1], p.u64++); 217 271 return 0; 218 272 } 219 273 220 274 static int nx_sha512_import(struct shash_desc *desc, const void *in) 221 275 { 222 276 struct sha512_state_be *sctx = shash_desc_ctx(desc); 277 + union { 278 + const u8 *u8; 279 + const u64 *u64; 280 + } p = { .u8 = in }; 281 + int i; 223 282 224 - memcpy(sctx, in, sizeof(*sctx)); 283 + for (i = 0; i < SHA512_DIGEST_SIZE / sizeof(*p.u64); i++) 284 + sctx->state[i] = cpu_to_be64(get_unaligned(p.u64++)); 225 285 286 + sctx->count[0] = get_unaligned(p.u64++); 287 + sctx->count[1] = get_unaligned(p.u64++); 226 288 return 0; 227 289 } 228 290 ··· 246 276 .digestsize = SHA512_DIGEST_SIZE, 247 277 .init = nx_sha512_init, 248 278 .update = nx_sha512_update, 249 - .final = nx_sha512_final, 279 + .finup = nx_sha512_finup, 250 280 .export = nx_sha512_export, 251 281 .import = nx_sha512_import, 282 + .init_tfm = nx_crypto_ctx_sha512_init, 283 + .exit_tfm = nx_crypto_ctx_shash_exit, 252 284 .descsize = sizeof(struct sha512_state_be), 253 285 .statesize = sizeof(struct sha512_state_be), 254 286 .base = { 255 287 .cra_name = "sha512", 256 288 .cra_driver_name = "sha512-nx", 257 289 .cra_priority = 300, 290 + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, 258 291 .cra_blocksize = SHA512_BLOCK_SIZE, 259 292 .cra_module = THIS_MODULE, 260 293 .cra_ctxsize = sizeof(struct nx_crypto_ctx), 261 - .cra_init = nx_crypto_ctx_sha512_init, 262 - .cra_exit = nx_crypto_ctx_exit, 263 294 } 264 295 };

+9 -6

drivers/crypto/nx/nx.c

··· 124 124 } 125 125 126 126 if ((sg - sg_head) == sgmax) { 127 - pr_err("nx: scatter/gather list overflow, pid: %d\n", 128 - current->pid); 129 127 sg++; 130 128 break; 131 129 } ··· 700 702 NX_MODE_AES_ECB); 701 703 } 702 704 703 - int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm) 705 + int nx_crypto_ctx_sha_init(struct crypto_shash *tfm) 704 706 { 705 - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); 707 + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_SHA, NX_MODE_SHA); 706 708 } 707 709 708 - int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm) 710 + int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm) 709 711 { 710 - return nx_crypto_ctx_init(crypto_tfm_ctx(tfm), NX_FC_AES, 712 + return nx_crypto_ctx_init(crypto_shash_ctx(tfm), NX_FC_AES, 711 713 NX_MODE_AES_XCBC_MAC); 712 714 } 713 715 ··· 740 742 struct nx_crypto_ctx *nx_ctx = crypto_aead_ctx(tfm); 741 743 742 744 kfree_sensitive(nx_ctx->kmem); 745 + } 746 + 747 + void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm) 748 + { 749 + nx_crypto_ctx_exit(crypto_shash_ctx(tfm)); 743 750 } 744 751 745 752 static int nx_probe(struct vio_dev *viodev, const struct vio_device_id *id)

+4 -2

drivers/crypto/nx/nx.h

··· 3 3 #ifndef __NX_H__ 4 4 #define __NX_H__ 5 5 6 + #include <asm/vio.h> 6 7 #include <crypto/ctr.h> 7 8 #include <crypto/internal/aead.h> 8 9 #include <crypto/internal/hash.h> ··· 148 147 /* prototypes */ 149 148 int nx_crypto_ctx_aes_ccm_init(struct crypto_aead *tfm); 150 149 int nx_crypto_ctx_aes_gcm_init(struct crypto_aead *tfm); 151 - int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm *tfm); 150 + int nx_crypto_ctx_aes_xcbc_init(struct crypto_shash *tfm); 152 151 int nx_crypto_ctx_aes_ctr_init(struct crypto_skcipher *tfm); 153 152 int nx_crypto_ctx_aes_cbc_init(struct crypto_skcipher *tfm); 154 153 int nx_crypto_ctx_aes_ecb_init(struct crypto_skcipher *tfm); 155 - int nx_crypto_ctx_sha_init(struct crypto_tfm *tfm); 154 + int nx_crypto_ctx_sha_init(struct crypto_shash *tfm); 156 155 void nx_crypto_ctx_exit(struct crypto_tfm *tfm); 157 156 void nx_crypto_ctx_skcipher_exit(struct crypto_skcipher *tfm); 158 157 void nx_crypto_ctx_aead_exit(struct crypto_aead *tfm); 158 + void nx_crypto_ctx_shash_exit(struct crypto_shash *tfm); 159 159 void nx_ctx_init(struct nx_crypto_ctx *nx_ctx, unsigned int function); 160 160 int nx_hcall_sync(struct nx_crypto_ctx *ctx, struct vio_pfo_op *op, 161 161 u32 may_sleep);