tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Shared descriptors for aead, ablkcipher algorithms
3 *
4 * Copyright 2016 NXP
5 */
6
7#include "compat.h"
8#include "desc_constr.h"
9#include "caamalg_desc.h"
10
11/*
12 * For aead functions, read payload and write payload,
13 * both of which are specified in req->src and req->dst
14 */
15static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
16{
17 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
18 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
19 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
20}
21
22/* Set DK bit in class 1 operation if shared */
23static inline void append_dec_op1(u32 *desc, u32 type)
24{
25 u32 *jump_cmd, *uncond_jump_cmd;
26
27 /* DK bit is valid only for AES */
28 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
29 append_operation(desc, type | OP_ALG_AS_INITFINAL |
30 OP_ALG_DECRYPT);
31 return;
32 }
33
34 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
35 append_operation(desc, type | OP_ALG_AS_INITFINAL |
36 OP_ALG_DECRYPT);
37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 set_jump_tgt_here(desc, jump_cmd);
39 append_operation(desc, type | OP_ALG_AS_INITFINAL |
40 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
41 set_jump_tgt_here(desc, uncond_jump_cmd);
42}
43
44/**
45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46 * (non-protocol) with no (null) encryption.
47 * @desc: pointer to buffer used for descriptor construction
48 * @adata: pointer to authentication transform definitions. Note that since a
49 * split key is to be used, the size of the split key itself is
50 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
51 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
52 * @icvsize: integrity check value (ICV) size (truncated or full)
53 *
54 * Note: Requires an MDHA split key.
55 */
56void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 unsigned int icvsize)
58{
59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61 init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63 /* Skip if already shared */
64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 JUMP_COND_SHRD);
66 if (adata->key_inline)
67 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
68 adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT |
69 KEY_ENC);
70 else
71 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
72 KEY_DEST_MDHA_SPLIT | KEY_ENC);
73 set_jump_tgt_here(desc, key_jump_cmd);
74
75 /* assoclen + cryptlen = seqinlen */
76 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
77
78 /* Prepare to read and write cryptlen + assoclen bytes */
79 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
80 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
81
82 /*
83 * MOVE_LEN opcode is not available in all SEC HW revisions,
84 * thus need to do some magic, i.e. self-patch the descriptor
85 * buffer.
86 */
87 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
88 MOVE_DEST_MATH3 |
89 (0x6 << MOVE_LEN_SHIFT));
90 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
91 MOVE_DEST_DESCBUF |
92 MOVE_WAITCOMP |
93 (0x8 << MOVE_LEN_SHIFT));
94
95 /* Class 2 operation */
96 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
97 OP_ALG_ENCRYPT);
98
99 /* Read and write cryptlen bytes */
100 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
101
102 set_move_tgt_here(desc, read_move_cmd);
103 set_move_tgt_here(desc, write_move_cmd);
104 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
105 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
106 MOVE_AUX_LS);
107
108 /* Write ICV */
109 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
110 LDST_SRCDST_BYTE_CONTEXT);
111
112#ifdef DEBUG
113 print_hex_dump(KERN_ERR,
114 "aead null enc shdesc@" __stringify(__LINE__)": ",
115 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
116#endif
117}
118EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
119
120/**
121 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
122 * (non-protocol) with no (null) decryption.
123 * @desc: pointer to buffer used for descriptor construction
124 * @adata: pointer to authentication transform definitions. Note that since a
125 * split key is to be used, the size of the split key itself is
126 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
127 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
128 * @icvsize: integrity check value (ICV) size (truncated or full)
129 *
130 * Note: Requires an MDHA split key.
131 */
132void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
133 unsigned int icvsize)
134{
135 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
136
137 init_sh_desc(desc, HDR_SHARE_SERIAL);
138
139 /* Skip if already shared */
140 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
141 JUMP_COND_SHRD);
142 if (adata->key_inline)
143 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
144 adata->keylen, CLASS_2 |
145 KEY_DEST_MDHA_SPLIT | KEY_ENC);
146 else
147 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
148 KEY_DEST_MDHA_SPLIT | KEY_ENC);
149 set_jump_tgt_here(desc, key_jump_cmd);
150
151 /* Class 2 operation */
152 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
153 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
154
155 /* assoclen + cryptlen = seqoutlen */
156 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
157
158 /* Prepare to read and write cryptlen + assoclen bytes */
159 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
160 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
161
162 /*
163 * MOVE_LEN opcode is not available in all SEC HW revisions,
164 * thus need to do some magic, i.e. self-patch the descriptor
165 * buffer.
166 */
167 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
168 MOVE_DEST_MATH2 |
169 (0x6 << MOVE_LEN_SHIFT));
170 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
171 MOVE_DEST_DESCBUF |
172 MOVE_WAITCOMP |
173 (0x8 << MOVE_LEN_SHIFT));
174
175 /* Read and write cryptlen bytes */
176 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
177
178 /*
179 * Insert a NOP here, since we need at least 4 instructions between
180 * code patching the descriptor buffer and the location being patched.
181 */
182 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
183 set_jump_tgt_here(desc, jump_cmd);
184
185 set_move_tgt_here(desc, read_move_cmd);
186 set_move_tgt_here(desc, write_move_cmd);
187 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
188 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
189 MOVE_AUX_LS);
190 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
191
192 /* Load ICV */
193 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
194 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
195
196#ifdef DEBUG
197 print_hex_dump(KERN_ERR,
198 "aead null dec shdesc@" __stringify(__LINE__)": ",
199 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
200#endif
201}
202EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
203
204static void init_sh_desc_key_aead(u32 * const desc,
205 struct alginfo * const cdata,
206 struct alginfo * const adata,
207 const bool is_rfc3686, u32 *nonce)
208{
209 u32 *key_jump_cmd;
210 unsigned int enckeylen = cdata->keylen;
211
212 /* Note: Context registers are saved. */
213 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
214
215 /* Skip if already shared */
216 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
217 JUMP_COND_SHRD);
218
219 /*
220 * RFC3686 specific:
221 * | key = {AUTH_KEY, ENC_KEY, NONCE}
222 * | enckeylen = encryption key size + nonce size
223 */
224 if (is_rfc3686)
225 enckeylen -= CTR_RFC3686_NONCE_SIZE;
226
227 if (adata->key_inline)
228 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
229 adata->keylen, CLASS_2 |
230 KEY_DEST_MDHA_SPLIT | KEY_ENC);
231 else
232 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
233 KEY_DEST_MDHA_SPLIT | KEY_ENC);
234
235 if (cdata->key_inline)
236 append_key_as_imm(desc, cdata->key_virt, enckeylen,
237 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
238 else
239 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
240 KEY_DEST_CLASS_REG);
241
242 /* Load Counter into CONTEXT1 reg */
243 if (is_rfc3686) {
244 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
245 LDST_CLASS_IND_CCB |
246 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
247 append_move(desc,
248 MOVE_SRC_OUTFIFO |
249 MOVE_DEST_CLASS1CTX |
250 (16 << MOVE_OFFSET_SHIFT) |
251 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
252 }
253
254 set_jump_tgt_here(desc, key_jump_cmd);
255}
256
257/**
258 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
259 * (non-protocol).
260 * @desc: pointer to buffer used for descriptor construction
261 * @cdata: pointer to block cipher transform definitions
262 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
263 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
264 * @adata: pointer to authentication transform definitions. Note that since a
265 * split key is to be used, the size of the split key itself is
266 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
267 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
268 * @ivsize: initialization vector size
269 * @icvsize: integrity check value (ICV) size (truncated or full)
270 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
271 * @nonce: pointer to rfc3686 nonce
272 * @ctx1_iv_off: IV offset in CONTEXT1 register
273 * @is_qi: true when called from caam/qi
274 *
275 * Note: Requires an MDHA split key.
276 */
277void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
278 struct alginfo *adata, unsigned int ivsize,
279 unsigned int icvsize, const bool is_rfc3686,
280 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi)
281{
282 /* Note: Context registers are saved. */
283 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
284
285 /* Class 2 operation */
286 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
287 OP_ALG_ENCRYPT);
288
289 if (is_qi) {
290 u32 *wait_load_cmd;
291
292 /* REG3 = assoclen */
293 append_seq_load(desc, 4, LDST_CLASS_DECO |
294 LDST_SRCDST_WORD_DECO_MATH3 |
295 (4 << LDST_OFFSET_SHIFT));
296
297 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
298 JUMP_COND_CALM | JUMP_COND_NCP |
299 JUMP_COND_NOP | JUMP_COND_NIP |
300 JUMP_COND_NIFP);
301 set_jump_tgt_here(desc, wait_load_cmd);
302
303 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
304 LDST_SRCDST_BYTE_CONTEXT |
305 (ctx1_iv_off << LDST_OFFSET_SHIFT));
306 }
307
308 /* Read and write assoclen bytes */
309 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
310 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
311
312 /* Skip assoc data */
313 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
314
315 /* read assoc before reading payload */
316 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
317 FIFOLDST_VLF);
318
319 /* Load Counter into CONTEXT1 reg */
320 if (is_rfc3686)
321 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
322 LDST_SRCDST_BYTE_CONTEXT |
323 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
324 LDST_OFFSET_SHIFT));
325
326 /* Class 1 operation */
327 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
328 OP_ALG_ENCRYPT);
329
330 /* Read and write cryptlen bytes */
331 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
332 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
334
335 /* Write ICV */
336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
337 LDST_SRCDST_BYTE_CONTEXT);
338
339#ifdef DEBUG
340 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
341 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
342#endif
343}
344EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
345
346/**
347 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
348 * (non-protocol).
349 * @desc: pointer to buffer used for descriptor construction
350 * @cdata: pointer to block cipher transform definitions
351 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
352 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
353 * @adata: pointer to authentication transform definitions. Note that since a
354 * split key is to be used, the size of the split key itself is
355 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
356 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
357 * @ivsize: initialization vector size
358 * @icvsize: integrity check value (ICV) size (truncated or full)
359 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
360 * @nonce: pointer to rfc3686 nonce
361 * @ctx1_iv_off: IV offset in CONTEXT1 register
362 * @is_qi: true when called from caam/qi
363 *
364 * Note: Requires an MDHA split key.
365 */
366void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
367 struct alginfo *adata, unsigned int ivsize,
368 unsigned int icvsize, const bool geniv,
369 const bool is_rfc3686, u32 *nonce,
370 const u32 ctx1_iv_off, const bool is_qi)
371{
372 /* Note: Context registers are saved. */
373 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
374
375 /* Class 2 operation */
376 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
377 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
378
379 if (is_qi) {
380 u32 *wait_load_cmd;
381
382 /* REG3 = assoclen */
383 append_seq_load(desc, 4, LDST_CLASS_DECO |
384 LDST_SRCDST_WORD_DECO_MATH3 |
385 (4 << LDST_OFFSET_SHIFT));
386
387 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
388 JUMP_COND_CALM | JUMP_COND_NCP |
389 JUMP_COND_NOP | JUMP_COND_NIP |
390 JUMP_COND_NIFP);
391 set_jump_tgt_here(desc, wait_load_cmd);
392
393 if (!geniv)
394 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
395 LDST_SRCDST_BYTE_CONTEXT |
396 (ctx1_iv_off << LDST_OFFSET_SHIFT));
397 }
398
399 /* Read and write assoclen bytes */
400 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
401 if (geniv)
402 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
403 else
404 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
405
406 /* Skip assoc data */
407 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
408
409 /* read assoc before reading payload */
410 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
411 KEY_VLF);
412
413 if (geniv) {
414 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
415 LDST_SRCDST_BYTE_CONTEXT |
416 (ctx1_iv_off << LDST_OFFSET_SHIFT));
417 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
418 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
419 }
420
421 /* Load Counter into CONTEXT1 reg */
422 if (is_rfc3686)
423 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
424 LDST_SRCDST_BYTE_CONTEXT |
425 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
426 LDST_OFFSET_SHIFT));
427
428 /* Choose operation */
429 if (ctx1_iv_off)
430 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
431 OP_ALG_DECRYPT);
432 else
433 append_dec_op1(desc, cdata->algtype);
434
435 /* Read and write cryptlen bytes */
436 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
437 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
438 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
439
440 /* Load ICV */
441 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
442 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
443
444#ifdef DEBUG
445 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
446 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
447#endif
448}
449EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
450
451/**
452 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
453 * (non-protocol) with HW-generated initialization
454 * vector.
455 * @desc: pointer to buffer used for descriptor construction
456 * @cdata: pointer to block cipher transform definitions
457 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
458 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
459 * @adata: pointer to authentication transform definitions. Note that since a
460 * split key is to be used, the size of the split key itself is
461 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
462 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
463 * @ivsize: initialization vector size
464 * @icvsize: integrity check value (ICV) size (truncated or full)
465 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
466 * @nonce: pointer to rfc3686 nonce
467 * @ctx1_iv_off: IV offset in CONTEXT1 register
468 * @is_qi: true when called from caam/qi
469 *
470 * Note: Requires an MDHA split key.
471 */
472void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
473 struct alginfo *adata, unsigned int ivsize,
474 unsigned int icvsize, const bool is_rfc3686,
475 u32 *nonce, const u32 ctx1_iv_off,
476 const bool is_qi)
477{
478 u32 geniv, moveiv;
479
480 /* Note: Context registers are saved. */
481 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
482
483 if (is_qi) {
484 u32 *wait_load_cmd;
485
486 /* REG3 = assoclen */
487 append_seq_load(desc, 4, LDST_CLASS_DECO |
488 LDST_SRCDST_WORD_DECO_MATH3 |
489 (4 << LDST_OFFSET_SHIFT));
490
491 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
492 JUMP_COND_CALM | JUMP_COND_NCP |
493 JUMP_COND_NOP | JUMP_COND_NIP |
494 JUMP_COND_NIFP);
495 set_jump_tgt_here(desc, wait_load_cmd);
496 }
497
498 if (is_rfc3686) {
499 if (is_qi)
500 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
501 LDST_SRCDST_BYTE_CONTEXT |
502 (ctx1_iv_off << LDST_OFFSET_SHIFT));
503
504 goto copy_iv;
505 }
506
507 /* Generate IV */
508 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
509 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
510 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
511 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
512 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
513 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
514 append_move(desc, MOVE_WAITCOMP |
515 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
516 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
517 (ivsize << MOVE_LEN_SHIFT));
518 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
519
520copy_iv:
521 /* Copy IV to class 1 context */
522 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
523 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
524 (ivsize << MOVE_LEN_SHIFT));
525
526 /* Return to encryption */
527 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
528 OP_ALG_ENCRYPT);
529
530 /* Read and write assoclen bytes */
531 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
532 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
533
534 /* Skip assoc data */
535 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
536
537 /* read assoc before reading payload */
538 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
539 KEY_VLF);
540
541 /* Copy iv from outfifo to class 2 fifo */
542 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
543 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
544 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
545 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
546 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
547 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
548
549 /* Load Counter into CONTEXT1 reg */
550 if (is_rfc3686)
551 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
552 LDST_SRCDST_BYTE_CONTEXT |
553 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
554 LDST_OFFSET_SHIFT));
555
556 /* Class 1 operation */
557 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
558 OP_ALG_ENCRYPT);
559
560 /* Will write ivsize + cryptlen */
561 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
562
563 /* Not need to reload iv */
564 append_seq_fifo_load(desc, ivsize,
565 FIFOLD_CLASS_SKIP);
566
567 /* Will read cryptlen */
568 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
569 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
570 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
571 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
572
573 /* Write ICV */
574 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
575 LDST_SRCDST_BYTE_CONTEXT);
576
577#ifdef DEBUG
578 print_hex_dump(KERN_ERR,
579 "aead givenc shdesc@" __stringify(__LINE__)": ",
580 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
581#endif
582}
583EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
584
585/**
586 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
587 * @desc: pointer to buffer used for descriptor construction
588 * @cdata: pointer to block cipher transform definitions
589 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
590 * @icvsize: integrity check value (ICV) size (truncated or full)
591 */
592void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
593 unsigned int icvsize)
594{
595 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
596 *zero_assoc_jump_cmd2;
597
598 init_sh_desc(desc, HDR_SHARE_SERIAL);
599
600 /* skip key loading if they are loaded due to sharing */
601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
602 JUMP_COND_SHRD | JUMP_COND_SELF);
603 if (cdata->key_inline)
604 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
605 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
606 else
607 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
608 KEY_DEST_CLASS_REG);
609 set_jump_tgt_here(desc, key_jump_cmd);
610
611 /* class 1 operation */
612 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
613 OP_ALG_ENCRYPT);
614
615 /* if assoclen + cryptlen is ZERO, skip to ICV write */
616 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
617 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
618 JUMP_COND_MATH_Z);
619
620 /* if assoclen is ZERO, skip reading the assoc data */
621 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
622 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
623 JUMP_COND_MATH_Z);
624
625 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
626
627 /* skip assoc data */
628 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
629
630 /* cryptlen = seqinlen - assoclen */
631 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
632
633 /* if cryptlen is ZERO jump to zero-payload commands */
634 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
635 JUMP_COND_MATH_Z);
636
637 /* read assoc data */
638 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
639 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
640 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
641
642 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
643
644 /* write encrypted data */
645 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
646
647 /* read payload data */
648 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
649 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
650
651 /* jump the zero-payload commands */
652 append_jump(desc, JUMP_TEST_ALL | 2);
653
654 /* zero-payload commands */
655 set_jump_tgt_here(desc, zero_payload_jump_cmd);
656
657 /* read assoc data */
658 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
659 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
660
661 /* There is no input data */
662 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
663
664 /* write ICV */
665 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
666 LDST_SRCDST_BYTE_CONTEXT);
667
668#ifdef DEBUG
669 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
670 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
671#endif
672}
673EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
674
675/**
676 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
677 * @desc: pointer to buffer used for descriptor construction
678 * @cdata: pointer to block cipher transform definitions
679 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
680 * @icvsize: integrity check value (ICV) size (truncated or full)
681 */
682void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
683 unsigned int icvsize)
684{
685 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
686
687 init_sh_desc(desc, HDR_SHARE_SERIAL);
688
689 /* skip key loading if they are loaded due to sharing */
690 key_jump_cmd = append_jump(desc, JUMP_JSL |
691 JUMP_TEST_ALL | JUMP_COND_SHRD |
692 JUMP_COND_SELF);
693 if (cdata->key_inline)
694 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
695 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
696 else
697 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
698 KEY_DEST_CLASS_REG);
699 set_jump_tgt_here(desc, key_jump_cmd);
700
701 /* class 1 operation */
702 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
703 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
704
705 /* if assoclen is ZERO, skip reading the assoc data */
706 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
707 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
708 JUMP_COND_MATH_Z);
709
710 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
711
712 /* skip assoc data */
713 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
714
715 /* read assoc data */
716 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
717 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
718
719 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
720
721 /* cryptlen = seqoutlen - assoclen */
722 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
723
724 /* jump to zero-payload command if cryptlen is zero */
725 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
726 JUMP_COND_MATH_Z);
727
728 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
729
730 /* store encrypted data */
731 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
732
733 /* read payload data */
734 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
735 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
736
737 /* zero-payload command */
738 set_jump_tgt_here(desc, zero_payload_jump_cmd);
739
740 /* read ICV */
741 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
742 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
743
744#ifdef DEBUG
745 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
746 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
747#endif
748}
749EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
750
751/**
752 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
753 * (non-protocol).
754 * @desc: pointer to buffer used for descriptor construction
755 * @cdata: pointer to block cipher transform definitions
756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757 * @icvsize: integrity check value (ICV) size (truncated or full)
758 */
759void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
760 unsigned int icvsize)
761{
762 u32 *key_jump_cmd;
763
764 init_sh_desc(desc, HDR_SHARE_SERIAL);
765
766 /* Skip key loading if it is loaded due to sharing */
767 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
768 JUMP_COND_SHRD);
769 if (cdata->key_inline)
770 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
771 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
772 else
773 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
774 KEY_DEST_CLASS_REG);
775 set_jump_tgt_here(desc, key_jump_cmd);
776
777 /* Class 1 operation */
778 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
779 OP_ALG_ENCRYPT);
780
781 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
782 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
783
784 /* Read assoc data */
785 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
786 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
787
788 /* Skip IV */
789 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
790
791 /* Will read cryptlen bytes */
792 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
793
794 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
795 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
796
797 /* Skip assoc data */
798 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
799
800 /* cryptlen = seqoutlen - assoclen */
801 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
802
803 /* Write encrypted data */
804 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
805
806 /* Read payload data */
807 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
808 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
809
810 /* Write ICV */
811 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
812 LDST_SRCDST_BYTE_CONTEXT);
813
814#ifdef DEBUG
815 print_hex_dump(KERN_ERR,
816 "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
817 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
818#endif
819}
820EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
821
822/**
823 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
824 * (non-protocol).
825 * @desc: pointer to buffer used for descriptor construction
826 * @cdata: pointer to block cipher transform definitions
827 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
828 * @icvsize: integrity check value (ICV) size (truncated or full)
829 */
830void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
831 unsigned int icvsize)
832{
833 u32 *key_jump_cmd;
834
835 init_sh_desc(desc, HDR_SHARE_SERIAL);
836
837 /* Skip key loading if it is loaded due to sharing */
838 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
839 JUMP_COND_SHRD);
840 if (cdata->key_inline)
841 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
842 cdata->keylen, CLASS_1 |
843 KEY_DEST_CLASS_REG);
844 else
845 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
846 KEY_DEST_CLASS_REG);
847 set_jump_tgt_here(desc, key_jump_cmd);
848
849 /* Class 1 operation */
850 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
851 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
852
853 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
854 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
855
856 /* Read assoc data */
857 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
858 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
859
860 /* Skip IV */
861 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
862
863 /* Will read cryptlen bytes */
864 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
865
866 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
867 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
868
869 /* Skip assoc data */
870 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
871
872 /* Will write cryptlen bytes */
873 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
874
875 /* Store payload data */
876 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
877
878 /* Read encrypted data */
879 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
880 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
881
882 /* Read ICV */
883 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
884 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
885
886#ifdef DEBUG
887 print_hex_dump(KERN_ERR,
888 "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
889 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
890#endif
891}
892EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
893
894/**
895 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
896 * (non-protocol).
897 * @desc: pointer to buffer used for descriptor construction
898 * @cdata: pointer to block cipher transform definitions
899 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
900 * @icvsize: integrity check value (ICV) size (truncated or full)
901 */
902void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
903 unsigned int icvsize)
904{
905 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
906
907 init_sh_desc(desc, HDR_SHARE_SERIAL);
908
909 /* Skip key loading if it is loaded due to sharing */
910 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
911 JUMP_COND_SHRD);
912 if (cdata->key_inline)
913 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
914 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
915 else
916 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
917 KEY_DEST_CLASS_REG);
918 set_jump_tgt_here(desc, key_jump_cmd);
919
920 /* Class 1 operation */
921 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
922 OP_ALG_ENCRYPT);
923
924 /* assoclen + cryptlen = seqinlen */
925 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
926
927 /*
928 * MOVE_LEN opcode is not available in all SEC HW revisions,
929 * thus need to do some magic, i.e. self-patch the descriptor
930 * buffer.
931 */
932 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
933 (0x6 << MOVE_LEN_SHIFT));
934 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
935 (0x8 << MOVE_LEN_SHIFT));
936
937 /* Will read assoclen + cryptlen bytes */
938 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
939
940 /* Will write assoclen + cryptlen bytes */
941 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
942
943 /* Read and write assoclen + cryptlen bytes */
944 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
945
946 set_move_tgt_here(desc, read_move_cmd);
947 set_move_tgt_here(desc, write_move_cmd);
948 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
949 /* Move payload data to OFIFO */
950 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
951
952 /* Write ICV */
953 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
954 LDST_SRCDST_BYTE_CONTEXT);
955
956#ifdef DEBUG
957 print_hex_dump(KERN_ERR,
958 "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
959 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
960#endif
961}
962EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
963
964/**
965 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
966 * (non-protocol).
967 * @desc: pointer to buffer used for descriptor construction
968 * @cdata: pointer to block cipher transform definitions
969 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
970 * @icvsize: integrity check value (ICV) size (truncated or full)
971 */
972void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
973 unsigned int icvsize)
974{
975 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
976
977 init_sh_desc(desc, HDR_SHARE_SERIAL);
978
979 /* Skip key loading if it is loaded due to sharing */
980 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
981 JUMP_COND_SHRD);
982 if (cdata->key_inline)
983 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
984 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
985 else
986 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
987 KEY_DEST_CLASS_REG);
988 set_jump_tgt_here(desc, key_jump_cmd);
989
990 /* Class 1 operation */
991 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
992 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
993
994 /* assoclen + cryptlen = seqoutlen */
995 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
996
997 /*
998 * MOVE_LEN opcode is not available in all SEC HW revisions,
999 * thus need to do some magic, i.e. self-patch the descriptor
1000 * buffer.
1001 */
1002 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1003 (0x6 << MOVE_LEN_SHIFT));
1004 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1005 (0x8 << MOVE_LEN_SHIFT));
1006
1007 /* Will read assoclen + cryptlen bytes */
1008 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1009
1010 /* Will write assoclen + cryptlen bytes */
1011 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1012
1013 /* Store payload data */
1014 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1015
1016 /* In-snoop assoclen + cryptlen data */
1017 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1018 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1019
1020 set_move_tgt_here(desc, read_move_cmd);
1021 set_move_tgt_here(desc, write_move_cmd);
1022 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1023 /* Move payload data to OFIFO */
1024 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1025 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1026
1027 /* Read ICV */
1028 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1029 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1030
1031#ifdef DEBUG
1032 print_hex_dump(KERN_ERR,
1033 "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1034 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1035#endif
1036}
1037EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1038
1039/*
1040 * For ablkcipher encrypt and decrypt, read from req->src and
1041 * write to req->dst
1042 */
1043static inline void ablkcipher_append_src_dst(u32 *desc)
1044{
1045 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1046 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1047 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1048 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1049 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1050}
1051
1052/**
1053 * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
1054 * @desc: pointer to buffer used for descriptor construction
1055 * @cdata: pointer to block cipher transform definitions
1056 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1057 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1058 * @ivsize: initialization vector size
1059 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1060 * @ctx1_iv_off: IV offset in CONTEXT1 register
1061 */
1062void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
1063 unsigned int ivsize, const bool is_rfc3686,
1064 const u32 ctx1_iv_off)
1065{
1066 u32 *key_jump_cmd;
1067
1068 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1069 /* Skip if already shared */
1070 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1071 JUMP_COND_SHRD);
1072
1073 /* Load class1 key only */
1074 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1075 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1076
1077 /* Load nonce into CONTEXT1 reg */
1078 if (is_rfc3686) {
1079 u8 *nonce = cdata->key_virt + cdata->keylen;
1080
1081 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1082 LDST_CLASS_IND_CCB |
1083 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1084 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1085 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1086 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1087 }
1088
1089 set_jump_tgt_here(desc, key_jump_cmd);
1090
1091 /* Load iv */
1092 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1093 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1094
1095 /* Load counter into CONTEXT1 reg */
1096 if (is_rfc3686)
1097 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1098 LDST_SRCDST_BYTE_CONTEXT |
1099 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1100 LDST_OFFSET_SHIFT));
1101
1102 /* Load operation */
1103 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1104 OP_ALG_ENCRYPT);
1105
1106 /* Perform operation */
1107 ablkcipher_append_src_dst(desc);
1108
1109#ifdef DEBUG
1110 print_hex_dump(KERN_ERR,
1111 "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
1112 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1113#endif
1114}
1115EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
1116
1117/**
1118 * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
1119 * @desc: pointer to buffer used for descriptor construction
1120 * @cdata: pointer to block cipher transform definitions
1121 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1122 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1123 * @ivsize: initialization vector size
1124 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1125 * @ctx1_iv_off: IV offset in CONTEXT1 register
1126 */
1127void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
1128 unsigned int ivsize, const bool is_rfc3686,
1129 const u32 ctx1_iv_off)
1130{
1131 u32 *key_jump_cmd;
1132
1133 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1134 /* Skip if already shared */
1135 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1136 JUMP_COND_SHRD);
1137
1138 /* Load class1 key only */
1139 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1140 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1141
1142 /* Load nonce into CONTEXT1 reg */
1143 if (is_rfc3686) {
1144 u8 *nonce = cdata->key_virt + cdata->keylen;
1145
1146 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1147 LDST_CLASS_IND_CCB |
1148 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1149 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1150 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1151 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1152 }
1153
1154 set_jump_tgt_here(desc, key_jump_cmd);
1155
1156 /* load IV */
1157 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1158 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1159
1160 /* Load counter into CONTEXT1 reg */
1161 if (is_rfc3686)
1162 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1163 LDST_SRCDST_BYTE_CONTEXT |
1164 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1165 LDST_OFFSET_SHIFT));
1166
1167 /* Choose operation */
1168 if (ctx1_iv_off)
1169 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1170 OP_ALG_DECRYPT);
1171 else
1172 append_dec_op1(desc, cdata->algtype);
1173
1174 /* Perform operation */
1175 ablkcipher_append_src_dst(desc);
1176
1177#ifdef DEBUG
1178 print_hex_dump(KERN_ERR,
1179 "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
1180 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1181#endif
1182}
1183EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
1184
1185/**
1186 * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
1187 * with HW-generated initialization vector.
1188 * @desc: pointer to buffer used for descriptor construction
1189 * @cdata: pointer to block cipher transform definitions
1190 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1191 * with OP_ALG_AAI_CBC.
1192 * @ivsize: initialization vector size
1193 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1194 * @ctx1_iv_off: IV offset in CONTEXT1 register
1195 */
1196void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
1197 unsigned int ivsize, const bool is_rfc3686,
1198 const u32 ctx1_iv_off)
1199{
1200 u32 *key_jump_cmd, geniv;
1201
1202 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1203 /* Skip if already shared */
1204 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1205 JUMP_COND_SHRD);
1206
1207 /* Load class1 key only */
1208 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1209 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1210
1211 /* Load Nonce into CONTEXT1 reg */
1212 if (is_rfc3686) {
1213 u8 *nonce = cdata->key_virt + cdata->keylen;
1214
1215 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1216 LDST_CLASS_IND_CCB |
1217 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1218 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1219 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1220 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1221 }
1222 set_jump_tgt_here(desc, key_jump_cmd);
1223
1224 /* Generate IV */
1225 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1226 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
1227 (ivsize << NFIFOENTRY_DLEN_SHIFT);
1228 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1229 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1230 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1231 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
1232 MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
1233 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1234 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1235
1236 /* Copy generated IV to memory */
1237 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1238 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1239
1240 /* Load Counter into CONTEXT1 reg */
1241 if (is_rfc3686)
1242 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1243 LDST_SRCDST_BYTE_CONTEXT |
1244 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1245 LDST_OFFSET_SHIFT));
1246
1247 if (ctx1_iv_off)
1248 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1249 (1 << JUMP_OFFSET_SHIFT));
1250
1251 /* Load operation */
1252 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1253 OP_ALG_ENCRYPT);
1254
1255 /* Perform operation */
1256 ablkcipher_append_src_dst(desc);
1257
1258#ifdef DEBUG
1259 print_hex_dump(KERN_ERR,
1260 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1261 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1262#endif
1263}
1264EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
1265
1266/**
1267 * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
1268 * descriptor
1269 * @desc: pointer to buffer used for descriptor construction
1270 * @cdata: pointer to block cipher transform definitions
1271 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1272 */
1273void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
1274{
1275 __be64 sector_size = cpu_to_be64(512);
1276 u32 *key_jump_cmd;
1277
1278 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1279 /* Skip if already shared */
1280 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1281 JUMP_COND_SHRD);
1282
1283 /* Load class1 keys only */
1284 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1285 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1286
1287 /* Load sector size with index 40 bytes (0x28) */
1288 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1289 LDST_SRCDST_BYTE_CONTEXT |
1290 (0x28 << LDST_OFFSET_SHIFT));
1291
1292 set_jump_tgt_here(desc, key_jump_cmd);
1293
1294 /*
1295 * create sequence for loading the sector index
1296 * Upper 8B of IV - will be used as sector index
1297 * Lower 8B of IV - will be discarded
1298 */
1299 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1300 (0x20 << LDST_OFFSET_SHIFT));
1301 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1302
1303 /* Load operation */
1304 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1305 OP_ALG_ENCRYPT);
1306
1307 /* Perform operation */
1308 ablkcipher_append_src_dst(desc);
1309
1310#ifdef DEBUG
1311 print_hex_dump(KERN_ERR,
1312 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1313 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1314#endif
1315}
1316EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
1317
1318/**
1319 * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
1320 * descriptor
1321 * @desc: pointer to buffer used for descriptor construction
1322 * @cdata: pointer to block cipher transform definitions
1323 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1324 */
1325void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
1326{
1327 __be64 sector_size = cpu_to_be64(512);
1328 u32 *key_jump_cmd;
1329
1330 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1331 /* Skip if already shared */
1332 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1333 JUMP_COND_SHRD);
1334
1335 /* Load class1 key only */
1336 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1337 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1338
1339 /* Load sector size with index 40 bytes (0x28) */
1340 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1341 LDST_SRCDST_BYTE_CONTEXT |
1342 (0x28 << LDST_OFFSET_SHIFT));
1343
1344 set_jump_tgt_here(desc, key_jump_cmd);
1345
1346 /*
1347 * create sequence for loading the sector index
1348 * Upper 8B of IV - will be used as sector index
1349 * Lower 8B of IV - will be discarded
1350 */
1351 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1352 (0x20 << LDST_OFFSET_SHIFT));
1353 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1354
1355 /* Load operation */
1356 append_dec_op1(desc, cdata->algtype);
1357
1358 /* Perform operation */
1359 ablkcipher_append_src_dst(desc);
1360
1361#ifdef DEBUG
1362 print_hex_dump(KERN_ERR,
1363 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1364 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1365#endif
1366}
1367EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
1368
1369MODULE_LICENSE("GPL");
1370MODULE_DESCRIPTION("FSL CAAM descriptor support");
1371MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");