tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 *
5 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
6 *
7 * Scatterlist Crypto API glue code copied from files with the following:
8 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9 *
10 * Crypto algorithm registration code copied from hifn driver:
11 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
12 * All rights reserved.
13 */
14
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/mod_devicetable.h>
18#include <linux/device.h>
19#include <linux/interrupt.h>
20#include <linux/crypto.h>
21#include <linux/hw_random.h>
22#include <linux/of_address.h>
23#include <linux/of_irq.h>
24#include <linux/of_platform.h>
25#include <linux/dma-mapping.h>
26#include <linux/io.h>
27#include <linux/spinlock.h>
28#include <linux/rtnetlink.h>
29#include <linux/slab.h>
30
31#include <crypto/algapi.h>
32#include <crypto/aes.h>
33#include <crypto/des.h>
34#include <crypto/sha.h>
35#include <crypto/md5.h>
36#include <crypto/internal/aead.h>
37#include <crypto/authenc.h>
38#include <crypto/skcipher.h>
39#include <crypto/hash.h>
40#include <crypto/internal/hash.h>
41#include <crypto/scatterwalk.h>
42
43#include "talitos.h"
44
45static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
46 unsigned int len, bool is_sec1)
47{
48 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
49 if (is_sec1) {
50 ptr->len1 = cpu_to_be16(len);
51 } else {
52 ptr->len = cpu_to_be16(len);
53 ptr->eptr = upper_32_bits(dma_addr);
54 }
55}
56
57static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
58 struct talitos_ptr *src_ptr, bool is_sec1)
59{
60 dst_ptr->ptr = src_ptr->ptr;
61 if (is_sec1) {
62 dst_ptr->len1 = src_ptr->len1;
63 } else {
64 dst_ptr->len = src_ptr->len;
65 dst_ptr->eptr = src_ptr->eptr;
66 }
67}
68
69static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
70 bool is_sec1)
71{
72 if (is_sec1)
73 return be16_to_cpu(ptr->len1);
74 else
75 return be16_to_cpu(ptr->len);
76}
77
78static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
79 bool is_sec1)
80{
81 if (!is_sec1)
82 ptr->j_extent = val;
83}
84
85static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
86{
87 if (!is_sec1)
88 ptr->j_extent |= val;
89}
90
91/*
92 * map virtual single (contiguous) pointer to h/w descriptor pointer
93 */
94static void __map_single_talitos_ptr(struct device *dev,
95 struct talitos_ptr *ptr,
96 unsigned int len, void *data,
97 enum dma_data_direction dir,
98 unsigned long attrs)
99{
100 dma_addr_t dma_addr = dma_map_single_attrs(dev, data, len, dir, attrs);
101 struct talitos_private *priv = dev_get_drvdata(dev);
102 bool is_sec1 = has_ftr_sec1(priv);
103
104 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
105}
106
107static void map_single_talitos_ptr(struct device *dev,
108 struct talitos_ptr *ptr,
109 unsigned int len, void *data,
110 enum dma_data_direction dir)
111{
112 __map_single_talitos_ptr(dev, ptr, len, data, dir, 0);
113}
114
115static void map_single_talitos_ptr_nosync(struct device *dev,
116 struct talitos_ptr *ptr,
117 unsigned int len, void *data,
118 enum dma_data_direction dir)
119{
120 __map_single_talitos_ptr(dev, ptr, len, data, dir,
121 DMA_ATTR_SKIP_CPU_SYNC);
122}
123
124/*
125 * unmap bus single (contiguous) h/w descriptor pointer
126 */
127static void unmap_single_talitos_ptr(struct device *dev,
128 struct talitos_ptr *ptr,
129 enum dma_data_direction dir)
130{
131 struct talitos_private *priv = dev_get_drvdata(dev);
132 bool is_sec1 = has_ftr_sec1(priv);
133
134 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
135 from_talitos_ptr_len(ptr, is_sec1), dir);
136}
137
138static int reset_channel(struct device *dev, int ch)
139{
140 struct talitos_private *priv = dev_get_drvdata(dev);
141 unsigned int timeout = TALITOS_TIMEOUT;
142 bool is_sec1 = has_ftr_sec1(priv);
143
144 if (is_sec1) {
145 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
146 TALITOS1_CCCR_LO_RESET);
147
148 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
149 TALITOS1_CCCR_LO_RESET) && --timeout)
150 cpu_relax();
151 } else {
152 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
153 TALITOS2_CCCR_RESET);
154
155 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
156 TALITOS2_CCCR_RESET) && --timeout)
157 cpu_relax();
158 }
159
160 if (timeout == 0) {
161 dev_err(dev, "failed to reset channel %d\n", ch);
162 return -EIO;
163 }
164
165 /* set 36-bit addressing, done writeback enable and done IRQ enable */
166 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
167 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
168 /* enable chaining descriptors */
169 if (is_sec1)
170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
171 TALITOS_CCCR_LO_NE);
172
173 /* and ICCR writeback, if available */
174 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
175 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
176 TALITOS_CCCR_LO_IWSE);
177
178 return 0;
179}
180
181static int reset_device(struct device *dev)
182{
183 struct talitos_private *priv = dev_get_drvdata(dev);
184 unsigned int timeout = TALITOS_TIMEOUT;
185 bool is_sec1 = has_ftr_sec1(priv);
186 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
187
188 setbits32(priv->reg + TALITOS_MCR, mcr);
189
190 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
191 && --timeout)
192 cpu_relax();
193
194 if (priv->irq[1]) {
195 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
196 setbits32(priv->reg + TALITOS_MCR, mcr);
197 }
198
199 if (timeout == 0) {
200 dev_err(dev, "failed to reset device\n");
201 return -EIO;
202 }
203
204 return 0;
205}
206
207/*
208 * Reset and initialize the device
209 */
210static int init_device(struct device *dev)
211{
212 struct talitos_private *priv = dev_get_drvdata(dev);
213 int ch, err;
214 bool is_sec1 = has_ftr_sec1(priv);
215
216 /*
217 * Master reset
218 * errata documentation: warning: certain SEC interrupts
219 * are not fully cleared by writing the MCR:SWR bit,
220 * set bit twice to completely reset
221 */
222 err = reset_device(dev);
223 if (err)
224 return err;
225
226 err = reset_device(dev);
227 if (err)
228 return err;
229
230 /* reset channels */
231 for (ch = 0; ch < priv->num_channels; ch++) {
232 err = reset_channel(dev, ch);
233 if (err)
234 return err;
235 }
236
237 /* enable channel done and error interrupts */
238 if (is_sec1) {
239 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
240 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
241 /* disable parity error check in DEU (erroneous? test vect.) */
242 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
243 } else {
244 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
245 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
246 }
247
248 /* disable integrity check error interrupts (use writeback instead) */
249 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
250 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
251 TALITOS_MDEUICR_LO_ICE);
252
253 return 0;
254}
255
256/**
257 * talitos_submit - submits a descriptor to the device for processing
258 * @dev: the SEC device to be used
259 * @ch: the SEC device channel to be used
260 * @desc: the descriptor to be processed by the device
261 * @callback: whom to call when processing is complete
262 * @context: a handle for use by caller (optional)
263 *
264 * desc must contain valid dma-mapped (bus physical) address pointers.
265 * callback must check err and feedback in descriptor header
266 * for device processing status.
267 */
268int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
269 void (*callback)(struct device *dev,
270 struct talitos_desc *desc,
271 void *context, int error),
272 void *context)
273{
274 struct talitos_private *priv = dev_get_drvdata(dev);
275 struct talitos_request *request;
276 unsigned long flags;
277 int head;
278 bool is_sec1 = has_ftr_sec1(priv);
279
280 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
281
282 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
283 /* h/w fifo is full */
284 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
285 return -EAGAIN;
286 }
287
288 head = priv->chan[ch].head;
289 request = &priv->chan[ch].fifo[head];
290
291 /* map descriptor and save caller data */
292 if (is_sec1) {
293 desc->hdr1 = desc->hdr;
294 request->dma_desc = dma_map_single(dev, &desc->hdr1,
295 TALITOS_DESC_SIZE,
296 DMA_BIDIRECTIONAL);
297 } else {
298 request->dma_desc = dma_map_single(dev, desc,
299 TALITOS_DESC_SIZE,
300 DMA_BIDIRECTIONAL);
301 }
302 request->callback = callback;
303 request->context = context;
304
305 /* increment fifo head */
306 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
307
308 smp_wmb();
309 request->desc = desc;
310
311 /* GO! */
312 wmb();
313 out_be32(priv->chan[ch].reg + TALITOS_FF,
314 upper_32_bits(request->dma_desc));
315 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
316 lower_32_bits(request->dma_desc));
317
318 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
319
320 return -EINPROGRESS;
321}
322EXPORT_SYMBOL(talitos_submit);
323
324/*
325 * process what was done, notify callback of error if not
326 */
327static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
328{
329 struct talitos_private *priv = dev_get_drvdata(dev);
330 struct talitos_request *request, saved_req;
331 unsigned long flags;
332 int tail, status;
333 bool is_sec1 = has_ftr_sec1(priv);
334
335 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
336
337 tail = priv->chan[ch].tail;
338 while (priv->chan[ch].fifo[tail].desc) {
339 __be32 hdr;
340
341 request = &priv->chan[ch].fifo[tail];
342
343 /* descriptors with their done bits set don't get the error */
344 rmb();
345 if (!is_sec1)
346 hdr = request->desc->hdr;
347 else if (request->desc->next_desc)
348 hdr = (request->desc + 1)->hdr1;
349 else
350 hdr = request->desc->hdr1;
351
352 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
353 status = 0;
354 else
355 if (!error)
356 break;
357 else
358 status = error;
359
360 dma_unmap_single(dev, request->dma_desc,
361 TALITOS_DESC_SIZE,
362 DMA_BIDIRECTIONAL);
363
364 /* copy entries so we can call callback outside lock */
365 saved_req.desc = request->desc;
366 saved_req.callback = request->callback;
367 saved_req.context = request->context;
368
369 /* release request entry in fifo */
370 smp_wmb();
371 request->desc = NULL;
372
373 /* increment fifo tail */
374 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
375
376 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
377
378 atomic_dec(&priv->chan[ch].submit_count);
379
380 saved_req.callback(dev, saved_req.desc, saved_req.context,
381 status);
382 /* channel may resume processing in single desc error case */
383 if (error && !reset_ch && status == error)
384 return;
385 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
386 tail = priv->chan[ch].tail;
387 }
388
389 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
390}
391
392/*
393 * process completed requests for channels that have done status
394 */
395#define DEF_TALITOS1_DONE(name, ch_done_mask) \
396static void talitos1_done_##name(unsigned long data) \
397{ \
398 struct device *dev = (struct device *)data; \
399 struct talitos_private *priv = dev_get_drvdata(dev); \
400 unsigned long flags; \
401 \
402 if (ch_done_mask & 0x10000000) \
403 flush_channel(dev, 0, 0, 0); \
404 if (ch_done_mask & 0x40000000) \
405 flush_channel(dev, 1, 0, 0); \
406 if (ch_done_mask & 0x00010000) \
407 flush_channel(dev, 2, 0, 0); \
408 if (ch_done_mask & 0x00040000) \
409 flush_channel(dev, 3, 0, 0); \
410 \
411 /* At this point, all completed channels have been processed */ \
412 /* Unmask done interrupts for channels completed later on. */ \
413 spin_lock_irqsave(&priv->reg_lock, flags); \
414 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
415 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
416 spin_unlock_irqrestore(&priv->reg_lock, flags); \
417}
418
419DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
420DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
421
422#define DEF_TALITOS2_DONE(name, ch_done_mask) \
423static void talitos2_done_##name(unsigned long data) \
424{ \
425 struct device *dev = (struct device *)data; \
426 struct talitos_private *priv = dev_get_drvdata(dev); \
427 unsigned long flags; \
428 \
429 if (ch_done_mask & 1) \
430 flush_channel(dev, 0, 0, 0); \
431 if (ch_done_mask & (1 << 2)) \
432 flush_channel(dev, 1, 0, 0); \
433 if (ch_done_mask & (1 << 4)) \
434 flush_channel(dev, 2, 0, 0); \
435 if (ch_done_mask & (1 << 6)) \
436 flush_channel(dev, 3, 0, 0); \
437 \
438 /* At this point, all completed channels have been processed */ \
439 /* Unmask done interrupts for channels completed later on. */ \
440 spin_lock_irqsave(&priv->reg_lock, flags); \
441 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
442 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
443 spin_unlock_irqrestore(&priv->reg_lock, flags); \
444}
445
446DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
447DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
448DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
449DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
450
451/*
452 * locate current (offending) descriptor
453 */
454static u32 current_desc_hdr(struct device *dev, int ch)
455{
456 struct talitos_private *priv = dev_get_drvdata(dev);
457 int tail, iter;
458 dma_addr_t cur_desc;
459
460 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
461 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
462
463 if (!cur_desc) {
464 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
465 return 0;
466 }
467
468 tail = priv->chan[ch].tail;
469
470 iter = tail;
471 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
472 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) {
473 iter = (iter + 1) & (priv->fifo_len - 1);
474 if (iter == tail) {
475 dev_err(dev, "couldn't locate current descriptor\n");
476 return 0;
477 }
478 }
479
480 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
481 return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
482
483 return priv->chan[ch].fifo[iter].desc->hdr;
484}
485
486/*
487 * user diagnostics; report root cause of error based on execution unit status
488 */
489static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
490{
491 struct talitos_private *priv = dev_get_drvdata(dev);
492 int i;
493
494 if (!desc_hdr)
495 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
496
497 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
498 case DESC_HDR_SEL0_AFEU:
499 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
500 in_be32(priv->reg_afeu + TALITOS_EUISR),
501 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
502 break;
503 case DESC_HDR_SEL0_DEU:
504 dev_err(dev, "DEUISR 0x%08x_%08x\n",
505 in_be32(priv->reg_deu + TALITOS_EUISR),
506 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
507 break;
508 case DESC_HDR_SEL0_MDEUA:
509 case DESC_HDR_SEL0_MDEUB:
510 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
511 in_be32(priv->reg_mdeu + TALITOS_EUISR),
512 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
513 break;
514 case DESC_HDR_SEL0_RNG:
515 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
516 in_be32(priv->reg_rngu + TALITOS_ISR),
517 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
518 break;
519 case DESC_HDR_SEL0_PKEU:
520 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
521 in_be32(priv->reg_pkeu + TALITOS_EUISR),
522 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
523 break;
524 case DESC_HDR_SEL0_AESU:
525 dev_err(dev, "AESUISR 0x%08x_%08x\n",
526 in_be32(priv->reg_aesu + TALITOS_EUISR),
527 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
528 break;
529 case DESC_HDR_SEL0_CRCU:
530 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
531 in_be32(priv->reg_crcu + TALITOS_EUISR),
532 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
533 break;
534 case DESC_HDR_SEL0_KEU:
535 dev_err(dev, "KEUISR 0x%08x_%08x\n",
536 in_be32(priv->reg_pkeu + TALITOS_EUISR),
537 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
538 break;
539 }
540
541 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
542 case DESC_HDR_SEL1_MDEUA:
543 case DESC_HDR_SEL1_MDEUB:
544 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
545 in_be32(priv->reg_mdeu + TALITOS_EUISR),
546 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
547 break;
548 case DESC_HDR_SEL1_CRCU:
549 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
550 in_be32(priv->reg_crcu + TALITOS_EUISR),
551 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
552 break;
553 }
554
555 for (i = 0; i < 8; i++)
556 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
557 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
558 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
559}
560
561/*
562 * recover from error interrupts
563 */
564static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
565{
566 struct talitos_private *priv = dev_get_drvdata(dev);
567 unsigned int timeout = TALITOS_TIMEOUT;
568 int ch, error, reset_dev = 0;
569 u32 v_lo;
570 bool is_sec1 = has_ftr_sec1(priv);
571 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
572
573 for (ch = 0; ch < priv->num_channels; ch++) {
574 /* skip channels without errors */
575 if (is_sec1) {
576 /* bits 29, 31, 17, 19 */
577 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
578 continue;
579 } else {
580 if (!(isr & (1 << (ch * 2 + 1))))
581 continue;
582 }
583
584 error = -EINVAL;
585
586 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
587
588 if (v_lo & TALITOS_CCPSR_LO_DOF) {
589 dev_err(dev, "double fetch fifo overflow error\n");
590 error = -EAGAIN;
591 reset_ch = 1;
592 }
593 if (v_lo & TALITOS_CCPSR_LO_SOF) {
594 /* h/w dropped descriptor */
595 dev_err(dev, "single fetch fifo overflow error\n");
596 error = -EAGAIN;
597 }
598 if (v_lo & TALITOS_CCPSR_LO_MDTE)
599 dev_err(dev, "master data transfer error\n");
600 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
601 dev_err(dev, is_sec1 ? "pointer not complete error\n"
602 : "s/g data length zero error\n");
603 if (v_lo & TALITOS_CCPSR_LO_FPZ)
604 dev_err(dev, is_sec1 ? "parity error\n"
605 : "fetch pointer zero error\n");
606 if (v_lo & TALITOS_CCPSR_LO_IDH)
607 dev_err(dev, "illegal descriptor header error\n");
608 if (v_lo & TALITOS_CCPSR_LO_IEU)
609 dev_err(dev, is_sec1 ? "static assignment error\n"
610 : "invalid exec unit error\n");
611 if (v_lo & TALITOS_CCPSR_LO_EU)
612 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
613 if (!is_sec1) {
614 if (v_lo & TALITOS_CCPSR_LO_GB)
615 dev_err(dev, "gather boundary error\n");
616 if (v_lo & TALITOS_CCPSR_LO_GRL)
617 dev_err(dev, "gather return/length error\n");
618 if (v_lo & TALITOS_CCPSR_LO_SB)
619 dev_err(dev, "scatter boundary error\n");
620 if (v_lo & TALITOS_CCPSR_LO_SRL)
621 dev_err(dev, "scatter return/length error\n");
622 }
623
624 flush_channel(dev, ch, error, reset_ch);
625
626 if (reset_ch) {
627 reset_channel(dev, ch);
628 } else {
629 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
630 TALITOS2_CCCR_CONT);
631 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
632 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
633 TALITOS2_CCCR_CONT) && --timeout)
634 cpu_relax();
635 if (timeout == 0) {
636 dev_err(dev, "failed to restart channel %d\n",
637 ch);
638 reset_dev = 1;
639 }
640 }
641 }
642 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
643 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
644 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
645 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
646 isr, isr_lo);
647 else
648 dev_err(dev, "done overflow, internal time out, or "
649 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
650
651 /* purge request queues */
652 for (ch = 0; ch < priv->num_channels; ch++)
653 flush_channel(dev, ch, -EIO, 1);
654
655 /* reset and reinitialize the device */
656 init_device(dev);
657 }
658}
659
660#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
661static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
662{ \
663 struct device *dev = data; \
664 struct talitos_private *priv = dev_get_drvdata(dev); \
665 u32 isr, isr_lo; \
666 unsigned long flags; \
667 \
668 spin_lock_irqsave(&priv->reg_lock, flags); \
669 isr = in_be32(priv->reg + TALITOS_ISR); \
670 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
671 /* Acknowledge interrupt */ \
672 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
673 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
674 \
675 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
676 spin_unlock_irqrestore(&priv->reg_lock, flags); \
677 talitos_error(dev, isr & ch_err_mask, isr_lo); \
678 } \
679 else { \
680 if (likely(isr & ch_done_mask)) { \
681 /* mask further done interrupts. */ \
682 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
683 /* done_task will unmask done interrupts at exit */ \
684 tasklet_schedule(&priv->done_task[tlet]); \
685 } \
686 spin_unlock_irqrestore(&priv->reg_lock, flags); \
687 } \
688 \
689 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
690 IRQ_NONE; \
691}
692
693DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
694
695#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
696static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
697{ \
698 struct device *dev = data; \
699 struct talitos_private *priv = dev_get_drvdata(dev); \
700 u32 isr, isr_lo; \
701 unsigned long flags; \
702 \
703 spin_lock_irqsave(&priv->reg_lock, flags); \
704 isr = in_be32(priv->reg + TALITOS_ISR); \
705 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
706 /* Acknowledge interrupt */ \
707 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
708 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
709 \
710 if (unlikely(isr & ch_err_mask || isr_lo)) { \
711 spin_unlock_irqrestore(&priv->reg_lock, flags); \
712 talitos_error(dev, isr & ch_err_mask, isr_lo); \
713 } \
714 else { \
715 if (likely(isr & ch_done_mask)) { \
716 /* mask further done interrupts. */ \
717 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
718 /* done_task will unmask done interrupts at exit */ \
719 tasklet_schedule(&priv->done_task[tlet]); \
720 } \
721 spin_unlock_irqrestore(&priv->reg_lock, flags); \
722 } \
723 \
724 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
725 IRQ_NONE; \
726}
727
728DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
729DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
730 0)
731DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
732 1)
733
734/*
735 * hwrng
736 */
737static int talitos_rng_data_present(struct hwrng *rng, int wait)
738{
739 struct device *dev = (struct device *)rng->priv;
740 struct talitos_private *priv = dev_get_drvdata(dev);
741 u32 ofl;
742 int i;
743
744 for (i = 0; i < 20; i++) {
745 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
746 TALITOS_RNGUSR_LO_OFL;
747 if (ofl || !wait)
748 break;
749 udelay(10);
750 }
751
752 return !!ofl;
753}
754
755static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
756{
757 struct device *dev = (struct device *)rng->priv;
758 struct talitos_private *priv = dev_get_drvdata(dev);
759
760 /* rng fifo requires 64-bit accesses */
761 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
762 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
763
764 return sizeof(u32);
765}
766
767static int talitos_rng_init(struct hwrng *rng)
768{
769 struct device *dev = (struct device *)rng->priv;
770 struct talitos_private *priv = dev_get_drvdata(dev);
771 unsigned int timeout = TALITOS_TIMEOUT;
772
773 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
774 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
775 & TALITOS_RNGUSR_LO_RD)
776 && --timeout)
777 cpu_relax();
778 if (timeout == 0) {
779 dev_err(dev, "failed to reset rng hw\n");
780 return -ENODEV;
781 }
782
783 /* start generating */
784 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
785
786 return 0;
787}
788
789static int talitos_register_rng(struct device *dev)
790{
791 struct talitos_private *priv = dev_get_drvdata(dev);
792 int err;
793
794 priv->rng.name = dev_driver_string(dev),
795 priv->rng.init = talitos_rng_init,
796 priv->rng.data_present = talitos_rng_data_present,
797 priv->rng.data_read = talitos_rng_data_read,
798 priv->rng.priv = (unsigned long)dev;
799
800 err = hwrng_register(&priv->rng);
801 if (!err)
802 priv->rng_registered = true;
803
804 return err;
805}
806
807static void talitos_unregister_rng(struct device *dev)
808{
809 struct talitos_private *priv = dev_get_drvdata(dev);
810
811 if (!priv->rng_registered)
812 return;
813
814 hwrng_unregister(&priv->rng);
815 priv->rng_registered = false;
816}
817
818/*
819 * crypto alg
820 */
821#define TALITOS_CRA_PRIORITY 3000
822/*
823 * Defines a priority for doing AEAD with descriptors type
824 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
825 */
826#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
827#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
828#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
829
830struct talitos_ctx {
831 struct device *dev;
832 int ch;
833 __be32 desc_hdr_template;
834 u8 key[TALITOS_MAX_KEY_SIZE];
835 u8 iv[TALITOS_MAX_IV_LENGTH];
836 dma_addr_t dma_key;
837 unsigned int keylen;
838 unsigned int enckeylen;
839 unsigned int authkeylen;
840};
841
842#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
843#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
844
845struct talitos_ahash_req_ctx {
846 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
847 unsigned int hw_context_size;
848 u8 buf[2][HASH_MAX_BLOCK_SIZE];
849 int buf_idx;
850 unsigned int swinit;
851 unsigned int first;
852 unsigned int last;
853 unsigned int to_hash_later;
854 unsigned int nbuf;
855 struct scatterlist bufsl[2];
856 struct scatterlist *psrc;
857};
858
859struct talitos_export_state {
860 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
861 u8 buf[HASH_MAX_BLOCK_SIZE];
862 unsigned int swinit;
863 unsigned int first;
864 unsigned int last;
865 unsigned int to_hash_later;
866 unsigned int nbuf;
867};
868
869static int aead_setkey(struct crypto_aead *authenc,
870 const u8 *key, unsigned int keylen)
871{
872 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
873 struct device *dev = ctx->dev;
874 struct crypto_authenc_keys keys;
875
876 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
877 goto badkey;
878
879 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
880 goto badkey;
881
882 if (ctx->keylen)
883 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
884
885 memcpy(ctx->key, keys.authkey, keys.authkeylen);
886 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
887
888 ctx->keylen = keys.authkeylen + keys.enckeylen;
889 ctx->enckeylen = keys.enckeylen;
890 ctx->authkeylen = keys.authkeylen;
891 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
892 DMA_TO_DEVICE);
893
894 memzero_explicit(&keys, sizeof(keys));
895 return 0;
896
897badkey:
898 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
899 memzero_explicit(&keys, sizeof(keys));
900 return -EINVAL;
901}
902
903static int aead_des3_setkey(struct crypto_aead *authenc,
904 const u8 *key, unsigned int keylen)
905{
906 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
907 struct device *dev = ctx->dev;
908 struct crypto_authenc_keys keys;
909 u32 flags;
910 int err;
911
912 err = crypto_authenc_extractkeys(&keys, key, keylen);
913 if (unlikely(err))
914 goto badkey;
915
916 err = -EINVAL;
917 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
918 goto badkey;
919
920 if (keys.enckeylen != DES3_EDE_KEY_SIZE)
921 goto badkey;
922
923 flags = crypto_aead_get_flags(authenc);
924 err = __des3_verify_key(&flags, keys.enckey);
925 if (unlikely(err)) {
926 crypto_aead_set_flags(authenc, flags);
927 goto out;
928 }
929
930 if (ctx->keylen)
931 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
932
933 memcpy(ctx->key, keys.authkey, keys.authkeylen);
934 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
935
936 ctx->keylen = keys.authkeylen + keys.enckeylen;
937 ctx->enckeylen = keys.enckeylen;
938 ctx->authkeylen = keys.authkeylen;
939 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
940 DMA_TO_DEVICE);
941
942out:
943 memzero_explicit(&keys, sizeof(keys));
944 return err;
945
946badkey:
947 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
948 goto out;
949}
950
951/*
952 * talitos_edesc - s/w-extended descriptor
953 * @src_nents: number of segments in input scatterlist
954 * @dst_nents: number of segments in output scatterlist
955 * @icv_ool: whether ICV is out-of-line
956 * @iv_dma: dma address of iv for checking continuity and link table
957 * @dma_len: length of dma mapped link_tbl space
958 * @dma_link_tbl: bus physical address of link_tbl/buf
959 * @desc: h/w descriptor
960 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
961 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
962 *
963 * if decrypting (with authcheck), or either one of src_nents or dst_nents
964 * is greater than 1, an integrity check value is concatenated to the end
965 * of link_tbl data
966 */
967struct talitos_edesc {
968 int src_nents;
969 int dst_nents;
970 bool icv_ool;
971 dma_addr_t iv_dma;
972 int dma_len;
973 dma_addr_t dma_link_tbl;
974 struct talitos_desc desc;
975 union {
976 struct talitos_ptr link_tbl[0];
977 u8 buf[0];
978 };
979};
980
981static void talitos_sg_unmap(struct device *dev,
982 struct talitos_edesc *edesc,
983 struct scatterlist *src,
984 struct scatterlist *dst,
985 unsigned int len, unsigned int offset)
986{
987 struct talitos_private *priv = dev_get_drvdata(dev);
988 bool is_sec1 = has_ftr_sec1(priv);
989 unsigned int src_nents = edesc->src_nents ? : 1;
990 unsigned int dst_nents = edesc->dst_nents ? : 1;
991
992 if (is_sec1 && dst && dst_nents > 1) {
993 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
994 len, DMA_FROM_DEVICE);
995 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
996 offset);
997 }
998 if (src != dst) {
999 if (src_nents == 1 || !is_sec1)
1000 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
1001
1002 if (dst && (dst_nents == 1 || !is_sec1))
1003 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
1004 } else if (src_nents == 1 || !is_sec1) {
1005 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
1006 }
1007}
1008
1009static void ipsec_esp_unmap(struct device *dev,
1010 struct talitos_edesc *edesc,
1011 struct aead_request *areq)
1012{
1013 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1014 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1015 unsigned int ivsize = crypto_aead_ivsize(aead);
1016 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
1017 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
1018
1019 if (is_ipsec_esp)
1020 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
1021 DMA_FROM_DEVICE);
1022 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
1023
1024 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
1025 areq->assoclen);
1026
1027 if (edesc->dma_len)
1028 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1029 DMA_BIDIRECTIONAL);
1030
1031 if (!is_ipsec_esp) {
1032 unsigned int dst_nents = edesc->dst_nents ? : 1;
1033
1034 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
1035 areq->assoclen + areq->cryptlen - ivsize);
1036 }
1037}
1038
1039/*
1040 * ipsec_esp descriptor callbacks
1041 */
1042static void ipsec_esp_encrypt_done(struct device *dev,
1043 struct talitos_desc *desc, void *context,
1044 int err)
1045{
1046 struct talitos_private *priv = dev_get_drvdata(dev);
1047 bool is_sec1 = has_ftr_sec1(priv);
1048 struct aead_request *areq = context;
1049 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1050 unsigned int authsize = crypto_aead_authsize(authenc);
1051 unsigned int ivsize = crypto_aead_ivsize(authenc);
1052 struct talitos_edesc *edesc;
1053 struct scatterlist *sg;
1054 void *icvdata;
1055
1056 edesc = container_of(desc, struct talitos_edesc, desc);
1057
1058 ipsec_esp_unmap(dev, edesc, areq);
1059
1060 /* copy the generated ICV to dst */
1061 if (edesc->icv_ool) {
1062 if (is_sec1)
1063 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
1064 else
1065 icvdata = &edesc->link_tbl[edesc->src_nents +
1066 edesc->dst_nents + 2];
1067 sg = sg_last(areq->dst, edesc->dst_nents);
1068 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1069 icvdata, authsize);
1070 }
1071
1072 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1073
1074 kfree(edesc);
1075
1076 aead_request_complete(areq, err);
1077}
1078
1079static void ipsec_esp_decrypt_swauth_done(struct device *dev,
1080 struct talitos_desc *desc,
1081 void *context, int err)
1082{
1083 struct aead_request *req = context;
1084 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1085 unsigned int authsize = crypto_aead_authsize(authenc);
1086 struct talitos_edesc *edesc;
1087 struct scatterlist *sg;
1088 char *oicv, *icv;
1089 struct talitos_private *priv = dev_get_drvdata(dev);
1090 bool is_sec1 = has_ftr_sec1(priv);
1091
1092 edesc = container_of(desc, struct talitos_edesc, desc);
1093
1094 ipsec_esp_unmap(dev, edesc, req);
1095
1096 if (!err) {
1097 /* auth check */
1098 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
1099 icv = (char *)sg_virt(sg) + sg->length - authsize;
1100
1101 if (edesc->dma_len) {
1102 if (is_sec1)
1103 oicv = (char *)&edesc->dma_link_tbl +
1104 req->assoclen + req->cryptlen;
1105 else
1106 oicv = (char *)
1107 &edesc->link_tbl[edesc->src_nents +
1108 edesc->dst_nents + 2];
1109 if (edesc->icv_ool)
1110 icv = oicv + authsize;
1111 } else
1112 oicv = (char *)&edesc->link_tbl[0];
1113
1114 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
1115 }
1116
1117 kfree(edesc);
1118
1119 aead_request_complete(req, err);
1120}
1121
1122static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
1123 struct talitos_desc *desc,
1124 void *context, int err)
1125{
1126 struct aead_request *req = context;
1127 struct talitos_edesc *edesc;
1128
1129 edesc = container_of(desc, struct talitos_edesc, desc);
1130
1131 ipsec_esp_unmap(dev, edesc, req);
1132
1133 /* check ICV auth status */
1134 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1135 DESC_HDR_LO_ICCR1_PASS))
1136 err = -EBADMSG;
1137
1138 kfree(edesc);
1139
1140 aead_request_complete(req, err);
1141}
1142
1143/*
1144 * convert scatterlist to SEC h/w link table format
1145 * stop at cryptlen bytes
1146 */
1147static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1148 unsigned int offset, int cryptlen,
1149 struct talitos_ptr *link_tbl_ptr)
1150{
1151 int n_sg = sg_count;
1152 int count = 0;
1153
1154 while (cryptlen && sg && n_sg--) {
1155 unsigned int len = sg_dma_len(sg);
1156
1157 if (offset >= len) {
1158 offset -= len;
1159 goto next;
1160 }
1161
1162 len -= offset;
1163
1164 if (len > cryptlen)
1165 len = cryptlen;
1166
1167 to_talitos_ptr(link_tbl_ptr + count,
1168 sg_dma_address(sg) + offset, len, 0);
1169 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
1170 count++;
1171 cryptlen -= len;
1172 offset = 0;
1173
1174next:
1175 sg = sg_next(sg);
1176 }
1177
1178 /* tag end of link table */
1179 if (count > 0)
1180 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1181 DESC_PTR_LNKTBL_RETURN, 0);
1182
1183 return count;
1184}
1185
1186static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src,
1187 unsigned int len, struct talitos_edesc *edesc,
1188 struct talitos_ptr *ptr, int sg_count,
1189 unsigned int offset, int tbl_off, int elen)
1190{
1191 struct talitos_private *priv = dev_get_drvdata(dev);
1192 bool is_sec1 = has_ftr_sec1(priv);
1193
1194 if (!src) {
1195 to_talitos_ptr(ptr, 0, 0, is_sec1);
1196 return 1;
1197 }
1198 to_talitos_ptr_ext_set(ptr, elen, is_sec1);
1199 if (sg_count == 1) {
1200 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
1201 return sg_count;
1202 }
1203 if (is_sec1) {
1204 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
1205 return sg_count;
1206 }
1207 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen,
1208 &edesc->link_tbl[tbl_off]);
1209 if (sg_count == 1) {
1210 /* Only one segment now, so no link tbl needed*/
1211 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1212 return sg_count;
1213 }
1214 to_talitos_ptr(ptr, edesc->dma_link_tbl +
1215 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
1216 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1217
1218 return sg_count;
1219}
1220
1221static int talitos_sg_map(struct device *dev, struct scatterlist *src,
1222 unsigned int len, struct talitos_edesc *edesc,
1223 struct talitos_ptr *ptr, int sg_count,
1224 unsigned int offset, int tbl_off)
1225{
1226 return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset,
1227 tbl_off, 0);
1228}
1229
1230/*
1231 * fill in and submit ipsec_esp descriptor
1232 */
1233static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
1234 void (*callback)(struct device *dev,
1235 struct talitos_desc *desc,
1236 void *context, int error))
1237{
1238 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
1239 unsigned int authsize = crypto_aead_authsize(aead);
1240 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1241 struct device *dev = ctx->dev;
1242 struct talitos_desc *desc = &edesc->desc;
1243 unsigned int cryptlen = areq->cryptlen;
1244 unsigned int ivsize = crypto_aead_ivsize(aead);
1245 int tbl_off = 0;
1246 int sg_count, ret;
1247 int elen = 0;
1248 bool sync_needed = false;
1249 struct talitos_private *priv = dev_get_drvdata(dev);
1250 bool is_sec1 = has_ftr_sec1(priv);
1251 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1252 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1253 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
1254
1255 /* hmac key */
1256 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
1257
1258 sg_count = edesc->src_nents ?: 1;
1259 if (is_sec1 && sg_count > 1)
1260 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1261 areq->assoclen + cryptlen);
1262 else
1263 sg_count = dma_map_sg(dev, areq->src, sg_count,
1264 (areq->src == areq->dst) ?
1265 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1266
1267 /* hmac data */
1268 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1269 &desc->ptr[1], sg_count, 0, tbl_off);
1270
1271 if (ret > 1) {
1272 tbl_off += ret;
1273 sync_needed = true;
1274 }
1275
1276 /* cipher iv */
1277 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
1278
1279 /* cipher key */
1280 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1281 ctx->enckeylen, is_sec1);
1282
1283 /*
1284 * cipher in
1285 * map and adjust cipher len to aead request cryptlen.
1286 * extent is bytes of HMAC postpended to ciphertext,
1287 * typically 12 for ipsec
1288 */
1289 if (is_ipsec_esp && (desc->hdr & DESC_HDR_MODE1_MDEU_CICV))
1290 elen = authsize;
1291
1292 ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4],
1293 sg_count, areq->assoclen, tbl_off, elen);
1294
1295 if (ret > 1) {
1296 tbl_off += ret;
1297 sync_needed = true;
1298 }
1299
1300 /* cipher out */
1301 if (areq->src != areq->dst) {
1302 sg_count = edesc->dst_nents ? : 1;
1303 if (!is_sec1 || sg_count == 1)
1304 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1305 }
1306
1307 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1308 sg_count, areq->assoclen, tbl_off);
1309
1310 if (is_ipsec_esp)
1311 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
1312
1313 /* ICV data */
1314 if (ret > 1) {
1315 tbl_off += ret;
1316 edesc->icv_ool = true;
1317 sync_needed = true;
1318
1319 if (is_ipsec_esp) {
1320 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1321 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1322 sizeof(struct talitos_ptr) + authsize;
1323
1324 /* Add an entry to the link table for ICV data */
1325 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
1326 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1327 is_sec1);
1328
1329 /* icv data follows link tables */
1330 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
1331 authsize, is_sec1);
1332 } else {
1333 dma_addr_t addr = edesc->dma_link_tbl;
1334
1335 if (is_sec1)
1336 addr += areq->assoclen + cryptlen;
1337 else
1338 addr += sizeof(struct talitos_ptr) * tbl_off;
1339
1340 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
1341 }
1342 } else if (!is_ipsec_esp) {
1343 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1344 &desc->ptr[6], sg_count, areq->assoclen +
1345 cryptlen,
1346 tbl_off);
1347 if (ret > 1) {
1348 tbl_off += ret;
1349 edesc->icv_ool = true;
1350 sync_needed = true;
1351 } else {
1352 edesc->icv_ool = false;
1353 }
1354 } else {
1355 edesc->icv_ool = false;
1356 }
1357
1358 /* iv out */
1359 if (is_ipsec_esp)
1360 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1361 DMA_FROM_DEVICE);
1362
1363 if (sync_needed)
1364 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1365 edesc->dma_len,
1366 DMA_BIDIRECTIONAL);
1367
1368 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1369 if (ret != -EINPROGRESS) {
1370 ipsec_esp_unmap(dev, edesc, areq);
1371 kfree(edesc);
1372 }
1373 return ret;
1374}
1375
1376/*
1377 * allocate and map the extended descriptor
1378 */
1379static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1380 struct scatterlist *src,
1381 struct scatterlist *dst,
1382 u8 *iv,
1383 unsigned int assoclen,
1384 unsigned int cryptlen,
1385 unsigned int authsize,
1386 unsigned int ivsize,
1387 int icv_stashing,
1388 u32 cryptoflags,
1389 bool encrypt)
1390{
1391 struct talitos_edesc *edesc;
1392 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
1393 dma_addr_t iv_dma = 0;
1394 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1395 GFP_ATOMIC;
1396 struct talitos_private *priv = dev_get_drvdata(dev);
1397 bool is_sec1 = has_ftr_sec1(priv);
1398 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
1399
1400 if (cryptlen + authsize > max_len) {
1401 dev_err(dev, "length exceeds h/w max limit\n");
1402 return ERR_PTR(-EINVAL);
1403 }
1404
1405 if (!dst || dst == src) {
1406 src_len = assoclen + cryptlen + authsize;
1407 src_nents = sg_nents_for_len(src, src_len);
1408 if (src_nents < 0) {
1409 dev_err(dev, "Invalid number of src SG.\n");
1410 return ERR_PTR(-EINVAL);
1411 }
1412 src_nents = (src_nents == 1) ? 0 : src_nents;
1413 dst_nents = dst ? src_nents : 0;
1414 dst_len = 0;
1415 } else { /* dst && dst != src*/
1416 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1417 src_nents = sg_nents_for_len(src, src_len);
1418 if (src_nents < 0) {
1419 dev_err(dev, "Invalid number of src SG.\n");
1420 return ERR_PTR(-EINVAL);
1421 }
1422 src_nents = (src_nents == 1) ? 0 : src_nents;
1423 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1424 dst_nents = sg_nents_for_len(dst, dst_len);
1425 if (dst_nents < 0) {
1426 dev_err(dev, "Invalid number of dst SG.\n");
1427 return ERR_PTR(-EINVAL);
1428 }
1429 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1430 }
1431
1432 /*
1433 * allocate space for base edesc plus the link tables,
1434 * allowing for two separate entries for AD and generated ICV (+ 2),
1435 * and space for two sets of ICVs (stashed and generated)
1436 */
1437 alloc_len = sizeof(struct talitos_edesc);
1438 if (src_nents || dst_nents) {
1439 if (is_sec1)
1440 dma_len = (src_nents ? src_len : 0) +
1441 (dst_nents ? dst_len : 0);
1442 else
1443 dma_len = (src_nents + dst_nents + 2) *
1444 sizeof(struct talitos_ptr) + authsize * 2;
1445 alloc_len += dma_len;
1446 } else {
1447 dma_len = 0;
1448 alloc_len += icv_stashing ? authsize : 0;
1449 }
1450
1451 /* if its a ahash, add space for a second desc next to the first one */
1452 if (is_sec1 && !dst)
1453 alloc_len += sizeof(struct talitos_desc);
1454 alloc_len += ivsize;
1455
1456 edesc = kmalloc(alloc_len, GFP_DMA | flags);
1457 if (!edesc)
1458 return ERR_PTR(-ENOMEM);
1459 if (ivsize) {
1460 iv = memcpy(((u8 *)edesc) + alloc_len - ivsize, iv, ivsize);
1461 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1462 }
1463 memset(&edesc->desc, 0, sizeof(edesc->desc));
1464
1465 edesc->src_nents = src_nents;
1466 edesc->dst_nents = dst_nents;
1467 edesc->iv_dma = iv_dma;
1468 edesc->dma_len = dma_len;
1469 if (dma_len) {
1470 void *addr = &edesc->link_tbl[0];
1471
1472 if (is_sec1 && !dst)
1473 addr += sizeof(struct talitos_desc);
1474 edesc->dma_link_tbl = dma_map_single(dev, addr,
1475 edesc->dma_len,
1476 DMA_BIDIRECTIONAL);
1477 }
1478 return edesc;
1479}
1480
1481static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
1482 int icv_stashing, bool encrypt)
1483{
1484 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1485 unsigned int authsize = crypto_aead_authsize(authenc);
1486 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1487 unsigned int ivsize = crypto_aead_ivsize(authenc);
1488
1489 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1490 iv, areq->assoclen, areq->cryptlen,
1491 authsize, ivsize, icv_stashing,
1492 areq->base.flags, encrypt);
1493}
1494
1495static int aead_encrypt(struct aead_request *req)
1496{
1497 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1498 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1499 struct talitos_edesc *edesc;
1500
1501 /* allocate extended descriptor */
1502 edesc = aead_edesc_alloc(req, req->iv, 0, true);
1503 if (IS_ERR(edesc))
1504 return PTR_ERR(edesc);
1505
1506 /* set encrypt */
1507 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1508
1509 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
1510}
1511
1512static int aead_decrypt(struct aead_request *req)
1513{
1514 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1515 unsigned int authsize = crypto_aead_authsize(authenc);
1516 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1517 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1518 struct talitos_edesc *edesc;
1519 struct scatterlist *sg;
1520 void *icvdata;
1521
1522 req->cryptlen -= authsize;
1523
1524 /* allocate extended descriptor */
1525 edesc = aead_edesc_alloc(req, req->iv, 1, false);
1526 if (IS_ERR(edesc))
1527 return PTR_ERR(edesc);
1528
1529 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1530 ((!edesc->src_nents && !edesc->dst_nents) ||
1531 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1532
1533 /* decrypt and check the ICV */
1534 edesc->desc.hdr = ctx->desc_hdr_template |
1535 DESC_HDR_DIR_INBOUND |
1536 DESC_HDR_MODE1_MDEU_CICV;
1537
1538 /* reset integrity check result bits */
1539
1540 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
1541 }
1542
1543 /* Have to check the ICV with software */
1544 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1545
1546 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1547 if (edesc->dma_len)
1548 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1549 edesc->dst_nents + 2];
1550 else
1551 icvdata = &edesc->link_tbl[0];
1552
1553 sg = sg_last(req->src, edesc->src_nents ? : 1);
1554
1555 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
1556
1557 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
1558}
1559
1560static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1561 const u8 *key, unsigned int keylen)
1562{
1563 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1564 struct device *dev = ctx->dev;
1565
1566 if (ctx->keylen)
1567 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1568
1569 memcpy(&ctx->key, key, keylen);
1570 ctx->keylen = keylen;
1571
1572 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1573
1574 return 0;
1575}
1576
1577static int ablkcipher_des_setkey(struct crypto_ablkcipher *cipher,
1578 const u8 *key, unsigned int keylen)
1579{
1580 u32 tmp[DES_EXPKEY_WORDS];
1581
1582 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1583 CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) &&
1584 !des_ekey(tmp, key)) {
1585 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1586 return -EINVAL;
1587 }
1588
1589 return ablkcipher_setkey(cipher, key, keylen);
1590}
1591
1592static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher,
1593 const u8 *key, unsigned int keylen)
1594{
1595 u32 flags;
1596 int err;
1597
1598 flags = crypto_ablkcipher_get_flags(cipher);
1599 err = __des3_verify_key(&flags, key);
1600 if (unlikely(err)) {
1601 crypto_ablkcipher_set_flags(cipher, flags);
1602 return err;
1603 }
1604
1605 return ablkcipher_setkey(cipher, key, keylen);
1606}
1607
1608static void common_nonsnoop_unmap(struct device *dev,
1609 struct talitos_edesc *edesc,
1610 struct ablkcipher_request *areq)
1611{
1612 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1613
1614 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
1615 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1616
1617 if (edesc->dma_len)
1618 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1619 DMA_BIDIRECTIONAL);
1620}
1621
1622static void ablkcipher_done(struct device *dev,
1623 struct talitos_desc *desc, void *context,
1624 int err)
1625{
1626 struct ablkcipher_request *areq = context;
1627 struct talitos_edesc *edesc;
1628
1629 edesc = container_of(desc, struct talitos_edesc, desc);
1630
1631 common_nonsnoop_unmap(dev, edesc, areq);
1632
1633 kfree(edesc);
1634
1635 areq->base.complete(&areq->base, err);
1636}
1637
1638static int common_nonsnoop(struct talitos_edesc *edesc,
1639 struct ablkcipher_request *areq,
1640 void (*callback) (struct device *dev,
1641 struct talitos_desc *desc,
1642 void *context, int error))
1643{
1644 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1645 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1646 struct device *dev = ctx->dev;
1647 struct talitos_desc *desc = &edesc->desc;
1648 unsigned int cryptlen = areq->nbytes;
1649 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1650 int sg_count, ret;
1651 bool sync_needed = false;
1652 struct talitos_private *priv = dev_get_drvdata(dev);
1653 bool is_sec1 = has_ftr_sec1(priv);
1654
1655 /* first DWORD empty */
1656
1657 /* cipher iv */
1658 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
1659
1660 /* cipher key */
1661 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
1662
1663 sg_count = edesc->src_nents ?: 1;
1664 if (is_sec1 && sg_count > 1)
1665 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1666 cryptlen);
1667 else
1668 sg_count = dma_map_sg(dev, areq->src, sg_count,
1669 (areq->src == areq->dst) ?
1670 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1671 /*
1672 * cipher in
1673 */
1674 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1675 &desc->ptr[3], sg_count, 0, 0);
1676 if (sg_count > 1)
1677 sync_needed = true;
1678
1679 /* cipher out */
1680 if (areq->src != areq->dst) {
1681 sg_count = edesc->dst_nents ? : 1;
1682 if (!is_sec1 || sg_count == 1)
1683 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1684 }
1685
1686 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1687 sg_count, 0, (edesc->src_nents + 1));
1688 if (ret > 1)
1689 sync_needed = true;
1690
1691 /* iv out */
1692 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
1693 DMA_FROM_DEVICE);
1694
1695 /* last DWORD empty */
1696
1697 if (sync_needed)
1698 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1699 edesc->dma_len, DMA_BIDIRECTIONAL);
1700
1701 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1702 if (ret != -EINPROGRESS) {
1703 common_nonsnoop_unmap(dev, edesc, areq);
1704 kfree(edesc);
1705 }
1706 return ret;
1707}
1708
1709static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1710 areq, bool encrypt)
1711{
1712 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1713 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1714 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
1715
1716 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1717 areq->info, 0, areq->nbytes, 0, ivsize, 0,
1718 areq->base.flags, encrypt);
1719}
1720
1721static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1722{
1723 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1724 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1725 struct talitos_edesc *edesc;
1726
1727 /* allocate extended descriptor */
1728 edesc = ablkcipher_edesc_alloc(areq, true);
1729 if (IS_ERR(edesc))
1730 return PTR_ERR(edesc);
1731
1732 /* set encrypt */
1733 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1734
1735 return common_nonsnoop(edesc, areq, ablkcipher_done);
1736}
1737
1738static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1739{
1740 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1741 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1742 struct talitos_edesc *edesc;
1743
1744 /* allocate extended descriptor */
1745 edesc = ablkcipher_edesc_alloc(areq, false);
1746 if (IS_ERR(edesc))
1747 return PTR_ERR(edesc);
1748
1749 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1750
1751 return common_nonsnoop(edesc, areq, ablkcipher_done);
1752}
1753
1754static void common_nonsnoop_hash_unmap(struct device *dev,
1755 struct talitos_edesc *edesc,
1756 struct ahash_request *areq)
1757{
1758 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1759 struct talitos_private *priv = dev_get_drvdata(dev);
1760 bool is_sec1 = has_ftr_sec1(priv);
1761 struct talitos_desc *desc = &edesc->desc;
1762 struct talitos_desc *desc2 = desc + 1;
1763
1764 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1765 if (desc->next_desc &&
1766 desc->ptr[5].ptr != desc2->ptr[5].ptr)
1767 unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
1768
1769 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
1770
1771 /* When using hashctx-in, must unmap it. */
1772 if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1))
1773 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1],
1774 DMA_TO_DEVICE);
1775 else if (desc->next_desc)
1776 unmap_single_talitos_ptr(dev, &desc2->ptr[1],
1777 DMA_TO_DEVICE);
1778
1779 if (is_sec1 && req_ctx->nbuf)
1780 unmap_single_talitos_ptr(dev, &desc->ptr[3],
1781 DMA_TO_DEVICE);
1782
1783 if (edesc->dma_len)
1784 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1785 DMA_BIDIRECTIONAL);
1786
1787 if (edesc->desc.next_desc)
1788 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1789 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
1790}
1791
1792static void ahash_done(struct device *dev,
1793 struct talitos_desc *desc, void *context,
1794 int err)
1795{
1796 struct ahash_request *areq = context;
1797 struct talitos_edesc *edesc =
1798 container_of(desc, struct talitos_edesc, desc);
1799 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1800
1801 if (!req_ctx->last && req_ctx->to_hash_later) {
1802 /* Position any partial block for next update/final/finup */
1803 req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
1804 req_ctx->nbuf = req_ctx->to_hash_later;
1805 }
1806 common_nonsnoop_hash_unmap(dev, edesc, areq);
1807
1808 kfree(edesc);
1809
1810 areq->base.complete(&areq->base, err);
1811}
1812
1813/*
1814 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1815 * ourself and submit a padded block
1816 */
1817static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
1818 struct talitos_edesc *edesc,
1819 struct talitos_ptr *ptr)
1820{
1821 static u8 padded_hash[64] = {
1822 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1823 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1824 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1825 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1826 };
1827
1828 pr_err_once("Bug in SEC1, padding ourself\n");
1829 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1830 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1831 (char *)padded_hash, DMA_TO_DEVICE);
1832}
1833
1834static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1835 struct ahash_request *areq, unsigned int length,
1836 unsigned int offset,
1837 void (*callback) (struct device *dev,
1838 struct talitos_desc *desc,
1839 void *context, int error))
1840{
1841 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1842 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1843 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1844 struct device *dev = ctx->dev;
1845 struct talitos_desc *desc = &edesc->desc;
1846 int ret;
1847 bool sync_needed = false;
1848 struct talitos_private *priv = dev_get_drvdata(dev);
1849 bool is_sec1 = has_ftr_sec1(priv);
1850 int sg_count;
1851
1852 /* first DWORD empty */
1853
1854 /* hash context in */
1855 if (!req_ctx->first || req_ctx->swinit) {
1856 map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
1857 req_ctx->hw_context_size,
1858 req_ctx->hw_context,
1859 DMA_TO_DEVICE);
1860 req_ctx->swinit = 0;
1861 }
1862 /* Indicate next op is not the first. */
1863 req_ctx->first = 0;
1864
1865 /* HMAC key */
1866 if (ctx->keylen)
1867 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1868 is_sec1);
1869
1870 if (is_sec1 && req_ctx->nbuf)
1871 length -= req_ctx->nbuf;
1872
1873 sg_count = edesc->src_nents ?: 1;
1874 if (is_sec1 && sg_count > 1)
1875 sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
1876 edesc->buf + sizeof(struct talitos_desc),
1877 length, req_ctx->nbuf);
1878 else if (length)
1879 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1880 DMA_TO_DEVICE);
1881 /*
1882 * data in
1883 */
1884 if (is_sec1 && req_ctx->nbuf) {
1885 map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
1886 req_ctx->buf[req_ctx->buf_idx],
1887 DMA_TO_DEVICE);
1888 } else {
1889 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1890 &desc->ptr[3], sg_count, offset, 0);
1891 if (sg_count > 1)
1892 sync_needed = true;
1893 }
1894
1895 /* fifth DWORD empty */
1896
1897 /* hash/HMAC out -or- hash context out */
1898 if (req_ctx->last)
1899 map_single_talitos_ptr(dev, &desc->ptr[5],
1900 crypto_ahash_digestsize(tfm),
1901 areq->result, DMA_FROM_DEVICE);
1902 else
1903 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1904 req_ctx->hw_context_size,
1905 req_ctx->hw_context,
1906 DMA_FROM_DEVICE);
1907
1908 /* last DWORD empty */
1909
1910 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1911 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1912
1913 if (is_sec1 && req_ctx->nbuf && length) {
1914 struct talitos_desc *desc2 = desc + 1;
1915 dma_addr_t next_desc;
1916
1917 memset(desc2, 0, sizeof(*desc2));
1918 desc2->hdr = desc->hdr;
1919 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1920 desc2->hdr1 = desc2->hdr;
1921 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1922 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1923 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1924
1925 if (desc->ptr[1].ptr)
1926 copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
1927 is_sec1);
1928 else
1929 map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
1930 req_ctx->hw_context_size,
1931 req_ctx->hw_context,
1932 DMA_TO_DEVICE);
1933 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1934 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1935 &desc2->ptr[3], sg_count, offset, 0);
1936 if (sg_count > 1)
1937 sync_needed = true;
1938 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1939 if (req_ctx->last)
1940 map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
1941 req_ctx->hw_context_size,
1942 req_ctx->hw_context,
1943 DMA_FROM_DEVICE);
1944
1945 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1946 DMA_BIDIRECTIONAL);
1947 desc->next_desc = cpu_to_be32(next_desc);
1948 }
1949
1950 if (sync_needed)
1951 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1952 edesc->dma_len, DMA_BIDIRECTIONAL);
1953
1954 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
1955 if (ret != -EINPROGRESS) {
1956 common_nonsnoop_hash_unmap(dev, edesc, areq);
1957 kfree(edesc);
1958 }
1959 return ret;
1960}
1961
1962static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1963 unsigned int nbytes)
1964{
1965 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1966 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1967 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1968 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1969 bool is_sec1 = has_ftr_sec1(priv);
1970
1971 if (is_sec1)
1972 nbytes -= req_ctx->nbuf;
1973
1974 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
1975 nbytes, 0, 0, 0, areq->base.flags, false);
1976}
1977
1978static int ahash_init(struct ahash_request *areq)
1979{
1980 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1981 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1982 struct device *dev = ctx->dev;
1983 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1984 unsigned int size;
1985 dma_addr_t dma;
1986
1987 /* Initialize the context */
1988 req_ctx->buf_idx = 0;
1989 req_ctx->nbuf = 0;
1990 req_ctx->first = 1; /* first indicates h/w must init its context */
1991 req_ctx->swinit = 0; /* assume h/w init of context */
1992 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
1993 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1994 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
1995 req_ctx->hw_context_size = size;
1996
1997 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
1998 DMA_TO_DEVICE);
1999 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2000
2001 return 0;
2002}
2003
2004/*
2005 * on h/w without explicit sha224 support, we initialize h/w context
2006 * manually with sha224 constants, and tell it to run sha256.
2007 */
2008static int ahash_init_sha224_swinit(struct ahash_request *areq)
2009{
2010 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2011
2012 req_ctx->hw_context[0] = SHA224_H0;
2013 req_ctx->hw_context[1] = SHA224_H1;
2014 req_ctx->hw_context[2] = SHA224_H2;
2015 req_ctx->hw_context[3] = SHA224_H3;
2016 req_ctx->hw_context[4] = SHA224_H4;
2017 req_ctx->hw_context[5] = SHA224_H5;
2018 req_ctx->hw_context[6] = SHA224_H6;
2019 req_ctx->hw_context[7] = SHA224_H7;
2020
2021 /* init 64-bit count */
2022 req_ctx->hw_context[8] = 0;
2023 req_ctx->hw_context[9] = 0;
2024
2025 ahash_init(areq);
2026 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
2027
2028 return 0;
2029}
2030
2031static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
2032{
2033 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2034 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2035 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2036 struct talitos_edesc *edesc;
2037 unsigned int blocksize =
2038 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2039 unsigned int nbytes_to_hash;
2040 unsigned int to_hash_later;
2041 unsigned int nsg;
2042 int nents;
2043 struct device *dev = ctx->dev;
2044 struct talitos_private *priv = dev_get_drvdata(dev);
2045 bool is_sec1 = has_ftr_sec1(priv);
2046 int offset = 0;
2047 u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
2048
2049 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
2050 /* Buffer up to one whole block */
2051 nents = sg_nents_for_len(areq->src, nbytes);
2052 if (nents < 0) {
2053 dev_err(ctx->dev, "Invalid number of src SG.\n");
2054 return nents;
2055 }
2056 sg_copy_to_buffer(areq->src, nents,
2057 ctx_buf + req_ctx->nbuf, nbytes);
2058 req_ctx->nbuf += nbytes;
2059 return 0;
2060 }
2061
2062 /* At least (blocksize + 1) bytes are available to hash */
2063 nbytes_to_hash = nbytes + req_ctx->nbuf;
2064 to_hash_later = nbytes_to_hash & (blocksize - 1);
2065
2066 if (req_ctx->last)
2067 to_hash_later = 0;
2068 else if (to_hash_later)
2069 /* There is a partial block. Hash the full block(s) now */
2070 nbytes_to_hash -= to_hash_later;
2071 else {
2072 /* Keep one block buffered */
2073 nbytes_to_hash -= blocksize;
2074 to_hash_later = blocksize;
2075 }
2076
2077 /* Chain in any previously buffered data */
2078 if (!is_sec1 && req_ctx->nbuf) {
2079 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2080 sg_init_table(req_ctx->bufsl, nsg);
2081 sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
2082 if (nsg > 1)
2083 sg_chain(req_ctx->bufsl, 2, areq->src);
2084 req_ctx->psrc = req_ctx->bufsl;
2085 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2086 if (nbytes_to_hash > blocksize)
2087 offset = blocksize - req_ctx->nbuf;
2088 else
2089 offset = nbytes_to_hash - req_ctx->nbuf;
2090 nents = sg_nents_for_len(areq->src, offset);
2091 if (nents < 0) {
2092 dev_err(ctx->dev, "Invalid number of src SG.\n");
2093 return nents;
2094 }
2095 sg_copy_to_buffer(areq->src, nents,
2096 ctx_buf + req_ctx->nbuf, offset);
2097 req_ctx->nbuf += offset;
2098 req_ctx->psrc = areq->src;
2099 } else
2100 req_ctx->psrc = areq->src;
2101
2102 if (to_hash_later) {
2103 nents = sg_nents_for_len(areq->src, nbytes);
2104 if (nents < 0) {
2105 dev_err(ctx->dev, "Invalid number of src SG.\n");
2106 return nents;
2107 }
2108 sg_pcopy_to_buffer(areq->src, nents,
2109 req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2110 to_hash_later,
2111 nbytes - to_hash_later);
2112 }
2113 req_ctx->to_hash_later = to_hash_later;
2114
2115 /* Allocate extended descriptor */
2116 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2117 if (IS_ERR(edesc))
2118 return PTR_ERR(edesc);
2119
2120 edesc->desc.hdr = ctx->desc_hdr_template;
2121
2122 /* On last one, request SEC to pad; otherwise continue */
2123 if (req_ctx->last)
2124 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2125 else
2126 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2127
2128 /* request SEC to INIT hash. */
2129 if (req_ctx->first && !req_ctx->swinit)
2130 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2131
2132 /* When the tfm context has a keylen, it's an HMAC.
2133 * A first or last (ie. not middle) descriptor must request HMAC.
2134 */
2135 if (ctx->keylen && (req_ctx->first || req_ctx->last))
2136 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2137
2138 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
2139 ahash_done);
2140}
2141
2142static int ahash_update(struct ahash_request *areq)
2143{
2144 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2145
2146 req_ctx->last = 0;
2147
2148 return ahash_process_req(areq, areq->nbytes);
2149}
2150
2151static int ahash_final(struct ahash_request *areq)
2152{
2153 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2154
2155 req_ctx->last = 1;
2156
2157 return ahash_process_req(areq, 0);
2158}
2159
2160static int ahash_finup(struct ahash_request *areq)
2161{
2162 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2163
2164 req_ctx->last = 1;
2165
2166 return ahash_process_req(areq, areq->nbytes);
2167}
2168
2169static int ahash_digest(struct ahash_request *areq)
2170{
2171 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2172 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2173
2174 ahash->init(areq);
2175 req_ctx->last = 1;
2176
2177 return ahash_process_req(areq, areq->nbytes);
2178}
2179
2180static int ahash_export(struct ahash_request *areq, void *out)
2181{
2182 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2183 struct talitos_export_state *export = out;
2184 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2185 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2186 struct device *dev = ctx->dev;
2187 dma_addr_t dma;
2188
2189 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2190 DMA_FROM_DEVICE);
2191 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_FROM_DEVICE);
2192
2193 memcpy(export->hw_context, req_ctx->hw_context,
2194 req_ctx->hw_context_size);
2195 memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
2196 export->swinit = req_ctx->swinit;
2197 export->first = req_ctx->first;
2198 export->last = req_ctx->last;
2199 export->to_hash_later = req_ctx->to_hash_later;
2200 export->nbuf = req_ctx->nbuf;
2201
2202 return 0;
2203}
2204
2205static int ahash_import(struct ahash_request *areq, const void *in)
2206{
2207 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2208 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2209 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2210 struct device *dev = ctx->dev;
2211 const struct talitos_export_state *export = in;
2212 unsigned int size;
2213 dma_addr_t dma;
2214
2215 memset(req_ctx, 0, sizeof(*req_ctx));
2216 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
2217 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2218 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2219 req_ctx->hw_context_size = size;
2220 memcpy(req_ctx->hw_context, export->hw_context, size);
2221 memcpy(req_ctx->buf[0], export->buf, export->nbuf);
2222 req_ctx->swinit = export->swinit;
2223 req_ctx->first = export->first;
2224 req_ctx->last = export->last;
2225 req_ctx->to_hash_later = export->to_hash_later;
2226 req_ctx->nbuf = export->nbuf;
2227
2228 dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2229 DMA_TO_DEVICE);
2230 dma_unmap_single(dev, dma, req_ctx->hw_context_size, DMA_TO_DEVICE);
2231
2232 return 0;
2233}
2234
2235static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2236 u8 *hash)
2237{
2238 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2239
2240 struct scatterlist sg[1];
2241 struct ahash_request *req;
2242 struct crypto_wait wait;
2243 int ret;
2244
2245 crypto_init_wait(&wait);
2246
2247 req = ahash_request_alloc(tfm, GFP_KERNEL);
2248 if (!req)
2249 return -ENOMEM;
2250
2251 /* Keep tfm keylen == 0 during hash of the long key */
2252 ctx->keylen = 0;
2253 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2254 crypto_req_done, &wait);
2255
2256 sg_init_one(&sg[0], key, keylen);
2257
2258 ahash_request_set_crypt(req, sg, hash, keylen);
2259 ret = crypto_wait_req(crypto_ahash_digest(req), &wait);
2260
2261 ahash_request_free(req);
2262
2263 return ret;
2264}
2265
2266static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2267 unsigned int keylen)
2268{
2269 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2270 struct device *dev = ctx->dev;
2271 unsigned int blocksize =
2272 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2273 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2274 unsigned int keysize = keylen;
2275 u8 hash[SHA512_DIGEST_SIZE];
2276 int ret;
2277
2278 if (keylen <= blocksize)
2279 memcpy(ctx->key, key, keysize);
2280 else {
2281 /* Must get the hash of the long key */
2282 ret = keyhash(tfm, key, keylen, hash);
2283
2284 if (ret) {
2285 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2286 return -EINVAL;
2287 }
2288
2289 keysize = digestsize;
2290 memcpy(ctx->key, hash, digestsize);
2291 }
2292
2293 if (ctx->keylen)
2294 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2295
2296 ctx->keylen = keysize;
2297 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
2298
2299 return 0;
2300}
2301
2302
2303struct talitos_alg_template {
2304 u32 type;
2305 u32 priority;
2306 union {
2307 struct crypto_alg crypto;
2308 struct ahash_alg hash;
2309 struct aead_alg aead;
2310 } alg;
2311 __be32 desc_hdr_template;
2312};
2313
2314static struct talitos_alg_template driver_algs[] = {
2315 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
2316 { .type = CRYPTO_ALG_TYPE_AEAD,
2317 .alg.aead = {
2318 .base = {
2319 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2320 .cra_driver_name = "authenc-hmac-sha1-"
2321 "cbc-aes-talitos",
2322 .cra_blocksize = AES_BLOCK_SIZE,
2323 .cra_flags = CRYPTO_ALG_ASYNC,
2324 },
2325 .ivsize = AES_BLOCK_SIZE,
2326 .maxauthsize = SHA1_DIGEST_SIZE,
2327 },
2328 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2329 DESC_HDR_SEL0_AESU |
2330 DESC_HDR_MODE0_AESU_CBC |
2331 DESC_HDR_SEL1_MDEUA |
2332 DESC_HDR_MODE1_MDEU_INIT |
2333 DESC_HDR_MODE1_MDEU_PAD |
2334 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2335 },
2336 { .type = CRYPTO_ALG_TYPE_AEAD,
2337 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2338 .alg.aead = {
2339 .base = {
2340 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2341 .cra_driver_name = "authenc-hmac-sha1-"
2342 "cbc-aes-talitos",
2343 .cra_blocksize = AES_BLOCK_SIZE,
2344 .cra_flags = CRYPTO_ALG_ASYNC,
2345 },
2346 .ivsize = AES_BLOCK_SIZE,
2347 .maxauthsize = SHA1_DIGEST_SIZE,
2348 },
2349 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2350 DESC_HDR_SEL0_AESU |
2351 DESC_HDR_MODE0_AESU_CBC |
2352 DESC_HDR_SEL1_MDEUA |
2353 DESC_HDR_MODE1_MDEU_INIT |
2354 DESC_HDR_MODE1_MDEU_PAD |
2355 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2356 },
2357 { .type = CRYPTO_ALG_TYPE_AEAD,
2358 .alg.aead = {
2359 .base = {
2360 .cra_name = "authenc(hmac(sha1),"
2361 "cbc(des3_ede))",
2362 .cra_driver_name = "authenc-hmac-sha1-"
2363 "cbc-3des-talitos",
2364 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2365 .cra_flags = CRYPTO_ALG_ASYNC,
2366 },
2367 .ivsize = DES3_EDE_BLOCK_SIZE,
2368 .maxauthsize = SHA1_DIGEST_SIZE,
2369 .setkey = aead_des3_setkey,
2370 },
2371 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2372 DESC_HDR_SEL0_DEU |
2373 DESC_HDR_MODE0_DEU_CBC |
2374 DESC_HDR_MODE0_DEU_3DES |
2375 DESC_HDR_SEL1_MDEUA |
2376 DESC_HDR_MODE1_MDEU_INIT |
2377 DESC_HDR_MODE1_MDEU_PAD |
2378 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2379 },
2380 { .type = CRYPTO_ALG_TYPE_AEAD,
2381 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2382 .alg.aead = {
2383 .base = {
2384 .cra_name = "authenc(hmac(sha1),"
2385 "cbc(des3_ede))",
2386 .cra_driver_name = "authenc-hmac-sha1-"
2387 "cbc-3des-talitos",
2388 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2389 .cra_flags = CRYPTO_ALG_ASYNC,
2390 },
2391 .ivsize = DES3_EDE_BLOCK_SIZE,
2392 .maxauthsize = SHA1_DIGEST_SIZE,
2393 .setkey = aead_des3_setkey,
2394 },
2395 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2396 DESC_HDR_SEL0_DEU |
2397 DESC_HDR_MODE0_DEU_CBC |
2398 DESC_HDR_MODE0_DEU_3DES |
2399 DESC_HDR_SEL1_MDEUA |
2400 DESC_HDR_MODE1_MDEU_INIT |
2401 DESC_HDR_MODE1_MDEU_PAD |
2402 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2403 },
2404 { .type = CRYPTO_ALG_TYPE_AEAD,
2405 .alg.aead = {
2406 .base = {
2407 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2408 .cra_driver_name = "authenc-hmac-sha224-"
2409 "cbc-aes-talitos",
2410 .cra_blocksize = AES_BLOCK_SIZE,
2411 .cra_flags = CRYPTO_ALG_ASYNC,
2412 },
2413 .ivsize = AES_BLOCK_SIZE,
2414 .maxauthsize = SHA224_DIGEST_SIZE,
2415 },
2416 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2417 DESC_HDR_SEL0_AESU |
2418 DESC_HDR_MODE0_AESU_CBC |
2419 DESC_HDR_SEL1_MDEUA |
2420 DESC_HDR_MODE1_MDEU_INIT |
2421 DESC_HDR_MODE1_MDEU_PAD |
2422 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2423 },
2424 { .type = CRYPTO_ALG_TYPE_AEAD,
2425 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2426 .alg.aead = {
2427 .base = {
2428 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2429 .cra_driver_name = "authenc-hmac-sha224-"
2430 "cbc-aes-talitos",
2431 .cra_blocksize = AES_BLOCK_SIZE,
2432 .cra_flags = CRYPTO_ALG_ASYNC,
2433 },
2434 .ivsize = AES_BLOCK_SIZE,
2435 .maxauthsize = SHA224_DIGEST_SIZE,
2436 },
2437 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2438 DESC_HDR_SEL0_AESU |
2439 DESC_HDR_MODE0_AESU_CBC |
2440 DESC_HDR_SEL1_MDEUA |
2441 DESC_HDR_MODE1_MDEU_INIT |
2442 DESC_HDR_MODE1_MDEU_PAD |
2443 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2444 },
2445 { .type = CRYPTO_ALG_TYPE_AEAD,
2446 .alg.aead = {
2447 .base = {
2448 .cra_name = "authenc(hmac(sha224),"
2449 "cbc(des3_ede))",
2450 .cra_driver_name = "authenc-hmac-sha224-"
2451 "cbc-3des-talitos",
2452 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2453 .cra_flags = CRYPTO_ALG_ASYNC,
2454 },
2455 .ivsize = DES3_EDE_BLOCK_SIZE,
2456 .maxauthsize = SHA224_DIGEST_SIZE,
2457 .setkey = aead_des3_setkey,
2458 },
2459 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2460 DESC_HDR_SEL0_DEU |
2461 DESC_HDR_MODE0_DEU_CBC |
2462 DESC_HDR_MODE0_DEU_3DES |
2463 DESC_HDR_SEL1_MDEUA |
2464 DESC_HDR_MODE1_MDEU_INIT |
2465 DESC_HDR_MODE1_MDEU_PAD |
2466 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2467 },
2468 { .type = CRYPTO_ALG_TYPE_AEAD,
2469 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2470 .alg.aead = {
2471 .base = {
2472 .cra_name = "authenc(hmac(sha224),"
2473 "cbc(des3_ede))",
2474 .cra_driver_name = "authenc-hmac-sha224-"
2475 "cbc-3des-talitos",
2476 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2477 .cra_flags = CRYPTO_ALG_ASYNC,
2478 },
2479 .ivsize = DES3_EDE_BLOCK_SIZE,
2480 .maxauthsize = SHA224_DIGEST_SIZE,
2481 .setkey = aead_des3_setkey,
2482 },
2483 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2484 DESC_HDR_SEL0_DEU |
2485 DESC_HDR_MODE0_DEU_CBC |
2486 DESC_HDR_MODE0_DEU_3DES |
2487 DESC_HDR_SEL1_MDEUA |
2488 DESC_HDR_MODE1_MDEU_INIT |
2489 DESC_HDR_MODE1_MDEU_PAD |
2490 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2491 },
2492 { .type = CRYPTO_ALG_TYPE_AEAD,
2493 .alg.aead = {
2494 .base = {
2495 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2496 .cra_driver_name = "authenc-hmac-sha256-"
2497 "cbc-aes-talitos",
2498 .cra_blocksize = AES_BLOCK_SIZE,
2499 .cra_flags = CRYPTO_ALG_ASYNC,
2500 },
2501 .ivsize = AES_BLOCK_SIZE,
2502 .maxauthsize = SHA256_DIGEST_SIZE,
2503 },
2504 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2505 DESC_HDR_SEL0_AESU |
2506 DESC_HDR_MODE0_AESU_CBC |
2507 DESC_HDR_SEL1_MDEUA |
2508 DESC_HDR_MODE1_MDEU_INIT |
2509 DESC_HDR_MODE1_MDEU_PAD |
2510 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2511 },
2512 { .type = CRYPTO_ALG_TYPE_AEAD,
2513 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2514 .alg.aead = {
2515 .base = {
2516 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2517 .cra_driver_name = "authenc-hmac-sha256-"
2518 "cbc-aes-talitos",
2519 .cra_blocksize = AES_BLOCK_SIZE,
2520 .cra_flags = CRYPTO_ALG_ASYNC,
2521 },
2522 .ivsize = AES_BLOCK_SIZE,
2523 .maxauthsize = SHA256_DIGEST_SIZE,
2524 },
2525 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2526 DESC_HDR_SEL0_AESU |
2527 DESC_HDR_MODE0_AESU_CBC |
2528 DESC_HDR_SEL1_MDEUA |
2529 DESC_HDR_MODE1_MDEU_INIT |
2530 DESC_HDR_MODE1_MDEU_PAD |
2531 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2532 },
2533 { .type = CRYPTO_ALG_TYPE_AEAD,
2534 .alg.aead = {
2535 .base = {
2536 .cra_name = "authenc(hmac(sha256),"
2537 "cbc(des3_ede))",
2538 .cra_driver_name = "authenc-hmac-sha256-"
2539 "cbc-3des-talitos",
2540 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2541 .cra_flags = CRYPTO_ALG_ASYNC,
2542 },
2543 .ivsize = DES3_EDE_BLOCK_SIZE,
2544 .maxauthsize = SHA256_DIGEST_SIZE,
2545 .setkey = aead_des3_setkey,
2546 },
2547 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2548 DESC_HDR_SEL0_DEU |
2549 DESC_HDR_MODE0_DEU_CBC |
2550 DESC_HDR_MODE0_DEU_3DES |
2551 DESC_HDR_SEL1_MDEUA |
2552 DESC_HDR_MODE1_MDEU_INIT |
2553 DESC_HDR_MODE1_MDEU_PAD |
2554 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2555 },
2556 { .type = CRYPTO_ALG_TYPE_AEAD,
2557 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2558 .alg.aead = {
2559 .base = {
2560 .cra_name = "authenc(hmac(sha256),"
2561 "cbc(des3_ede))",
2562 .cra_driver_name = "authenc-hmac-sha256-"
2563 "cbc-3des-talitos",
2564 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2565 .cra_flags = CRYPTO_ALG_ASYNC,
2566 },
2567 .ivsize = DES3_EDE_BLOCK_SIZE,
2568 .maxauthsize = SHA256_DIGEST_SIZE,
2569 .setkey = aead_des3_setkey,
2570 },
2571 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2572 DESC_HDR_SEL0_DEU |
2573 DESC_HDR_MODE0_DEU_CBC |
2574 DESC_HDR_MODE0_DEU_3DES |
2575 DESC_HDR_SEL1_MDEUA |
2576 DESC_HDR_MODE1_MDEU_INIT |
2577 DESC_HDR_MODE1_MDEU_PAD |
2578 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2579 },
2580 { .type = CRYPTO_ALG_TYPE_AEAD,
2581 .alg.aead = {
2582 .base = {
2583 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2584 .cra_driver_name = "authenc-hmac-sha384-"
2585 "cbc-aes-talitos",
2586 .cra_blocksize = AES_BLOCK_SIZE,
2587 .cra_flags = CRYPTO_ALG_ASYNC,
2588 },
2589 .ivsize = AES_BLOCK_SIZE,
2590 .maxauthsize = SHA384_DIGEST_SIZE,
2591 },
2592 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2593 DESC_HDR_SEL0_AESU |
2594 DESC_HDR_MODE0_AESU_CBC |
2595 DESC_HDR_SEL1_MDEUB |
2596 DESC_HDR_MODE1_MDEU_INIT |
2597 DESC_HDR_MODE1_MDEU_PAD |
2598 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2599 },
2600 { .type = CRYPTO_ALG_TYPE_AEAD,
2601 .alg.aead = {
2602 .base = {
2603 .cra_name = "authenc(hmac(sha384),"
2604 "cbc(des3_ede))",
2605 .cra_driver_name = "authenc-hmac-sha384-"
2606 "cbc-3des-talitos",
2607 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2608 .cra_flags = CRYPTO_ALG_ASYNC,
2609 },
2610 .ivsize = DES3_EDE_BLOCK_SIZE,
2611 .maxauthsize = SHA384_DIGEST_SIZE,
2612 .setkey = aead_des3_setkey,
2613 },
2614 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2615 DESC_HDR_SEL0_DEU |
2616 DESC_HDR_MODE0_DEU_CBC |
2617 DESC_HDR_MODE0_DEU_3DES |
2618 DESC_HDR_SEL1_MDEUB |
2619 DESC_HDR_MODE1_MDEU_INIT |
2620 DESC_HDR_MODE1_MDEU_PAD |
2621 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2622 },
2623 { .type = CRYPTO_ALG_TYPE_AEAD,
2624 .alg.aead = {
2625 .base = {
2626 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2627 .cra_driver_name = "authenc-hmac-sha512-"
2628 "cbc-aes-talitos",
2629 .cra_blocksize = AES_BLOCK_SIZE,
2630 .cra_flags = CRYPTO_ALG_ASYNC,
2631 },
2632 .ivsize = AES_BLOCK_SIZE,
2633 .maxauthsize = SHA512_DIGEST_SIZE,
2634 },
2635 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2636 DESC_HDR_SEL0_AESU |
2637 DESC_HDR_MODE0_AESU_CBC |
2638 DESC_HDR_SEL1_MDEUB |
2639 DESC_HDR_MODE1_MDEU_INIT |
2640 DESC_HDR_MODE1_MDEU_PAD |
2641 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2642 },
2643 { .type = CRYPTO_ALG_TYPE_AEAD,
2644 .alg.aead = {
2645 .base = {
2646 .cra_name = "authenc(hmac(sha512),"
2647 "cbc(des3_ede))",
2648 .cra_driver_name = "authenc-hmac-sha512-"
2649 "cbc-3des-talitos",
2650 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2651 .cra_flags = CRYPTO_ALG_ASYNC,
2652 },
2653 .ivsize = DES3_EDE_BLOCK_SIZE,
2654 .maxauthsize = SHA512_DIGEST_SIZE,
2655 .setkey = aead_des3_setkey,
2656 },
2657 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2658 DESC_HDR_SEL0_DEU |
2659 DESC_HDR_MODE0_DEU_CBC |
2660 DESC_HDR_MODE0_DEU_3DES |
2661 DESC_HDR_SEL1_MDEUB |
2662 DESC_HDR_MODE1_MDEU_INIT |
2663 DESC_HDR_MODE1_MDEU_PAD |
2664 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2665 },
2666 { .type = CRYPTO_ALG_TYPE_AEAD,
2667 .alg.aead = {
2668 .base = {
2669 .cra_name = "authenc(hmac(md5),cbc(aes))",
2670 .cra_driver_name = "authenc-hmac-md5-"
2671 "cbc-aes-talitos",
2672 .cra_blocksize = AES_BLOCK_SIZE,
2673 .cra_flags = CRYPTO_ALG_ASYNC,
2674 },
2675 .ivsize = AES_BLOCK_SIZE,
2676 .maxauthsize = MD5_DIGEST_SIZE,
2677 },
2678 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2679 DESC_HDR_SEL0_AESU |
2680 DESC_HDR_MODE0_AESU_CBC |
2681 DESC_HDR_SEL1_MDEUA |
2682 DESC_HDR_MODE1_MDEU_INIT |
2683 DESC_HDR_MODE1_MDEU_PAD |
2684 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2685 },
2686 { .type = CRYPTO_ALG_TYPE_AEAD,
2687 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2688 .alg.aead = {
2689 .base = {
2690 .cra_name = "authenc(hmac(md5),cbc(aes))",
2691 .cra_driver_name = "authenc-hmac-md5-"
2692 "cbc-aes-talitos",
2693 .cra_blocksize = AES_BLOCK_SIZE,
2694 .cra_flags = CRYPTO_ALG_ASYNC,
2695 },
2696 .ivsize = AES_BLOCK_SIZE,
2697 .maxauthsize = MD5_DIGEST_SIZE,
2698 },
2699 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2700 DESC_HDR_SEL0_AESU |
2701 DESC_HDR_MODE0_AESU_CBC |
2702 DESC_HDR_SEL1_MDEUA |
2703 DESC_HDR_MODE1_MDEU_INIT |
2704 DESC_HDR_MODE1_MDEU_PAD |
2705 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2706 },
2707 { .type = CRYPTO_ALG_TYPE_AEAD,
2708 .alg.aead = {
2709 .base = {
2710 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2711 .cra_driver_name = "authenc-hmac-md5-"
2712 "cbc-3des-talitos",
2713 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2714 .cra_flags = CRYPTO_ALG_ASYNC,
2715 },
2716 .ivsize = DES3_EDE_BLOCK_SIZE,
2717 .maxauthsize = MD5_DIGEST_SIZE,
2718 .setkey = aead_des3_setkey,
2719 },
2720 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2721 DESC_HDR_SEL0_DEU |
2722 DESC_HDR_MODE0_DEU_CBC |
2723 DESC_HDR_MODE0_DEU_3DES |
2724 DESC_HDR_SEL1_MDEUA |
2725 DESC_HDR_MODE1_MDEU_INIT |
2726 DESC_HDR_MODE1_MDEU_PAD |
2727 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2728 },
2729 { .type = CRYPTO_ALG_TYPE_AEAD,
2730 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2731 .alg.aead = {
2732 .base = {
2733 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2734 .cra_driver_name = "authenc-hmac-md5-"
2735 "cbc-3des-talitos",
2736 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2737 .cra_flags = CRYPTO_ALG_ASYNC,
2738 },
2739 .ivsize = DES3_EDE_BLOCK_SIZE,
2740 .maxauthsize = MD5_DIGEST_SIZE,
2741 .setkey = aead_des3_setkey,
2742 },
2743 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2744 DESC_HDR_SEL0_DEU |
2745 DESC_HDR_MODE0_DEU_CBC |
2746 DESC_HDR_MODE0_DEU_3DES |
2747 DESC_HDR_SEL1_MDEUA |
2748 DESC_HDR_MODE1_MDEU_INIT |
2749 DESC_HDR_MODE1_MDEU_PAD |
2750 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2751 },
2752 /* ABLKCIPHER algorithms. */
2753 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2754 .alg.crypto = {
2755 .cra_name = "ecb(aes)",
2756 .cra_driver_name = "ecb-aes-talitos",
2757 .cra_blocksize = AES_BLOCK_SIZE,
2758 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2759 CRYPTO_ALG_ASYNC,
2760 .cra_ablkcipher = {
2761 .min_keysize = AES_MIN_KEY_SIZE,
2762 .max_keysize = AES_MAX_KEY_SIZE,
2763 .ivsize = AES_BLOCK_SIZE,
2764 }
2765 },
2766 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2767 DESC_HDR_SEL0_AESU,
2768 },
2769 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2770 .alg.crypto = {
2771 .cra_name = "cbc(aes)",
2772 .cra_driver_name = "cbc-aes-talitos",
2773 .cra_blocksize = AES_BLOCK_SIZE,
2774 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2775 CRYPTO_ALG_ASYNC,
2776 .cra_ablkcipher = {
2777 .min_keysize = AES_MIN_KEY_SIZE,
2778 .max_keysize = AES_MAX_KEY_SIZE,
2779 .ivsize = AES_BLOCK_SIZE,
2780 }
2781 },
2782 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2783 DESC_HDR_SEL0_AESU |
2784 DESC_HDR_MODE0_AESU_CBC,
2785 },
2786 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2787 .alg.crypto = {
2788 .cra_name = "ctr(aes)",
2789 .cra_driver_name = "ctr-aes-talitos",
2790 .cra_blocksize = AES_BLOCK_SIZE,
2791 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2792 CRYPTO_ALG_ASYNC,
2793 .cra_ablkcipher = {
2794 .min_keysize = AES_MIN_KEY_SIZE,
2795 .max_keysize = AES_MAX_KEY_SIZE,
2796 .ivsize = AES_BLOCK_SIZE,
2797 }
2798 },
2799 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
2800 DESC_HDR_SEL0_AESU |
2801 DESC_HDR_MODE0_AESU_CTR,
2802 },
2803 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2804 .alg.crypto = {
2805 .cra_name = "ecb(des)",
2806 .cra_driver_name = "ecb-des-talitos",
2807 .cra_blocksize = DES_BLOCK_SIZE,
2808 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2809 CRYPTO_ALG_ASYNC,
2810 .cra_ablkcipher = {
2811 .min_keysize = DES_KEY_SIZE,
2812 .max_keysize = DES_KEY_SIZE,
2813 .ivsize = DES_BLOCK_SIZE,
2814 .setkey = ablkcipher_des_setkey,
2815 }
2816 },
2817 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2818 DESC_HDR_SEL0_DEU,
2819 },
2820 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2821 .alg.crypto = {
2822 .cra_name = "cbc(des)",
2823 .cra_driver_name = "cbc-des-talitos",
2824 .cra_blocksize = DES_BLOCK_SIZE,
2825 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2826 CRYPTO_ALG_ASYNC,
2827 .cra_ablkcipher = {
2828 .min_keysize = DES_KEY_SIZE,
2829 .max_keysize = DES_KEY_SIZE,
2830 .ivsize = DES_BLOCK_SIZE,
2831 .setkey = ablkcipher_des_setkey,
2832 }
2833 },
2834 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2835 DESC_HDR_SEL0_DEU |
2836 DESC_HDR_MODE0_DEU_CBC,
2837 },
2838 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2839 .alg.crypto = {
2840 .cra_name = "ecb(des3_ede)",
2841 .cra_driver_name = "ecb-3des-talitos",
2842 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2843 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2844 CRYPTO_ALG_ASYNC,
2845 .cra_ablkcipher = {
2846 .min_keysize = DES3_EDE_KEY_SIZE,
2847 .max_keysize = DES3_EDE_KEY_SIZE,
2848 .ivsize = DES3_EDE_BLOCK_SIZE,
2849 .setkey = ablkcipher_des3_setkey,
2850 }
2851 },
2852 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2853 DESC_HDR_SEL0_DEU |
2854 DESC_HDR_MODE0_DEU_3DES,
2855 },
2856 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2857 .alg.crypto = {
2858 .cra_name = "cbc(des3_ede)",
2859 .cra_driver_name = "cbc-3des-talitos",
2860 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2861 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2862 CRYPTO_ALG_ASYNC,
2863 .cra_ablkcipher = {
2864 .min_keysize = DES3_EDE_KEY_SIZE,
2865 .max_keysize = DES3_EDE_KEY_SIZE,
2866 .ivsize = DES3_EDE_BLOCK_SIZE,
2867 .setkey = ablkcipher_des3_setkey,
2868 }
2869 },
2870 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2871 DESC_HDR_SEL0_DEU |
2872 DESC_HDR_MODE0_DEU_CBC |
2873 DESC_HDR_MODE0_DEU_3DES,
2874 },
2875 /* AHASH algorithms. */
2876 { .type = CRYPTO_ALG_TYPE_AHASH,
2877 .alg.hash = {
2878 .halg.digestsize = MD5_DIGEST_SIZE,
2879 .halg.statesize = sizeof(struct talitos_export_state),
2880 .halg.base = {
2881 .cra_name = "md5",
2882 .cra_driver_name = "md5-talitos",
2883 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2884 .cra_flags = CRYPTO_ALG_ASYNC,
2885 }
2886 },
2887 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2888 DESC_HDR_SEL0_MDEUA |
2889 DESC_HDR_MODE0_MDEU_MD5,
2890 },
2891 { .type = CRYPTO_ALG_TYPE_AHASH,
2892 .alg.hash = {
2893 .halg.digestsize = SHA1_DIGEST_SIZE,
2894 .halg.statesize = sizeof(struct talitos_export_state),
2895 .halg.base = {
2896 .cra_name = "sha1",
2897 .cra_driver_name = "sha1-talitos",
2898 .cra_blocksize = SHA1_BLOCK_SIZE,
2899 .cra_flags = CRYPTO_ALG_ASYNC,
2900 }
2901 },
2902 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2903 DESC_HDR_SEL0_MDEUA |
2904 DESC_HDR_MODE0_MDEU_SHA1,
2905 },
2906 { .type = CRYPTO_ALG_TYPE_AHASH,
2907 .alg.hash = {
2908 .halg.digestsize = SHA224_DIGEST_SIZE,
2909 .halg.statesize = sizeof(struct talitos_export_state),
2910 .halg.base = {
2911 .cra_name = "sha224",
2912 .cra_driver_name = "sha224-talitos",
2913 .cra_blocksize = SHA224_BLOCK_SIZE,
2914 .cra_flags = CRYPTO_ALG_ASYNC,
2915 }
2916 },
2917 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2918 DESC_HDR_SEL0_MDEUA |
2919 DESC_HDR_MODE0_MDEU_SHA224,
2920 },
2921 { .type = CRYPTO_ALG_TYPE_AHASH,
2922 .alg.hash = {
2923 .halg.digestsize = SHA256_DIGEST_SIZE,
2924 .halg.statesize = sizeof(struct talitos_export_state),
2925 .halg.base = {
2926 .cra_name = "sha256",
2927 .cra_driver_name = "sha256-talitos",
2928 .cra_blocksize = SHA256_BLOCK_SIZE,
2929 .cra_flags = CRYPTO_ALG_ASYNC,
2930 }
2931 },
2932 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2933 DESC_HDR_SEL0_MDEUA |
2934 DESC_HDR_MODE0_MDEU_SHA256,
2935 },
2936 { .type = CRYPTO_ALG_TYPE_AHASH,
2937 .alg.hash = {
2938 .halg.digestsize = SHA384_DIGEST_SIZE,
2939 .halg.statesize = sizeof(struct talitos_export_state),
2940 .halg.base = {
2941 .cra_name = "sha384",
2942 .cra_driver_name = "sha384-talitos",
2943 .cra_blocksize = SHA384_BLOCK_SIZE,
2944 .cra_flags = CRYPTO_ALG_ASYNC,
2945 }
2946 },
2947 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2948 DESC_HDR_SEL0_MDEUB |
2949 DESC_HDR_MODE0_MDEUB_SHA384,
2950 },
2951 { .type = CRYPTO_ALG_TYPE_AHASH,
2952 .alg.hash = {
2953 .halg.digestsize = SHA512_DIGEST_SIZE,
2954 .halg.statesize = sizeof(struct talitos_export_state),
2955 .halg.base = {
2956 .cra_name = "sha512",
2957 .cra_driver_name = "sha512-talitos",
2958 .cra_blocksize = SHA512_BLOCK_SIZE,
2959 .cra_flags = CRYPTO_ALG_ASYNC,
2960 }
2961 },
2962 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2963 DESC_HDR_SEL0_MDEUB |
2964 DESC_HDR_MODE0_MDEUB_SHA512,
2965 },
2966 { .type = CRYPTO_ALG_TYPE_AHASH,
2967 .alg.hash = {
2968 .halg.digestsize = MD5_DIGEST_SIZE,
2969 .halg.statesize = sizeof(struct talitos_export_state),
2970 .halg.base = {
2971 .cra_name = "hmac(md5)",
2972 .cra_driver_name = "hmac-md5-talitos",
2973 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
2974 .cra_flags = CRYPTO_ALG_ASYNC,
2975 }
2976 },
2977 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2978 DESC_HDR_SEL0_MDEUA |
2979 DESC_HDR_MODE0_MDEU_MD5,
2980 },
2981 { .type = CRYPTO_ALG_TYPE_AHASH,
2982 .alg.hash = {
2983 .halg.digestsize = SHA1_DIGEST_SIZE,
2984 .halg.statesize = sizeof(struct talitos_export_state),
2985 .halg.base = {
2986 .cra_name = "hmac(sha1)",
2987 .cra_driver_name = "hmac-sha1-talitos",
2988 .cra_blocksize = SHA1_BLOCK_SIZE,
2989 .cra_flags = CRYPTO_ALG_ASYNC,
2990 }
2991 },
2992 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2993 DESC_HDR_SEL0_MDEUA |
2994 DESC_HDR_MODE0_MDEU_SHA1,
2995 },
2996 { .type = CRYPTO_ALG_TYPE_AHASH,
2997 .alg.hash = {
2998 .halg.digestsize = SHA224_DIGEST_SIZE,
2999 .halg.statesize = sizeof(struct talitos_export_state),
3000 .halg.base = {
3001 .cra_name = "hmac(sha224)",
3002 .cra_driver_name = "hmac-sha224-talitos",
3003 .cra_blocksize = SHA224_BLOCK_SIZE,
3004 .cra_flags = CRYPTO_ALG_ASYNC,
3005 }
3006 },
3007 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3008 DESC_HDR_SEL0_MDEUA |
3009 DESC_HDR_MODE0_MDEU_SHA224,
3010 },
3011 { .type = CRYPTO_ALG_TYPE_AHASH,
3012 .alg.hash = {
3013 .halg.digestsize = SHA256_DIGEST_SIZE,
3014 .halg.statesize = sizeof(struct talitos_export_state),
3015 .halg.base = {
3016 .cra_name = "hmac(sha256)",
3017 .cra_driver_name = "hmac-sha256-talitos",
3018 .cra_blocksize = SHA256_BLOCK_SIZE,
3019 .cra_flags = CRYPTO_ALG_ASYNC,
3020 }
3021 },
3022 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3023 DESC_HDR_SEL0_MDEUA |
3024 DESC_HDR_MODE0_MDEU_SHA256,
3025 },
3026 { .type = CRYPTO_ALG_TYPE_AHASH,
3027 .alg.hash = {
3028 .halg.digestsize = SHA384_DIGEST_SIZE,
3029 .halg.statesize = sizeof(struct talitos_export_state),
3030 .halg.base = {
3031 .cra_name = "hmac(sha384)",
3032 .cra_driver_name = "hmac-sha384-talitos",
3033 .cra_blocksize = SHA384_BLOCK_SIZE,
3034 .cra_flags = CRYPTO_ALG_ASYNC,
3035 }
3036 },
3037 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3038 DESC_HDR_SEL0_MDEUB |
3039 DESC_HDR_MODE0_MDEUB_SHA384,
3040 },
3041 { .type = CRYPTO_ALG_TYPE_AHASH,
3042 .alg.hash = {
3043 .halg.digestsize = SHA512_DIGEST_SIZE,
3044 .halg.statesize = sizeof(struct talitos_export_state),
3045 .halg.base = {
3046 .cra_name = "hmac(sha512)",
3047 .cra_driver_name = "hmac-sha512-talitos",
3048 .cra_blocksize = SHA512_BLOCK_SIZE,
3049 .cra_flags = CRYPTO_ALG_ASYNC,
3050 }
3051 },
3052 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3053 DESC_HDR_SEL0_MDEUB |
3054 DESC_HDR_MODE0_MDEUB_SHA512,
3055 }
3056};
3057
3058struct talitos_crypto_alg {
3059 struct list_head entry;
3060 struct device *dev;
3061 struct talitos_alg_template algt;
3062};
3063
3064static int talitos_init_common(struct talitos_ctx *ctx,
3065 struct talitos_crypto_alg *talitos_alg)
3066{
3067 struct talitos_private *priv;
3068
3069 /* update context with ptr to dev */
3070 ctx->dev = talitos_alg->dev;
3071
3072 /* assign SEC channel to tfm in round-robin fashion */
3073 priv = dev_get_drvdata(ctx->dev);
3074 ctx->ch = atomic_inc_return(&priv->last_chan) &
3075 (priv->num_channels - 1);
3076
3077 /* copy descriptor header template value */
3078 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
3079
3080 /* select done notification */
3081 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3082
3083 return 0;
3084}
3085
3086static int talitos_cra_init(struct crypto_tfm *tfm)
3087{
3088 struct crypto_alg *alg = tfm->__crt_alg;
3089 struct talitos_crypto_alg *talitos_alg;
3090 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3091
3092 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
3093 talitos_alg = container_of(__crypto_ahash_alg(alg),
3094 struct talitos_crypto_alg,
3095 algt.alg.hash);
3096 else
3097 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3098 algt.alg.crypto);
3099
3100 return talitos_init_common(ctx, talitos_alg);
3101}
3102
3103static int talitos_cra_init_aead(struct crypto_aead *tfm)
3104{
3105 struct aead_alg *alg = crypto_aead_alg(tfm);
3106 struct talitos_crypto_alg *talitos_alg;
3107 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3108
3109 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3110 algt.alg.aead);
3111
3112 return talitos_init_common(ctx, talitos_alg);
3113}
3114
3115static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3116{
3117 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3118
3119 talitos_cra_init(tfm);
3120
3121 ctx->keylen = 0;
3122 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3123 sizeof(struct talitos_ahash_req_ctx));
3124
3125 return 0;
3126}
3127
3128static void talitos_cra_exit(struct crypto_tfm *tfm)
3129{
3130 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3131 struct device *dev = ctx->dev;
3132
3133 if (ctx->keylen)
3134 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3135}
3136
3137/*
3138 * given the alg's descriptor header template, determine whether descriptor
3139 * type and primary/secondary execution units required match the hw
3140 * capabilities description provided in the device tree node.
3141 */
3142static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3143{
3144 struct talitos_private *priv = dev_get_drvdata(dev);
3145 int ret;
3146
3147 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3148 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3149
3150 if (SECONDARY_EU(desc_hdr_template))
3151 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3152 & priv->exec_units);
3153
3154 return ret;
3155}
3156
3157static int talitos_remove(struct platform_device *ofdev)
3158{
3159 struct device *dev = &ofdev->dev;
3160 struct talitos_private *priv = dev_get_drvdata(dev);
3161 struct talitos_crypto_alg *t_alg, *n;
3162 int i;
3163
3164 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
3165 switch (t_alg->algt.type) {
3166 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3167 break;
3168 case CRYPTO_ALG_TYPE_AEAD:
3169 crypto_unregister_aead(&t_alg->algt.alg.aead);
3170 case CRYPTO_ALG_TYPE_AHASH:
3171 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3172 break;
3173 }
3174 list_del(&t_alg->entry);
3175 }
3176
3177 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3178 talitos_unregister_rng(dev);
3179
3180 for (i = 0; i < 2; i++)
3181 if (priv->irq[i]) {
3182 free_irq(priv->irq[i], dev);
3183 irq_dispose_mapping(priv->irq[i]);
3184 }
3185
3186 tasklet_kill(&priv->done_task[0]);
3187 if (priv->irq[1])
3188 tasklet_kill(&priv->done_task[1]);
3189
3190 return 0;
3191}
3192
3193static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3194 struct talitos_alg_template
3195 *template)
3196{
3197 struct talitos_private *priv = dev_get_drvdata(dev);
3198 struct talitos_crypto_alg *t_alg;
3199 struct crypto_alg *alg;
3200
3201 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3202 GFP_KERNEL);
3203 if (!t_alg)
3204 return ERR_PTR(-ENOMEM);
3205
3206 t_alg->algt = *template;
3207
3208 switch (t_alg->algt.type) {
3209 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3210 alg = &t_alg->algt.alg.crypto;
3211 alg->cra_init = talitos_cra_init;
3212 alg->cra_exit = talitos_cra_exit;
3213 alg->cra_type = &crypto_ablkcipher_type;
3214 alg->cra_ablkcipher.setkey = alg->cra_ablkcipher.setkey ?:
3215 ablkcipher_setkey;
3216 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3217 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3218 break;
3219 case CRYPTO_ALG_TYPE_AEAD:
3220 alg = &t_alg->algt.alg.aead.base;
3221 alg->cra_exit = talitos_cra_exit;
3222 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3223 t_alg->algt.alg.aead.setkey = t_alg->algt.alg.aead.setkey ?:
3224 aead_setkey;
3225 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3226 t_alg->algt.alg.aead.decrypt = aead_decrypt;
3227 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3228 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
3229 devm_kfree(dev, t_alg);
3230 return ERR_PTR(-ENOTSUPP);
3231 }
3232 break;
3233 case CRYPTO_ALG_TYPE_AHASH:
3234 alg = &t_alg->algt.alg.hash.halg.base;
3235 alg->cra_init = talitos_cra_init_ahash;
3236 alg->cra_exit = talitos_cra_exit;
3237 t_alg->algt.alg.hash.init = ahash_init;
3238 t_alg->algt.alg.hash.update = ahash_update;
3239 t_alg->algt.alg.hash.final = ahash_final;
3240 t_alg->algt.alg.hash.finup = ahash_finup;
3241 t_alg->algt.alg.hash.digest = ahash_digest;
3242 if (!strncmp(alg->cra_name, "hmac", 4))
3243 t_alg->algt.alg.hash.setkey = ahash_setkey;
3244 t_alg->algt.alg.hash.import = ahash_import;
3245 t_alg->algt.alg.hash.export = ahash_export;
3246
3247 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
3248 !strncmp(alg->cra_name, "hmac", 4)) {
3249 devm_kfree(dev, t_alg);
3250 return ERR_PTR(-ENOTSUPP);
3251 }
3252 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3253 (!strcmp(alg->cra_name, "sha224") ||
3254 !strcmp(alg->cra_name, "hmac(sha224)"))) {
3255 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3256 t_alg->algt.desc_hdr_template =
3257 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3258 DESC_HDR_SEL0_MDEUA |
3259 DESC_HDR_MODE0_MDEU_SHA256;
3260 }
3261 break;
3262 default:
3263 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
3264 devm_kfree(dev, t_alg);
3265 return ERR_PTR(-EINVAL);
3266 }
3267
3268 alg->cra_module = THIS_MODULE;
3269 if (t_alg->algt.priority)
3270 alg->cra_priority = t_alg->algt.priority;
3271 else
3272 alg->cra_priority = TALITOS_CRA_PRIORITY;
3273 alg->cra_alignmask = 0;
3274 alg->cra_ctxsize = sizeof(struct talitos_ctx);
3275 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
3276
3277 t_alg->dev = dev;
3278
3279 return t_alg;
3280}
3281
3282static int talitos_probe_irq(struct platform_device *ofdev)
3283{
3284 struct device *dev = &ofdev->dev;
3285 struct device_node *np = ofdev->dev.of_node;
3286 struct talitos_private *priv = dev_get_drvdata(dev);
3287 int err;
3288 bool is_sec1 = has_ftr_sec1(priv);
3289
3290 priv->irq[0] = irq_of_parse_and_map(np, 0);
3291 if (!priv->irq[0]) {
3292 dev_err(dev, "failed to map irq\n");
3293 return -EINVAL;
3294 }
3295 if (is_sec1) {
3296 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3297 dev_driver_string(dev), dev);
3298 goto primary_out;
3299 }
3300
3301 priv->irq[1] = irq_of_parse_and_map(np, 1);
3302
3303 /* get the primary irq line */
3304 if (!priv->irq[1]) {
3305 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
3306 dev_driver_string(dev), dev);
3307 goto primary_out;
3308 }
3309
3310 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
3311 dev_driver_string(dev), dev);
3312 if (err)
3313 goto primary_out;
3314
3315 /* get the secondary irq line */
3316 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
3317 dev_driver_string(dev), dev);
3318 if (err) {
3319 dev_err(dev, "failed to request secondary irq\n");
3320 irq_dispose_mapping(priv->irq[1]);
3321 priv->irq[1] = 0;
3322 }
3323
3324 return err;
3325
3326primary_out:
3327 if (err) {
3328 dev_err(dev, "failed to request primary irq\n");
3329 irq_dispose_mapping(priv->irq[0]);
3330 priv->irq[0] = 0;
3331 }
3332
3333 return err;
3334}
3335
3336static int talitos_probe(struct platform_device *ofdev)
3337{
3338 struct device *dev = &ofdev->dev;
3339 struct device_node *np = ofdev->dev.of_node;
3340 struct talitos_private *priv;
3341 int i, err;
3342 int stride;
3343 struct resource *res;
3344
3345 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
3346 if (!priv)
3347 return -ENOMEM;
3348
3349 INIT_LIST_HEAD(&priv->alg_list);
3350
3351 dev_set_drvdata(dev, priv);
3352
3353 priv->ofdev = ofdev;
3354
3355 spin_lock_init(&priv->reg_lock);
3356
3357 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3358 if (!res)
3359 return -ENXIO;
3360 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
3361 if (!priv->reg) {
3362 dev_err(dev, "failed to of_iomap\n");
3363 err = -ENOMEM;
3364 goto err_out;
3365 }
3366
3367 /* get SEC version capabilities from device tree */
3368 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3369 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3370 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3371 of_property_read_u32(np, "fsl,descriptor-types-mask",
3372 &priv->desc_types);
3373
3374 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3375 !priv->exec_units || !priv->desc_types) {
3376 dev_err(dev, "invalid property data in device tree node\n");
3377 err = -EINVAL;
3378 goto err_out;
3379 }
3380
3381 if (of_device_is_compatible(np, "fsl,sec3.0"))
3382 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3383
3384 if (of_device_is_compatible(np, "fsl,sec2.1"))
3385 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
3386 TALITOS_FTR_SHA224_HWINIT |
3387 TALITOS_FTR_HMAC_OK;
3388
3389 if (of_device_is_compatible(np, "fsl,sec1.0"))
3390 priv->features |= TALITOS_FTR_SEC1;
3391
3392 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3393 priv->reg_deu = priv->reg + TALITOS12_DEU;
3394 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3395 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3396 stride = TALITOS1_CH_STRIDE;
3397 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3398 priv->reg_deu = priv->reg + TALITOS10_DEU;
3399 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3400 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3401 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3402 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3403 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3404 stride = TALITOS1_CH_STRIDE;
3405 } else {
3406 priv->reg_deu = priv->reg + TALITOS2_DEU;
3407 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3408 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3409 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3410 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3411 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3412 priv->reg_keu = priv->reg + TALITOS2_KEU;
3413 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3414 stride = TALITOS2_CH_STRIDE;
3415 }
3416
3417 err = talitos_probe_irq(ofdev);
3418 if (err)
3419 goto err_out;
3420
3421 if (of_device_is_compatible(np, "fsl,sec1.0")) {
3422 if (priv->num_channels == 1)
3423 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
3424 (unsigned long)dev);
3425 else
3426 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3427 (unsigned long)dev);
3428 } else {
3429 if (priv->irq[1]) {
3430 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3431 (unsigned long)dev);
3432 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3433 (unsigned long)dev);
3434 } else if (priv->num_channels == 1) {
3435 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3436 (unsigned long)dev);
3437 } else {
3438 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3439 (unsigned long)dev);
3440 }
3441 }
3442
3443 priv->chan = devm_kcalloc(dev,
3444 priv->num_channels,
3445 sizeof(struct talitos_channel),
3446 GFP_KERNEL);
3447 if (!priv->chan) {
3448 dev_err(dev, "failed to allocate channel management space\n");
3449 err = -ENOMEM;
3450 goto err_out;
3451 }
3452
3453 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3454
3455 for (i = 0; i < priv->num_channels; i++) {
3456 priv->chan[i].reg = priv->reg + stride * (i + 1);
3457 if (!priv->irq[1] || !(i & 1))
3458 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
3459
3460 spin_lock_init(&priv->chan[i].head_lock);
3461 spin_lock_init(&priv->chan[i].tail_lock);
3462
3463 priv->chan[i].fifo = devm_kcalloc(dev,
3464 priv->fifo_len,
3465 sizeof(struct talitos_request),
3466 GFP_KERNEL);
3467 if (!priv->chan[i].fifo) {
3468 dev_err(dev, "failed to allocate request fifo %d\n", i);
3469 err = -ENOMEM;
3470 goto err_out;
3471 }
3472
3473 atomic_set(&priv->chan[i].submit_count,
3474 -(priv->chfifo_len - 1));
3475 }
3476
3477 dma_set_mask(dev, DMA_BIT_MASK(36));
3478
3479 /* reset and initialize the h/w */
3480 err = init_device(dev);
3481 if (err) {
3482 dev_err(dev, "failed to initialize device\n");
3483 goto err_out;
3484 }
3485
3486 /* register the RNG, if available */
3487 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3488 err = talitos_register_rng(dev);
3489 if (err) {
3490 dev_err(dev, "failed to register hwrng: %d\n", err);
3491 goto err_out;
3492 } else
3493 dev_info(dev, "hwrng\n");
3494 }
3495
3496 /* register crypto algorithms the device supports */
3497 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3498 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3499 struct talitos_crypto_alg *t_alg;
3500 struct crypto_alg *alg = NULL;
3501
3502 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3503 if (IS_ERR(t_alg)) {
3504 err = PTR_ERR(t_alg);
3505 if (err == -ENOTSUPP)
3506 continue;
3507 goto err_out;
3508 }
3509
3510 switch (t_alg->algt.type) {
3511 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3512 err = crypto_register_alg(
3513 &t_alg->algt.alg.crypto);
3514 alg = &t_alg->algt.alg.crypto;
3515 break;
3516
3517 case CRYPTO_ALG_TYPE_AEAD:
3518 err = crypto_register_aead(
3519 &t_alg->algt.alg.aead);
3520 alg = &t_alg->algt.alg.aead.base;
3521 break;
3522
3523 case CRYPTO_ALG_TYPE_AHASH:
3524 err = crypto_register_ahash(
3525 &t_alg->algt.alg.hash);
3526 alg = &t_alg->algt.alg.hash.halg.base;
3527 break;
3528 }
3529 if (err) {
3530 dev_err(dev, "%s alg registration failed\n",
3531 alg->cra_driver_name);
3532 devm_kfree(dev, t_alg);
3533 } else
3534 list_add_tail(&t_alg->entry, &priv->alg_list);
3535 }
3536 }
3537 if (!list_empty(&priv->alg_list))
3538 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3539 (char *)of_get_property(np, "compatible", NULL));
3540
3541 return 0;
3542
3543err_out:
3544 talitos_remove(ofdev);
3545
3546 return err;
3547}
3548
3549static const struct of_device_id talitos_match[] = {
3550#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3551 {
3552 .compatible = "fsl,sec1.0",
3553 },
3554#endif
3555#ifdef CONFIG_CRYPTO_DEV_TALITOS2
3556 {
3557 .compatible = "fsl,sec2.0",
3558 },
3559#endif
3560 {},
3561};
3562MODULE_DEVICE_TABLE(of, talitos_match);
3563
3564static struct platform_driver talitos_driver = {
3565 .driver = {
3566 .name = "talitos",
3567 .of_match_table = talitos_match,
3568 },
3569 .probe = talitos_probe,
3570 .remove = talitos_remove,
3571};
3572
3573module_platform_driver(talitos_driver);
3574
3575MODULE_LICENSE("GPL");
3576MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3577MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");