Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

crypto: rsa - implement Chinese Remainder Theorem for faster private key operations

Changes from v1:
* exported mpi_sub and mpi_mul, otherwise the build fails when RSA is a module

The kernel RSA ASN.1 private key parser already supports only private keys with
additional values to be used with the Chinese Remainder Theorem [1], but these
values are currently not used.

This rudimentary CRT implementation speeds up RSA private key operations for the
following Go benchmark up to ~3x.

This implementation also tries to minimise the allocation of additional MPIs,
so existing MPIs are reused as much as possible (hence the variable names are a
bit weird).

The benchmark used:

```
package keyring_test

import (
"crypto"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"io"
"syscall"
"testing"
"unsafe"
)

type KeySerial int32
type Keyring int32

const (
KEY_SPEC_PROCESS_KEYRING Keyring = -2
KEYCTL_PKEY_SIGN = 27
)

var (
keyTypeAsym = []byte("asymmetric\x00")
sha256pkcs1 = []byte("enc=pkcs1 hash=sha256\x00")
)

func (keyring Keyring) LoadAsym(desc string, payload []byte) (KeySerial, error) {
cdesc := []byte(desc + "\x00")
serial, _, errno := syscall.Syscall6(syscall.SYS_ADD_KEY, uintptr(unsafe.Pointer(&keyTypeAsym[0])), uintptr(unsafe.Pointer(&cdesc[0])), uintptr(unsafe.Pointer(&payload[0])), uintptr(len(payload)), uintptr(keyring), uintptr(0))
if errno == 0 {
return KeySerial(serial), nil
}

return KeySerial(serial), errno
}

type pkeyParams struct {
key_id KeySerial
in_len uint32
out_or_in2_len uint32
__spare [7]uint32
}

// the output signature buffer is an input parameter here, because we want to
// avoid Go buffer allocation leaking into our benchmarks
func (key KeySerial) Sign(info, digest, out []byte) error {
var params pkeyParams
params.key_id = key
params.in_len = uint32(len(digest))
params.out_or_in2_len = uint32(len(out))

_, _, errno := syscall.Syscall6(syscall.SYS_KEYCTL, KEYCTL_PKEY_SIGN, uintptr(unsafe.Pointer(&params)), uintptr(unsafe.Pointer(&info[0])), uintptr(unsafe.Pointer(&digest[0])), uintptr(unsafe.Pointer(&out[0])), uintptr(0))
if errno == 0 {
return nil
}

return errno
}

func BenchmarkSign(b *testing.B) {
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
b.Fatalf("failed to generate private key: %v", err)
}

pkcs8, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
b.Fatalf("failed to serialize the private key to PKCS8 blob: %v", err)
}

serial, err := KEY_SPEC_PROCESS_KEYRING.LoadAsym("test rsa key", pkcs8)
if err != nil {
b.Fatalf("failed to load the private key into the keyring: %v", err)
}

b.Logf("loaded test rsa key: %v", serial)

digest := make([]byte, 32)
_, err = io.ReadFull(rand.Reader, digest)
if err != nil {
b.Fatalf("failed to generate a random digest: %v", err)
}

sig := make([]byte, 256)
for n := 0; n < b.N; n++ {
err = serial.Sign(sha256pkcs1, digest, sig)
if err != nil {
b.Fatalf("failed to sign the digest: %v", err)
}
}

err = rsa.VerifyPKCS1v15(&priv.PublicKey, crypto.SHA256, digest, sig)
if err != nil {
b.Fatalf("failed to verify the signature: %v", err)
}
}
```

[1]: https://en.wikipedia.org/wiki/RSA_(cryptosystem)#Using_the_Chinese_remainder_algorithm

Signed-off-by: Ignat Korchagin <ignat@cloudflare.com>
Reported-by: kernel test robot <lkp@intel.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

authored by

Ignat Korchagin and committed by
Herbert Xu
f145d411 1b05ece0

+75 -6
+73 -5
crypto/rsa.c
··· 17 17 MPI n; 18 18 MPI e; 19 19 MPI d; 20 + MPI p; 21 + MPI q; 22 + MPI dp; 23 + MPI dq; 24 + MPI qinv; 20 25 }; 21 26 22 27 /* ··· 40 35 41 36 /* 42 37 * RSADP function [RFC3447 sec 5.1.2] 43 - * m = c^d mod n; 38 + * m_1 = c^dP mod p; 39 + * m_2 = c^dQ mod q; 40 + * h = (m_1 - m_2) * qInv mod p; 41 + * m = m_2 + q * h; 44 42 */ 45 - static int _rsa_dec(const struct rsa_mpi_key *key, MPI m, MPI c) 43 + static int _rsa_dec_crt(const struct rsa_mpi_key *key, MPI m_or_m1_or_h, MPI c) 46 44 { 45 + MPI m2, m12_or_qh; 46 + int ret = -ENOMEM; 47 + 47 48 /* (1) Validate 0 <= c < n */ 48 49 if (mpi_cmp_ui(c, 0) < 0 || mpi_cmp(c, key->n) >= 0) 49 50 return -EINVAL; 50 51 51 - /* (2) m = c^d mod n */ 52 - return mpi_powm(m, c, key->d, key->n); 52 + m2 = mpi_alloc(0); 53 + m12_or_qh = mpi_alloc(0); 54 + if (!m2 || !m12_or_qh) 55 + goto err_free_mpi; 56 + 57 + /* (2i) m_1 = c^dP mod p */ 58 + ret = mpi_powm(m_or_m1_or_h, c, key->dp, key->p); 59 + if (ret) 60 + goto err_free_mpi; 61 + 62 + /* (2i) m_2 = c^dQ mod q */ 63 + ret = mpi_powm(m2, c, key->dq, key->q); 64 + if (ret) 65 + goto err_free_mpi; 66 + 67 + /* (2iii) h = (m_1 - m_2) * qInv mod p */ 68 + mpi_sub(m12_or_qh, m_or_m1_or_h, m2); 69 + mpi_mulm(m_or_m1_or_h, m12_or_qh, key->qinv, key->p); 70 + 71 + /* (2iv) m = m_2 + q * h */ 72 + mpi_mul(m12_or_qh, key->q, m_or_m1_or_h); 73 + mpi_addm(m_or_m1_or_h, m2, m12_or_qh, key->n); 74 + 75 + ret = 0; 76 + 77 + err_free_mpi: 78 + mpi_free(m12_or_qh); 79 + mpi_free(m2); 80 + return ret; 53 81 } 54 82 55 83 static inline struct rsa_mpi_key *rsa_get_key(struct crypto_akcipher *tfm) ··· 150 112 if (!c) 151 113 goto err_free_m; 152 114 153 - ret = _rsa_dec(pkey, m, c); 115 + ret = _rsa_dec_crt(pkey, m, c); 154 116 if (ret) 155 117 goto err_free_c; 156 118 ··· 172 134 mpi_free(key->d); 173 135 mpi_free(key->e); 174 136 mpi_free(key->n); 137 + mpi_free(key->p); 138 + mpi_free(key->q); 139 + mpi_free(key->dp); 140 + mpi_free(key->dq); 141 + mpi_free(key->qinv); 175 142 key->d = NULL; 176 143 key->e = NULL; 177 144 key->n = NULL; 145 + key->p = NULL; 146 + key->q = NULL; 147 + key->dp = NULL; 148 + key->dq = NULL; 149 + key->qinv = NULL; 178 150 } 179 151 180 152 static int rsa_check_key_length(unsigned int len) ··· 263 215 264 216 mpi_key->n = mpi_read_raw_data(raw_key.n, raw_key.n_sz); 265 217 if (!mpi_key->n) 218 + goto err; 219 + 220 + mpi_key->p = mpi_read_raw_data(raw_key.p, raw_key.p_sz); 221 + if (!mpi_key->p) 222 + goto err; 223 + 224 + mpi_key->q = mpi_read_raw_data(raw_key.q, raw_key.q_sz); 225 + if (!mpi_key->q) 226 + goto err; 227 + 228 + mpi_key->dp = mpi_read_raw_data(raw_key.dp, raw_key.dp_sz); 229 + if (!mpi_key->dp) 230 + goto err; 231 + 232 + mpi_key->dq = mpi_read_raw_data(raw_key.dq, raw_key.dq_sz); 233 + if (!mpi_key->dq) 234 + goto err; 235 + 236 + mpi_key->qinv = mpi_read_raw_data(raw_key.qinv, raw_key.qinv_sz); 237 + if (!mpi_key->qinv) 266 238 goto err; 267 239 268 240 if (rsa_check_key_length(mpi_get_size(mpi_key->n) << 3)) {
+1 -1
lib/mpi/mpi-add.c
··· 138 138 mpi_add(w, u, vv); 139 139 mpi_free(vv); 140 140 } 141 - 141 + EXPORT_SYMBOL_GPL(mpi_sub); 142 142 143 143 void mpi_addm(MPI w, MPI u, MPI v, MPI m) 144 144 {
+1
lib/mpi/mpi-mul.c
··· 82 82 if (tmp_limb) 83 83 mpi_free_limb_space(tmp_limb); 84 84 } 85 + EXPORT_SYMBOL_GPL(mpi_mul); 85 86 86 87 void mpi_mulm(MPI w, MPI u, MPI v, MPI m) 87 88 {