Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

crypto: acomp - Add support for folios

For many users, it's easier to supply a folio rather than an SG
list since they already have them. Add support for folios to the
acomp interface.

Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

+182 -33
+35 -5
crypto/acompress.c
··· 12 12 #include <linux/errno.h> 13 13 #include <linux/kernel.h> 14 14 #include <linux/module.h> 15 + #include <linux/page-flags.h> 15 16 #include <linux/seq_file.h> 16 17 #include <linux/slab.h> 17 18 #include <linux/string.h> ··· 190 189 req->base.err = err; 191 190 state = &req->chain; 192 191 193 - if (state->src) 192 + if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) 194 193 acomp_request_set_src_dma(req, state->src, slen); 195 - if (state->dst) 194 + else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) 195 + acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); 196 + if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) 196 197 acomp_request_set_dst_dma(req, state->dst, dlen); 197 - state->src = NULL; 198 - state->dst = NULL; 198 + else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) 199 + acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); 199 200 } 200 201 201 202 static void acomp_virt_to_sg(struct acomp_req *req) 202 203 { 203 204 struct acomp_req_chain *state = &req->chain; 205 + 206 + state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | 207 + CRYPTO_ACOMP_REQ_DST_VIRT | 208 + CRYPTO_ACOMP_REQ_SRC_FOLIO | 209 + CRYPTO_ACOMP_REQ_DST_FOLIO); 204 210 205 211 if (acomp_request_src_isvirt(req)) { 206 212 unsigned int slen = req->slen; ··· 215 207 216 208 state->src = svirt; 217 209 sg_init_one(&state->ssg, svirt, slen); 210 + acomp_request_set_src_sg(req, &state->ssg, slen); 211 + } else if (acomp_request_src_isfolio(req)) { 212 + struct folio *folio = req->sfolio; 213 + unsigned int slen = req->slen; 214 + size_t off = req->soff; 215 + 216 + state->sfolio = folio; 217 + state->soff = off; 218 + sg_init_table(&state->ssg, 1); 219 + sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), 220 + slen, off % PAGE_SIZE); 218 221 acomp_request_set_src_sg(req, &state->ssg, slen); 219 222 } 220 223 ··· 236 217 state->dst = dvirt; 237 218 sg_init_one(&state->dsg, dvirt, dlen); 238 219 acomp_request_set_dst_sg(req, &state->dsg, dlen); 220 + } else if (acomp_request_dst_isfolio(req)) { 221 + struct folio *folio = req->dfolio; 222 + unsigned int dlen = req->dlen; 223 + size_t off = req->doff; 224 + 225 + state->dfolio = folio; 226 + state->doff = off; 227 + sg_init_table(&state->dsg, 1); 228 + sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), 229 + dlen, off % PAGE_SIZE); 230 + acomp_request_set_src_sg(req, &state->dsg, dlen); 239 231 } 240 232 } 241 233 ··· 358 328 int err; 359 329 360 330 if (crypto_acomp_req_chain(tfm) || 361 - (!acomp_request_chained(req) && !acomp_request_isvirt(req))) 331 + (!acomp_request_chained(req) && acomp_request_issg(req))) 362 332 return op(req); 363 333 364 334 if (acomp_is_async(tfm)) {
+44 -24
crypto/scompress.c
··· 177 177 unsigned int slen = req->slen; 178 178 unsigned int dlen = req->dlen; 179 179 struct page *spage, *dpage; 180 - unsigned int soff, doff; 181 180 unsigned int n; 182 181 const u8 *src; 182 + size_t soff; 183 + size_t doff; 183 184 u8 *dst; 184 185 int ret; 185 186 ··· 193 192 if (acomp_request_src_isvirt(req)) 194 193 src = req->svirt; 195 194 else { 196 - soff = req->src->offset; 197 - spage = nth_page(sg_page(req->src), soff / PAGE_SIZE); 198 - soff = offset_in_page(soff); 195 + src = scratch->src; 196 + do { 197 + if (acomp_request_src_isfolio(req)) { 198 + spage = folio_page(req->sfolio, 0); 199 + soff = req->soff; 200 + } else if (slen <= req->src->length) { 201 + spage = sg_page(req->src); 202 + soff = req->src->offset; 203 + } else 204 + break; 199 205 200 - n = slen / PAGE_SIZE; 201 - n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; 202 - if (slen <= req->src->length && 203 - (!PageHighMem(nth_page(spage, n)) || 204 - size_add(soff, slen) <= PAGE_SIZE)) 206 + spage = nth_page(spage, soff / PAGE_SIZE); 207 + soff = offset_in_page(soff); 208 + 209 + n = slen / PAGE_SIZE; 210 + n += (offset_in_page(slen) + soff - 1) / PAGE_SIZE; 211 + if (PageHighMem(nth_page(spage, n)) && 212 + size_add(soff, slen) > PAGE_SIZE) 213 + break; 205 214 src = kmap_local_page(spage) + soff; 206 - else 207 - src = scratch->src; 215 + } while (0); 208 216 } 209 217 210 218 if (acomp_request_dst_isvirt(req)) 211 219 dst = req->dvirt; 212 220 else { 213 - doff = req->dst->offset; 214 - dpage = nth_page(sg_page(req->dst), doff / PAGE_SIZE); 215 - doff = offset_in_page(doff); 221 + unsigned int max = SCOMP_SCRATCH_SIZE; 216 222 217 - n = dlen / PAGE_SIZE; 218 - n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; 219 - if (dlen <= req->dst->length && 220 - (!PageHighMem(nth_page(dpage, n)) || 221 - size_add(doff, dlen) <= PAGE_SIZE)) 223 + dst = scratch->dst; 224 + do { 225 + if (acomp_request_dst_isfolio(req)) { 226 + dpage = folio_page(req->dfolio, 0); 227 + doff = req->doff; 228 + } else if (dlen <= req->dst->length) { 229 + dpage = sg_page(req->dst); 230 + doff = req->dst->offset; 231 + } else 232 + break; 233 + 234 + dpage = nth_page(dpage, doff / PAGE_SIZE); 235 + doff = offset_in_page(doff); 236 + 237 + n = dlen / PAGE_SIZE; 238 + n += (offset_in_page(dlen) + doff - 1) / PAGE_SIZE; 239 + if (PageHighMem(dpage + n) && 240 + size_add(doff, dlen) > PAGE_SIZE) 241 + break; 222 242 dst = kmap_local_page(dpage) + doff; 223 - else { 224 - if (dlen > SCOMP_SCRATCH_SIZE) 225 - dlen = SCOMP_SCRATCH_SIZE; 226 - dst = scratch->dst; 227 - } 243 + max = dlen; 244 + } while (0); 245 + dlen = min(dlen, max); 228 246 } 229 247 230 248 spin_lock_bh(&scratch->lock);
+85 -4
include/crypto/acompress.h
··· 32 32 /* Set this bit for if virtual address destination cannot be used for DMA. */ 33 33 #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 34 34 35 + /* Set this bit if source is a folio. */ 36 + #define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 37 + 38 + /* Set this bit if destination is a folio. */ 39 + #define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 40 + 35 41 #define CRYPTO_ACOMP_DST_MAX 131072 36 42 37 43 #define MAX_SYNC_COMP_REQSIZE 0 ··· 49 43 __##name##_req, (tfm), (gfp), false) 50 44 51 45 struct acomp_req; 46 + struct folio; 52 47 53 48 struct acomp_req_chain { 54 49 struct list_head head; ··· 60 53 void *data; 61 54 struct scatterlist ssg; 62 55 struct scatterlist dsg; 63 - const u8 *src; 64 - u8 *dst; 56 + union { 57 + const u8 *src; 58 + struct folio *sfolio; 59 + }; 60 + union { 61 + u8 *dst; 62 + struct folio *dfolio; 63 + }; 64 + size_t soff; 65 + size_t doff; 66 + u32 flags; 65 67 }; 66 68 67 69 /** 68 70 * struct acomp_req - asynchronous (de)compression request 69 71 * 70 72 * @base: Common attributes for asynchronous crypto requests 71 - * @src: Source Data 72 - * @dst: Destination data 73 + * @src: Source scatterlist 74 + * @dst: Destination scatterlist 75 + * @svirt: Source virtual address 76 + * @dvirt: Destination virtual address 77 + * @sfolio: Source folio 78 + * @soff: Source folio offset 79 + * @dfolio: Destination folio 80 + * @doff: Destination folio offset 73 81 * @slen: Size of the input buffer 74 82 * @dlen: Size of the output buffer and number of bytes produced 75 83 * @chain: Private API code data, do not use ··· 95 73 union { 96 74 struct scatterlist *src; 97 75 const u8 *svirt; 76 + struct folio *sfolio; 98 77 }; 99 78 union { 100 79 struct scatterlist *dst; 101 80 u8 *dvirt; 81 + struct folio *dfolio; 102 82 }; 83 + size_t soff; 84 + size_t doff; 103 85 unsigned int slen; 104 86 unsigned int dlen; 105 87 ··· 342 316 { 343 317 u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | 344 318 CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | 319 + CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | 345 320 CRYPTO_TFM_REQ_ON_STACK; 346 321 347 322 req->base.complete = cmpl; ··· 379 352 380 353 req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | 381 354 CRYPTO_ACOMP_REQ_SRC_NONDMA | 355 + CRYPTO_ACOMP_REQ_SRC_FOLIO | 356 + CRYPTO_ACOMP_REQ_DST_FOLIO | 382 357 CRYPTO_ACOMP_REQ_DST_VIRT | 383 358 CRYPTO_ACOMP_REQ_DST_NONDMA); 384 359 } ··· 403 374 404 375 req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; 405 376 req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; 377 + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; 406 378 } 407 379 408 380 /** ··· 423 393 req->slen = slen; 424 394 425 395 req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; 396 + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; 426 397 req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; 427 398 } 428 399 ··· 444 413 req->svirt = src; 445 414 req->slen = slen; 446 415 416 + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; 447 417 req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; 448 418 req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; 419 + } 420 + 421 + /** 422 + * acomp_request_set_src_folio() -- Sets source folio 423 + * 424 + * Sets source folio required by an acomp operation. 425 + * 426 + * @req: asynchronous compress request 427 + * @folio: pointer to input folio 428 + * @off: input folio offset 429 + * @len: size of the input buffer 430 + */ 431 + static inline void acomp_request_set_src_folio(struct acomp_req *req, 432 + struct folio *folio, size_t off, 433 + unsigned int len) 434 + { 435 + req->sfolio = folio; 436 + req->soff = off; 437 + req->slen = len; 438 + 439 + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; 440 + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; 441 + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; 449 442 } 450 443 451 444 /** ··· 490 435 491 436 req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; 492 437 req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; 438 + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; 493 439 } 494 440 495 441 /** ··· 510 454 req->dlen = dlen; 511 455 512 456 req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; 457 + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; 513 458 req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; 514 459 } 515 460 ··· 530 473 req->dvirt = dst; 531 474 req->dlen = dlen; 532 475 476 + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; 533 477 req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; 534 478 req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; 479 + } 480 + 481 + /** 482 + * acomp_request_set_dst_folio() -- Sets destination folio 483 + * 484 + * Sets destination folio required by an acomp operation. 485 + * 486 + * @req: asynchronous compress request 487 + * @folio: pointer to input folio 488 + * @off: input folio offset 489 + * @len: size of the input buffer 490 + */ 491 + static inline void acomp_request_set_dst_folio(struct acomp_req *req, 492 + struct folio *folio, size_t off, 493 + unsigned int len) 494 + { 495 + req->dfolio = folio; 496 + req->doff = off; 497 + req->dlen = len; 498 + 499 + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; 500 + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; 501 + req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; 535 502 } 536 503 537 504 static inline void acomp_request_chain(struct acomp_req *req,
+18
include/crypto/internal/acompress.h
··· 103 103 return crypto_request_chained(&req->base); 104 104 } 105 105 106 + static inline bool acomp_request_issg(struct acomp_req *req) 107 + { 108 + return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | 109 + CRYPTO_ACOMP_REQ_DST_VIRT | 110 + CRYPTO_ACOMP_REQ_SRC_FOLIO | 111 + CRYPTO_ACOMP_REQ_DST_FOLIO)); 112 + } 113 + 106 114 static inline bool acomp_request_src_isvirt(struct acomp_req *req) 107 115 { 108 116 return req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT; ··· 141 133 { 142 134 return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_NONDMA | 143 135 CRYPTO_ACOMP_REQ_DST_NONDMA); 136 + } 137 + 138 + static inline bool acomp_request_src_isfolio(struct acomp_req *req) 139 + { 140 + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; 141 + } 142 + 143 + static inline bool acomp_request_dst_isfolio(struct acomp_req *req) 144 + { 145 + return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; 144 146 } 145 147 146 148 static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm)