tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

dmatest: add xor test

Extend dmatest to launch a thread per supported operation type and add
an xor test.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>

Dan Williams b54d5cb9 729b5d1b

+191 -87
+191 -87
drivers/dma/dmatest.c
··· 38 38 MODULE_PARM_DESC(max_channels, 39 39 "Maximum number of channels to use (default: all)"); 40 40 41 + static unsigned int xor_sources = 3; 42 + module_param(xor_sources, uint, S_IRUGO); 43 + MODULE_PARM_DESC(xor_sources, 44 + "Number of xor source buffers (default: 3)"); 45 + 41 46 /* 42 47 * Initialization patterns. All bytes in the source buffer has bit 7 43 48 * set, all bytes in the destination buffer has bit 7 cleared. ··· 64 59 struct list_head node; 65 60 struct task_struct *task; 66 61 struct dma_chan *chan; 67 - u8 *srcbuf; 68 - u8 *dstbuf; 62 + u8 **srcs; 63 + u8 **dsts; 64 + enum dma_transaction_type type; 69 65 }; 70 66 71 67 struct dmatest_chan { ··· 104 98 return buf; 105 99 } 106 100 107 - static void dmatest_init_srcbuf(u8 *buf, unsigned int start, unsigned int len) 101 + static void dmatest_init_srcs(u8 **bufs, unsigned int start, unsigned int len) 108 102 { 109 103 unsigned int i; 104 + u8 *buf; 110 105 111 - for (i = 0; i < start; i++) 112 - buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 113 - for ( ; i < start + len; i++) 114 - buf[i] = PATTERN_SRC | PATTERN_COPY 115 - | (~i & PATTERN_COUNT_MASK);; 116 - for ( ; i < test_buf_size; i++) 117 - buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 106 + for (; (buf = *bufs); bufs++) { 107 + for (i = 0; i < start; i++) 108 + buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 109 + for ( ; i < start + len; i++) 110 + buf[i] = PATTERN_SRC | PATTERN_COPY 111 + | (~i & PATTERN_COUNT_MASK);; 112 + for ( ; i < test_buf_size; i++) 113 + buf[i] = PATTERN_SRC | (~i & PATTERN_COUNT_MASK); 114 + buf++; 115 + } 118 116 } 119 117 120 - static void dmatest_init_dstbuf(u8 *buf, unsigned int start, unsigned int len) 118 + static void dmatest_init_dsts(u8 **bufs, unsigned int start, unsigned int len) 121 119 { 122 120 unsigned int i; 121 + u8 *buf; 123 122 124 - for (i = 0; i < start; i++) 125 - buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 126 - for ( ; i < start + len; i++) 127 - buf[i] = PATTERN_DST | PATTERN_OVERWRITE 128 - | (~i & PATTERN_COUNT_MASK); 129 - for ( ; i < test_buf_size; i++) 130 - buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 123 + for (; (buf = *bufs); bufs++) { 124 + for (i = 0; i < start; i++) 125 + buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 126 + for ( ; i < start + len; i++) 127 + buf[i] = PATTERN_DST | PATTERN_OVERWRITE 128 + | (~i & PATTERN_COUNT_MASK); 129 + for ( ; i < test_buf_size; i++) 130 + buf[i] = PATTERN_DST | (~i & PATTERN_COUNT_MASK); 131 + } 131 132 } 132 133 133 134 static void dmatest_mismatch(u8 actual, u8 pattern, unsigned int index, ··· 163 150 thread_name, index, expected, actual); 164 151 } 165 152 166 - static unsigned int dmatest_verify(u8 *buf, unsigned int start, 153 + static unsigned int dmatest_verify(u8 **bufs, unsigned int start, 167 154 unsigned int end, unsigned int counter, u8 pattern, 168 155 bool is_srcbuf) 169 156 { 170 157 unsigned int i; 171 158 unsigned int error_count = 0; 172 159 u8 actual; 160 + u8 expected; 161 + u8 *buf; 162 + unsigned int counter_orig = counter; 173 163 174 - for (i = start; i < end; i++) { 175 - actual = buf[i]; 176 - if (actual != (pattern | (~counter & PATTERN_COUNT_MASK))) { 177 - if (error_count < 32) 178 - dmatest_mismatch(actual, pattern, i, counter, 179 - is_srcbuf); 180 - error_count++; 164 + for (; (buf = *bufs); bufs++) { 165 + counter = counter_orig; 166 + for (i = start; i < end; i++) { 167 + actual = buf[i]; 168 + expected = pattern | (~counter & PATTERN_COUNT_MASK); 169 + if (actual != expected) { 170 + if (error_count < 32) 171 + dmatest_mismatch(actual, pattern, i, 172 + counter, is_srcbuf); 173 + error_count++; 174 + } 175 + counter++; 181 176 } 182 - counter++; 183 177 } 184 178 185 179 if (error_count > 32) ··· 198 178 199 179 /* 200 180 * This function repeatedly tests DMA transfers of various lengths and 201 - * offsets until it is told to exit by kthread_stop(). There may be 202 - * multiple threads running this function in parallel for a single 203 - * channel, and there may be multiple channels being tested in 204 - * parallel. 181 + * offsets for a given operation type until it is told to exit by 182 + * kthread_stop(). There may be multiple threads running this function 183 + * in parallel for a single channel, and there may be multiple channels 184 + * being tested in parallel. 205 185 * 206 186 * Before each test, the source and destination buffer is initialized 207 187 * with a known pattern. This pattern is different depending on ··· 221 201 unsigned int total_tests = 0; 222 202 dma_cookie_t cookie; 223 203 enum dma_status status; 204 + enum dma_ctrl_flags flags; 224 205 int ret; 206 + int src_cnt; 207 + int dst_cnt; 208 + int i; 225 209 226 210 thread_name = current->comm; 227 211 228 212 ret = -ENOMEM; 229 - thread->srcbuf = kmalloc(test_buf_size, GFP_KERNEL); 230 - if (!thread->srcbuf) 231 - goto err_srcbuf; 232 - thread->dstbuf = kmalloc(test_buf_size, GFP_KERNEL); 233 - if (!thread->dstbuf) 234 - goto err_dstbuf; 235 213 236 214 smp_rmb(); 237 215 chan = thread->chan; 216 + if (thread->type == DMA_MEMCPY) 217 + src_cnt = dst_cnt = 1; 218 + else if (thread->type == DMA_XOR) { 219 + src_cnt = xor_sources | 1; /* force odd to ensure dst = src */ 220 + dst_cnt = 1; 221 + } else 222 + goto err_srcs; 223 + 224 + thread->srcs = kcalloc(src_cnt+1, sizeof(u8 *), GFP_KERNEL); 225 + if (!thread->srcs) 226 + goto err_srcs; 227 + for (i = 0; i < src_cnt; i++) { 228 + thread->srcs[i] = kmalloc(test_buf_size, GFP_KERNEL); 229 + if (!thread->srcs[i]) 230 + goto err_srcbuf; 231 + } 232 + thread->srcs[i] = NULL; 233 + 234 + thread->dsts = kcalloc(dst_cnt+1, sizeof(u8 *), GFP_KERNEL); 235 + if (!thread->dsts) 236 + goto err_dsts; 237 + for (i = 0; i < dst_cnt; i++) { 238 + thread->dsts[i] = kmalloc(test_buf_size, GFP_KERNEL); 239 + if (!thread->dsts[i]) 240 + goto err_dstbuf; 241 + } 242 + thread->dsts[i] = NULL; 243 + 244 + flags = DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP; 238 245 239 246 while (!kthread_should_stop()) { 240 247 struct dma_device *dev = chan->device; 241 - struct dma_async_tx_descriptor *tx; 242 - dma_addr_t dma_src, dma_dest; 248 + struct dma_async_tx_descriptor *tx = NULL; 249 + dma_addr_t dma_srcs[src_cnt]; 250 + dma_addr_t dma_dsts[dst_cnt]; 243 251 244 252 total_tests++; 245 253 ··· 275 227 src_off = dmatest_random() % (test_buf_size - len + 1); 276 228 dst_off = dmatest_random() % (test_buf_size - len + 1); 277 229 278 - dmatest_init_srcbuf(thread->srcbuf, src_off, len); 279 - dmatest_init_dstbuf(thread->dstbuf, dst_off, len); 230 + dmatest_init_srcs(thread->srcs, src_off, len); 231 + dmatest_init_dsts(thread->dsts, dst_off, len); 280 232 281 - dma_src = dma_map_single(dev->dev, thread->srcbuf + src_off, 282 - len, DMA_TO_DEVICE); 233 + for (i = 0; i < src_cnt; i++) { 234 + u8 *buf = thread->srcs[i] + src_off; 235 + 236 + dma_srcs[i] = dma_map_single(dev->dev, buf, len, 237 + DMA_TO_DEVICE); 238 + } 283 239 /* map with DMA_BIDIRECTIONAL to force writeback/invalidate */ 284 - dma_dest = dma_map_single(dev->dev, thread->dstbuf, 285 - test_buf_size, DMA_BIDIRECTIONAL); 240 + for (i = 0; i < dst_cnt; i++) { 241 + dma_dsts[i] = dma_map_single(dev->dev, thread->dsts[i], 242 + test_buf_size, 243 + DMA_BIDIRECTIONAL); 244 + } 286 245 287 - tx = dev->device_prep_dma_memcpy(chan, dma_dest + dst_off, 288 - dma_src, len, 289 - DMA_CTRL_ACK | DMA_COMPL_SKIP_DEST_UNMAP); 246 + if (thread->type == DMA_MEMCPY) 247 + tx = dev->device_prep_dma_memcpy(chan, 248 + dma_dsts[0] + dst_off, 249 + dma_srcs[0], len, 250 + flags); 251 + else if (thread->type == DMA_XOR) 252 + tx = dev->device_prep_dma_xor(chan, 253 + dma_dsts[0] + dst_off, 254 + dma_srcs, xor_sources, 255 + len, flags); 256 + 290 257 if (!tx) { 291 - dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); 292 - dma_unmap_single(dev->dev, dma_dest, 293 - test_buf_size, DMA_BIDIRECTIONAL); 258 + for (i = 0; i < src_cnt; i++) 259 + dma_unmap_single(dev->dev, dma_srcs[i], len, 260 + DMA_TO_DEVICE); 261 + for (i = 0; i < dst_cnt; i++) 262 + dma_unmap_single(dev->dev, dma_dsts[i], 263 + test_buf_size, 264 + DMA_BIDIRECTIONAL); 294 265 pr_warning("%s: #%u: prep error with src_off=0x%x " 295 266 "dst_off=0x%x len=0x%x\n", 296 267 thread_name, total_tests - 1, ··· 330 263 failed_tests++; 331 264 continue; 332 265 } 333 - dma_async_memcpy_issue_pending(chan); 266 + dma_async_issue_pending(chan); 334 267 335 268 do { 336 269 msleep(1); 337 - status = dma_async_memcpy_complete( 270 + status = dma_async_is_tx_complete( 338 271 chan, cookie, NULL, NULL); 339 272 } while (status == DMA_IN_PROGRESS); 340 273 ··· 345 278 continue; 346 279 } 347 280 /* Unmap by myself (see DMA_COMPL_SKIP_DEST_UNMAP above) */ 348 - dma_unmap_single(dev->dev, dma_dest, 349 - test_buf_size, DMA_BIDIRECTIONAL); 281 + for (i = 0; i < dst_cnt; i++) 282 + dma_unmap_single(dev->dev, dma_dsts[i], test_buf_size, 283 + DMA_BIDIRECTIONAL); 350 284 351 285 error_count = 0; 352 286 353 287 pr_debug("%s: verifying source buffer...\n", thread_name); 354 - error_count += dmatest_verify(thread->srcbuf, 0, src_off, 288 + error_count += dmatest_verify(thread->srcs, 0, src_off, 355 289 0, PATTERN_SRC, true); 356 - error_count += dmatest_verify(thread->srcbuf, src_off, 290 + error_count += dmatest_verify(thread->srcs, src_off, 357 291 src_off + len, src_off, 358 292 PATTERN_SRC | PATTERN_COPY, true); 359 - error_count += dmatest_verify(thread->srcbuf, src_off + len, 293 + error_count += dmatest_verify(thread->srcs, src_off + len, 360 294 test_buf_size, src_off + len, 361 295 PATTERN_SRC, true); 362 296 363 297 pr_debug("%s: verifying dest buffer...\n", 364 298 thread->task->comm); 365 - error_count += dmatest_verify(thread->dstbuf, 0, dst_off, 299 + error_count += dmatest_verify(thread->dsts, 0, dst_off, 366 300 0, PATTERN_DST, false); 367 - error_count += dmatest_verify(thread->dstbuf, dst_off, 301 + error_count += dmatest_verify(thread->dsts, dst_off, 368 302 dst_off + len, src_off, 369 303 PATTERN_SRC | PATTERN_COPY, false); 370 - error_count += dmatest_verify(thread->dstbuf, dst_off + len, 304 + error_count += dmatest_verify(thread->dsts, dst_off + len, 371 305 test_buf_size, dst_off + len, 372 306 PATTERN_DST, false); 373 307 ··· 387 319 } 388 320 389 321 ret = 0; 390 - kfree(thread->dstbuf); 322 + for (i = 0; thread->dsts[i]; i++) 323 + kfree(thread->dsts[i]); 391 324 err_dstbuf: 392 - kfree(thread->srcbuf); 325 + kfree(thread->dsts); 326 + err_dsts: 327 + for (i = 0; thread->srcs[i]; i++) 328 + kfree(thread->srcs[i]); 393 329 err_srcbuf: 330 + kfree(thread->srcs); 331 + err_srcs: 394 332 pr_notice("%s: terminating after %u tests, %u failures (status %d)\n", 395 333 thread_name, total_tests, failed_tests, ret); 396 334 return ret; ··· 418 344 kfree(dtc); 419 345 } 420 346 347 + static int dmatest_add_threads(struct dmatest_chan *dtc, enum dma_transaction_type type) 348 + { 349 + struct dmatest_thread *thread; 350 + struct dma_chan *chan = dtc->chan; 351 + char *op; 352 + unsigned int i; 353 + 354 + if (type == DMA_MEMCPY) 355 + op = "copy"; 356 + else if (type == DMA_XOR) 357 + op = "xor"; 358 + else 359 + return -EINVAL; 360 + 361 + for (i = 0; i < threads_per_chan; i++) { 362 + thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 363 + if (!thread) { 364 + pr_warning("dmatest: No memory for %s-%s%u\n", 365 + dma_chan_name(chan), op, i); 366 + 367 + break; 368 + } 369 + thread->chan = dtc->chan; 370 + thread->type = type; 371 + smp_wmb(); 372 + thread->task = kthread_run(dmatest_func, thread, "%s-%s%u", 373 + dma_chan_name(chan), op, i); 374 + if (IS_ERR(thread->task)) { 375 + pr_warning("dmatest: Failed to run thread %s-%s%u\n", 376 + dma_chan_name(chan), op, i); 377 + kfree(thread); 378 + break; 379 + } 380 + 381 + /* srcbuf and dstbuf are allocated by the thread itself */ 382 + 383 + list_add_tail(&thread->node, &dtc->threads); 384 + } 385 + 386 + return i; 387 + } 388 + 421 389 static int dmatest_add_channel(struct dma_chan *chan) 422 390 { 423 391 struct dmatest_chan *dtc; 424 - struct dmatest_thread *thread; 425 - unsigned int i; 392 + struct dma_device *dma_dev = chan->device; 393 + unsigned int thread_count = 0; 394 + unsigned int cnt; 426 395 427 396 dtc = kmalloc(sizeof(struct dmatest_chan), GFP_KERNEL); 428 397 if (!dtc) { ··· 476 359 dtc->chan = chan; 477 360 INIT_LIST_HEAD(&dtc->threads); 478 361 479 - for (i = 0; i < threads_per_chan; i++) { 480 - thread = kzalloc(sizeof(struct dmatest_thread), GFP_KERNEL); 481 - if (!thread) { 482 - pr_warning("dmatest: No memory for %s-test%u\n", 483 - dma_chan_name(chan), i); 484 - break; 485 - } 486 - thread->chan = dtc->chan; 487 - smp_wmb(); 488 - thread->task = kthread_run(dmatest_func, thread, "%s-test%u", 489 - dma_chan_name(chan), i); 490 - if (IS_ERR(thread->task)) { 491 - pr_warning("dmatest: Failed to run thread %s-test%u\n", 492 - dma_chan_name(chan), i); 493 - kfree(thread); 494 - break; 495 - } 496 - 497 - /* srcbuf and dstbuf are allocated by the thread itself */ 498 - 499 - list_add_tail(&thread->node, &dtc->threads); 362 + if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { 363 + cnt = dmatest_add_threads(dtc, DMA_MEMCPY); 364 + thread_count += cnt > 0 ?: 0; 365 + } 366 + if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 367 + cnt = dmatest_add_threads(dtc, DMA_XOR); 368 + thread_count += cnt > 0 ?: 0; 500 369 } 501 370 502 - pr_info("dmatest: Started %u threads using %s\n", i, dma_chan_name(chan)); 371 + pr_info("dmatest: Started %u threads using %s\n", 372 + thread_count, dma_chan_name(chan)); 503 373 504 374 list_add_tail(&dtc->node, &dmatest_channels); 505 375 nr_channels++;