tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * x86_64 accelerated implementation of NH
4 *
5 * Copyright 2018 Google LLC
6 */
7
8#include <asm/fpu/api.h>
9#include <linux/static_call.h>
10
11static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sse2);
12static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_avx2);
13
14asmlinkage void nh_sse2(const u32 *key, const u8 *message, size_t message_len,
15 __le64 hash[NH_NUM_PASSES]);
16asmlinkage void nh_avx2(const u32 *key, const u8 *message, size_t message_len,
17 __le64 hash[NH_NUM_PASSES]);
18
19static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
20 __le64 hash[NH_NUM_PASSES])
21{
22 if (message_len >= 64 && static_branch_likely(&have_sse2) &&
23 irq_fpu_usable()) {
24 kernel_fpu_begin();
25 if (static_branch_likely(&have_avx2))
26 nh_avx2(key, message, message_len, hash);
27 else
28 nh_sse2(key, message, message_len, hash);
29 kernel_fpu_end();
30 return true;
31 }
32 return false;
33}
34
35#define nh_mod_init_arch nh_mod_init_arch
36static void nh_mod_init_arch(void)
37{
38 if (boot_cpu_has(X86_FEATURE_XMM2)) {
39 static_branch_enable(&have_sse2);
40 if (boot_cpu_has(X86_FEATURE_AVX2) &&
41 cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
42 NULL))
43 static_branch_enable(&have_avx2);
44 }
45}