instrumented.h: add KMSAN support

To avoid false positives, KMSAN needs to unpoison the data copied from the
userspace. To detect infoleaks - check the memory buffer passed to
copy_to_user().

Link: https://lkml.kernel.org/r/[e-mail]
Signed-off-by: Alexander Potapenko <[e-mail]>
Reviewed-by: Marco Elver <[e-mail]>
Cc: Alexander Viro <[e-mail]>
Cc: Alexei Starovoitov <[e-mail]>
Cc: Andrey Konovalov <[e-mail]>
Cc: Andrey Konovalov <[e-mail]>
Cc: Andy Lutomirski <[e-mail]>
Cc: Arnd Bergmann <[e-mail]>
Cc: Borislav Petkov <[e-mail]>
Cc: Christoph Hellwig <[e-mail]>
Cc: Christoph Lameter <[e-mail]>
Cc: David Rientjes <[e-mail]>
Cc: Dmitry Vyukov <[e-mail]>
Cc: Eric Biggers <[e-mail]>
Cc: Eric Biggers <[e-mail]>
Cc: Eric Dumazet <[e-mail]>
Cc: Greg Kroah-Hartman <[e-mail]>
Cc: Herbert Xu <[e-mail]>
Cc: Ilya Leoshkevich <[e-mail]>
Cc: Ingo Molnar <[e-mail]>
Cc: Jens Axboe <[e-mail]>
Cc: Joonsoo Kim <[e-mail]>
Cc: Kees Cook <[e-mail]>
Cc: Mark Rutland <[e-mail]>
Cc: Matthew Wilcox <[e-mail]>
Cc: Michael S. Tsirkin <[e-mail]>
Cc: Pekka Enberg <[e-mail]>
Cc: Peter Zijlstra <[e-mail]>
Cc: Petr Mladek <[e-mail]>
Cc: Stephen Rothwell <[e-mail]>
Cc: Steven Rostedt <[e-mail]>
Cc: Thomas Gleixner <[e-mail]>
Cc: Vasily Gorbik <[e-mail]>
Cc: Vegard Nossum <[e-mail]>
Cc: Vlastimil Babka <[e-mail]>
Signed-off-by: Andrew Morton <[e-mail]>

include/linux/instrumented.h

···
 
 /*
  * This header provides generic wrappers for memory access instrumentation that
- * the compiler cannot emit for: KASAN, KCSAN.
+ * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
  */
 #ifndef _LINUX_INSTRUMENTED_H
 #define _LINUX_INSTRUMENTED_H
···
 #include <linux/compiler.h>
 #include <linux/kasan-checks.h>
 #include <linux/kcsan-checks.h>
+#include <linux/kmsan-checks.h>
 #include <linux/types.h>
 
 /**
···
 {
 	kasan_check_read(from, n);
 	kcsan_check_read(from, n);
+	kmsan_copy_to_user(to, from, n, 0);
 }
 
 /**
···
 instrument_copy_from_user_after(const void *to, const void __user *from,
 				unsigned long n, unsigned long left)
 {
+	kmsan_unpoison_memory(to, n - left);
 }
 
 /**
···
  *
  * @to destination variable, may not be address-taken
  */
-#define instrument_get_user(to)                         \
-({                                                      \
+#define instrument_get_user(to)				\
+({							\
+	u64 __tmp = (u64)(to);				\
+	kmsan_unpoison_memory(&__tmp, sizeof(__tmp));	\
+	to = __tmp;					\
 })
+
 
 /**
  * instrument_put_user() - add instrumentation to put_user()-like macros
···
  * @ptr userspace pointer to copy to
  * @size number of bytes to copy
  */
-#define instrument_put_user(from, ptr, size)                    \
-({                                                              \
+#define instrument_put_user(from, ptr, size)			\
+({								\
+	kmsan_copy_to_user(ptr, &from, sizeof(from), 0);	\
 })
 
 #endif /* _LINUX_INSTRUMENTED_H */

include/linux/kmsan-checks.h

···
  */
 void kmsan_check_memory(const void *address, size_t size);
 
+/**
+ * kmsan_copy_to_user() - Notify KMSAN about a data transfer to userspace.
+ * @to:      destination address in the userspace.
+ * @from:    source address in the kernel.
+ * @to_copy: number of bytes to copy.
+ * @left:    number of bytes not copied.
+ *
+ * If this is a real userspace data transfer, KMSAN checks the bytes that were
+ * actually copied to ensure there was no information leak. If @to belongs to
+ * the kernel space (which is possible for compat syscalls), KMSAN just copies
+ * the metadata.
+ */
+void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
+			size_t left);
+
 #else
 
 static inline void kmsan_poison_memory(const void *address, size_t size,
···
 {
 }
 static inline void kmsan_check_memory(const void *address, size_t size)
+{
+}
+static inline void kmsan_copy_to_user(void __user *to, const void *from,
+				      size_t to_copy, size_t left)
 {
 }
 

mm/kmsan/hooks.c

···
 	kmsan_leave_runtime();
 }
 
+void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
+			size_t left)
+{
+	unsigned long ua_flags;
+
+	if (!kmsan_enabled || kmsan_in_runtime())
+		return;
+	/*
+	 * At this point we've copied the memory already. It's hard to check it
+	 * before copying, as the size of actually copied buffer is unknown.
+	 */
+
+	/* copy_to_user() may copy zero bytes. No need to check. */
+	if (!to_copy)
+		return;
+	/* Or maybe copy_to_user() failed to copy anything. */
+	if (to_copy <= left)
+		return;
+
+	ua_flags = user_access_save();
+	if ((u64)to < TASK_SIZE) {
+		/* This is a user memory access, check it. */
+		kmsan_internal_check_memory((void *)from, to_copy - left, to,
+					    REASON_COPY_TO_USER);
+	} else {
+		/* Otherwise this is a kernel memory access. This happens when a
+		 * compat syscall passes an argument allocated on the kernel
+		 * stack to a real syscall.
+		 * Don't check anything, just copy the shadow of the copied
+		 * bytes.
+		 */
+		kmsan_internal_memmove_metadata((void *)to, (void *)from,
+						to_copy - left);
+	}
+	user_access_restore(ua_flags);
+}
+EXPORT_SYMBOL(kmsan_copy_to_user);
+
 /* Functions from kmsan-checks.h follow. */
 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
 {