x86/mm/pti: Introduce pti_finalize()

Introduce a new function to finalize the kernel mappings for the userspace
page-table after all ro/nx protections have been applied to the kernel
mappings.

Also move the call to pti_clone_kernel_text() to that function so that it
will run on 32 bit kernels too.

Signed-off-by: Joerg Roedel <[e-mail]>
Signed-off-by: Thomas Gleixner <[e-mail]>
Tested-by: Pavel Machek <[e-mail]>
Cc: "H . Peter Anvin" <[e-mail]>
Cc: [e-mail]
Cc: Linus Torvalds <[e-mail]>
Cc: Andy Lutomirski <[e-mail]>
Cc: Dave Hansen <[e-mail]>
Cc: Josh Poimboeuf <[e-mail]>
Cc: Juergen Gross <[e-mail]>
Cc: Peter Zijlstra <[e-mail]>
Cc: Borislav Petkov <[e-mail]>
Cc: Jiri Kosina <[e-mail]>
Cc: Boris Ostrovsky <[e-mail]>
Cc: Brian Gerst <[e-mail]>
Cc: David Laight <[e-mail]>
Cc: Denys Vlasenko <[e-mail]>
Cc: Eduardo Valentin <[e-mail]>
Cc: Greg KH <[e-mail]>
Cc: Will Deacon <[e-mail]>
Cc: [e-mail]
Cc: [e-mail]
Cc: [e-mail]
Cc: [e-mail]
Cc: Andrea Arcangeli <[e-mail]>
Cc: Waiman Long <[e-mail]>
Cc: "David H . Gutteridge" <[e-mail]>
Cc: [e-mail]
Link: https://lkml.kernel.org/r/[e-mail]

arch/x86/include/asm/pti.h

···
 #ifdef CONFIG_PAGE_TABLE_ISOLATION
 extern void pti_init(void);
 extern void pti_check_boottime_disable(void);
-extern void pti_clone_kernel_text(void);
+extern void pti_finalize(void);
 #else
 static inline void pti_check_boottime_disable(void) { }
-static inline void pti_clone_kernel_text(void) { }
 #endif
 
 #endif /* __ASSEMBLY__ */

arch/x86/mm/init_64.c

···
 			(unsigned long) __va(__pa_symbol(_sdata)));
 
 	debug_checkwx();
-
-	/*
-	 * Do this after all of the manipulation of the
-	 * kernel text page tables are complete.
-	 */
-	pti_clone_kernel_text();
 }
 
 int kern_addr_valid(unsigned long addr)

arch/x86/mm/pti.c

···
  * For some configurations, map all of kernel text into the user page
  * tables.  This reduces TLB misses, especially on non-PCID systems.
  */
-void pti_clone_kernel_text(void)
+static void pti_clone_kernel_text(void)
 {
 	/*
 	 * rodata is part of the kernel image and is normally
···
 	pti_clone_entry_text();
 	pti_setup_espfix64();
 	pti_setup_vsyscall();
+}
+
+/*
+ * Finalize the kernel mappings in the userspace page-table.
+ */
+void pti_finalize(void)
+{
+	/*
+	 * Do this after all of the manipulation of the
+	 * kernel text page tables are complete.
+	 */
+	pti_clone_kernel_text();
 }

include/linux/pti.h

···
 #include <asm/pti.h>
 #else
 static inline void pti_init(void) { }
+static inline void pti_finalize(void) { }
 #endif
 
 #endif

init/main.c

···
 	jump_label_invalidate_initmem();
 	free_initmem();
 	mark_readonly();
+
+	/*
+	 * Kernel mappings are now finalized - update the userspace page-table
+	 * to finalize PTI.
+	 */
+	pti_finalize();
+
 	system_state = SYSTEM_RUNNING;
 	numa_default_policy();