ARM: pgtable: add pud-level code

Add pud_offset() et.al. between the pgd and pmd code in preparation of
using pgtable-nopud.h rather than 4level-fixup.h.

This incorporates a fix from Jamie Iles <[e-mail]> for
uaccess_with_memcpy.c.

Signed-off-by: Russell King <[e-mail]>

arch/arm/include/asm/pgtable.h

···
 #define pgd_present(pgd)	(1)
 #define pgd_clear(pgdp)		do { } while (0)
 #define set_pgd(pgd,pgdp)	do { } while (0)
+#define set_pud(pud,pudp)	do { } while (0)
 
 
 /* Find an entry in the second-level page table.. */

arch/arm/lib/uaccess_with_memcpy.c

···
 	pgd_t *pgd;
 	pmd_t *pmd;
 	pte_t *pte;
+	pud_t *pud;
 	spinlock_t *ptl;
 
 	pgd = pgd_offset(current->mm, addr);
 	if (unlikely(pgd_none(*pgd) || pgd_bad(*pgd)))
 		return 0;
 
-	pmd = pmd_offset(pgd, addr);
+	pud = pud_offset(pgd, addr);
+	if (unlikely(pud_none(*pud) || pud_bad(*pud)))
+		return 0;
+
+	pmd = pmd_offset(pud, addr);
 	if (unlikely(pmd_none(*pmd) || pmd_bad(*pmd)))
 		return 0;
 

arch/arm/mm/dma-mapping.c

···
 {
 	int ret = 0;
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	int i = 0;
···
 
 	do {
 		pgd = pgd_offset(&init_mm, base);
-		pmd = pmd_alloc(&init_mm, pgd, base);
+
+		pud = pud_alloc(&init_mm, pgd, base);
+		if (!pud) {
+			printk(KERN_ERR "%s: no pud tables\n", __func__);
+			ret = -ENOMEM;
+			break;
+		}
+
+		pmd = pmd_alloc(&init_mm, pud, base);
 		if (!pmd) {
 			printk(KERN_ERR "%s: no pmd tables\n", __func__);
 			ret = -ENOMEM;

arch/arm/mm/fault-armv.c

···
 {
 	spinlock_t *ptl;
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pte_t *pte;
 	int ret;
···
 	if (pgd_none_or_clear_bad(pgd))
 		return 0;
 
-	pmd = pmd_offset(pgd, address);
+	pud = pud_offset(pgd, address);
+	if (pud_none_or_clear_bad(pud))
+		return 0;
+
+	pmd = pmd_offset(pud, address);
 	if (pmd_none_or_clear_bad(pmd))
 		return 0;
 

arch/arm/mm/fault.c

···
 			addr, (long long)pgd_val(*pgd));
 
 	do {
+		pud_t *pud;
 		pmd_t *pmd;
 		pte_t *pte;
 
···
 			break;
 		}
 
-		pmd = pmd_offset(pgd, addr);
+		pud = pud_offset(pgd, addr);
+		if (PTRS_PER_PUD != 1)
+			printk(", *pud=%08lx", pud_val(*pud));
+
+		if (pud_none(*pud))
+			break;
+
+		if (pud_bad(*pud)) {
+			printk("(bad)");
+			break;
+		}
+
+		pmd = pmd_offset(pud, addr);
 		if (PTRS_PER_PMD != 1)
 			printk(", *pmd=%08llx", (long long)pmd_val(*pmd));
 
···
 {
 	unsigned int index;
 	pgd_t *pgd, *pgd_k;
+	pud_t *pud, *pud_k;
 	pmd_t *pmd, *pmd_k;
 
 	if (addr < TASK_SIZE)
···
 
 	if (pgd_none(*pgd_k))
 		goto bad_area;
-
 	if (!pgd_present(*pgd))
 		set_pgd(pgd, *pgd_k);
 
-	pmd_k = pmd_offset(pgd_k, addr);
-	pmd   = pmd_offset(pgd, addr);
+	pud = pud_offset(pgd, addr);
+	pud_k = pud_offset(pgd_k, addr);
+
+	if (pud_none(*pud_k))
+		goto bad_area;
+	if (!pud_present(*pud))
+		set_pud(pud, *pud_k);
+
+	pmd = pmd_offset(pud, addr);
+	pmd_k = pmd_offset(pud_k, addr);
 
 	/*
 	 * On ARM one Linux PGD entry contains two hardware entries (see page

arch/arm/mm/idmap.c

···
 #include <asm/pgalloc.h>
 #include <asm/pgtable.h>
 
-static void idmap_add_pmd(pgd_t *pgd, unsigned long addr, unsigned long end,
+static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
 	unsigned long prot)
 {
-	pmd_t *pmd = pmd_offset(pgd, addr);
+	pmd_t *pmd = pmd_offset(pud, addr);
 
 	addr = (addr & PMD_MASK) | prot;
 	pmd[0] = __pmd(addr);
 	addr += SECTION_SIZE;
 	pmd[1] = __pmd(addr);
 	flush_pmd_entry(pmd);
+}
+
+static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+	unsigned long prot)
+{
+	pud_t *pud = pud_offset(pgd, addr);
+	unsigned long next;
+
+	do {
+		next = pud_addr_end(addr, end);
+		idmap_add_pmd(pud, addr, next, prot);
+	} while (pud++, addr = next, addr != end);
 }
 
 void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end)
···
 	pgd += pgd_index(addr);
 	do {
 		next = pgd_addr_end(addr, end);
-		idmap_add_pmd(pgd, addr, next, prot);
+		idmap_add_pud(pgd, addr, next, prot);
 	} while (pgd++, addr = next, addr != end);
 }
 
 #ifdef CONFIG_SMP
-static void idmap_del_pmd(pgd_t *pgd, unsigned long addr, unsigned long end)
+static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end)
 {
-	pmd_t *pmd = pmd_offset(pgd, addr);
+	pmd_t *pmd = pmd_offset(pud, addr);
 	pmd_clear(pmd);
+}
+
+static void idmap_del_pud(pgd_t *pgd, unsigned long addr, unsigned long end)
+{
+	pud_t *pud = pud_offset(pgd, addr);
+	unsigned long next;
+
+	do {
+		next = pud_addr_end(addr, end);
+		idmap_del_pmd(pud, addr, next);
+	} while (pud++, addr = next, addr != end);
 }
 
 void identity_mapping_del(pgd_t *pgd, unsigned long addr, unsigned long end)
···
 	pgd += pgd_index(addr);
 	do {
 		next = pgd_addr_end(addr, end);
-		idmap_del_pmd(pgd, addr, next);
+		idmap_del_pud(pgd, addr, next);
 	} while (pgd++, addr = next, addr != end);
 }
 #endif

arch/arm/mm/mm.h

···
 
 static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt)
 {
-	return pmd_offset(pgd, virt);
+	return pmd_offset(pud_offset(pgd, virt), virt);
 }
 
 static inline pmd_t *pmd_off_k(unsigned long virt)

arch/arm/mm/mmu.c

···
 	} while (pte++, addr += PAGE_SIZE, addr != end);
 }
 
-static void __init alloc_init_section(pgd_t *pgd, unsigned long addr,
+static void __init alloc_init_section(pud_t *pud, unsigned long addr,
 				      unsigned long end, phys_addr_t phys,
 				      const struct mem_type *type)
 {
-	pmd_t *pmd = pmd_offset(pgd, addr);
+	pmd_t *pmd = pmd_offset(pud, addr);
 
 	/*
 	 * Try a section mapping - end, addr and phys must all be aligned
···
 		 */
 		alloc_init_pte(pmd, addr, end, __phys_to_pfn(phys), type);
 	}
+}
+
+static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
+	unsigned long phys, const struct mem_type *type)
+{
+	pud_t *pud = pud_offset(pgd, addr);
+	unsigned long next;
+
+	do {
+		next = pud_addr_end(addr, end);
+		alloc_init_section(pud, addr, next, phys, type);
+		phys += next - addr;
+	} while (pud++, addr = next, addr != end);
 }
 
 static void __init create_36bit_mapping(struct map_desc *md,
···
 	pgd = pgd_offset_k(addr);
 	end = addr + length;
 	do {
-		pmd_t *pmd = pmd_offset(pgd, addr);
+		pud_t *pud = pud_offset(pgd, addr);
+		pmd_t *pmd = pmd_offset(pud, addr);
 		int i;
 
 		for (i = 0; i < 16; i++)
···
 	do {
 		unsigned long next = pgd_addr_end(addr, end);
 
-		alloc_init_section(pgd, addr, next, phys, type);
+		alloc_init_pud(pgd, addr, next, phys, type);
 
 		phys += next - addr;
 		addr = next;

arch/arm/mm/pgd.c

···
 pgd_t *pgd_alloc(struct mm_struct *mm)
 {
 	pgd_t *new_pgd, *init_pgd;
+	pud_t *new_pud, *init_pud;
 	pmd_t *new_pmd, *init_pmd;
 	pte_t *new_pte, *init_pte;
 
···
 		 * On ARM, first page must always be allocated since it
 		 * contains the machine vectors.
 		 */
-		new_pmd = pmd_alloc(mm, new_pgd, 0);
+		new_pud = pud_alloc(mm, new_pgd, 0);
+		if (!new_pud)
+			goto no_pud;
+
+		new_pmd = pmd_alloc(mm, new_pud, 0);
 		if (!new_pmd)
 			goto no_pmd;
 
···
 		if (!new_pte)
 			goto no_pte;
 
-		init_pmd = pmd_offset(init_pgd, 0);
+		init_pud = pud_offset(init_pgd, 0);
+		init_pmd = pmd_offset(init_pud, 0);
 		init_pte = pte_offset_map(init_pmd, 0);
 		set_pte_ext(new_pte, *init_pte, 0);
 		pte_unmap(init_pte);
···
 no_pte:
 	pmd_free(mm, new_pmd);
 no_pmd:
+	pud_free(mm, new_pud);
+no_pud:
 	free_pages((unsigned long)new_pgd, 2);
 no_pgd:
 	return NULL;
···
 void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
 {
 	pgd_t *pgd;
+	pud_t *pud;
 	pmd_t *pmd;
 	pgtable_t pte;
 
···
 	if (pgd_none_or_clear_bad(pgd))
 		goto no_pgd;
 
-	pmd = pmd_offset(pgd, 0);
+	pud = pud_offset(pgd, 0);
+	if (pud_none_or_clear_bad(pud))
+		goto no_pud;
+
+	pmd = pmd_offset(pud, 0);
 	if (pmd_none_or_clear_bad(pmd))
 		goto no_pmd;
 
···
 	pmd_clear(pmd);
 	pte_free(mm, pte);
 no_pmd:
-	pgd_clear(pgd);
+	pud_clear(pud);
 	pmd_free(mm, pmd);
+no_pud:
+	pgd_clear(pgd);
+	pud_free(mm, pud);
 no_pgd:
 	free_pages((unsigned long) pgd_base, 2);
 }