diff options
Diffstat (limited to 'arch/arm/include/asm')
-rw-r--r-- | arch/arm/include/asm/hugetlb-2level.h | 121 | ||||
-rw-r--r-- | arch/arm/include/asm/hugetlb.h | 4 | ||||
-rw-r--r-- | arch/arm/include/asm/pgtable-2level.h | 102 | ||||
-rw-r--r-- | arch/arm/include/asm/pgtable.h | 2 | ||||
-rw-r--r-- | arch/arm/include/asm/tlb.h | 10 |
5 files changed, 237 insertions, 2 deletions
diff --git a/arch/arm/include/asm/hugetlb-2level.h b/arch/arm/include/asm/hugetlb-2level.h new file mode 100644 index 000000000000..eb0e8b684a00 --- /dev/null +++ b/arch/arm/include/asm/hugetlb-2level.h @@ -0,0 +1,121 @@ +/* + * arch/arm/include/asm/hugetlb-2level.h + * + * Copyright (C) 2012 ARM Ltd. + * + * Based on arch/x86/include/asm/hugetlb.h and Bill Carson's patches + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#ifndef _ASM_ARM_HUGETLB_2LEVEL_H +#define _ASM_ARM_HUGETLB_2LEVEL_H + + +static inline pte_t huge_ptep_get(pte_t *ptep) +{ + pmd_t pmd = *((pmd_t *)ptep); + pte_t retval; + + if (!pmd_val(pmd)) + return __pte(0); + + retval = __pte((pteval_t) (pmd_val(pmd) & HPAGE_MASK) + | arm_hugepteprotval); + + if (pmd_exec(pmd)) + retval = pte_mkexec(retval); + else + retval = pte_mknexec(retval); + + if (pmd_young(pmd)) + retval = pte_mkyoung(retval); + else + retval = pte_mkold(retval); + + if (pmd_dirty(pmd)) + retval = pte_mkdirty(retval); + else + retval = pte_mkclean(retval); + + if (pmd_write(pmd)) + retval = pte_mkwrite(retval); + else + retval = pte_wrprotect(retval); + + return retval; +} + +static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t pte) +{ + pmdval_t pmdval = (pmdval_t) pte_val(pte); + pmd_t *pmdp = (pmd_t *) ptep; + + /* take the target address bits from the pte only */ + pmdval &= HPAGE_MASK; + + /* + * now use pmd_modify to translate the permission bits from the pte + * and set the memory type information. + */ + pmdval = pmd_val(pmd_modify(__pmd(pmdval), __pgprot(pte_val(pte)))); + + __sync_icache_dcache(pte); + + set_pmd_at(mm, addr, pmdp, __pmd(pmdval)); +} + +static inline pte_t pte_mkhuge(pte_t pte) { return pte; } + +static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + pmd_clear(pmdp); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); +} + +static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *) ptep; + set_pmd_at(mm, addr, pmdp, pmd_wrprotect(*pmdp)); +} + + +static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm, + unsigned long addr, pte_t *ptep) +{ + pmd_t *pmdp = (pmd_t *)ptep; + pte_t pte = huge_ptep_get(ptep); + pmd_clear(pmdp); + + return pte; +} + +static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, + pte_t pte, int dirty) +{ + int changed = !pte_same(huge_ptep_get(ptep), pte); + if (changed) { + set_huge_pte_at(vma->vm_mm, addr, ptep, pte); + flush_tlb_range(vma, addr, addr + HPAGE_SIZE); + } + + return changed; +} + +#endif /* _ASM_ARM_HUGETLB_2LEVEL_H */ diff --git a/arch/arm/include/asm/hugetlb.h b/arch/arm/include/asm/hugetlb.h index 1f1b1cd112f3..6c0372aa95ab 100644 --- a/arch/arm/include/asm/hugetlb.h +++ b/arch/arm/include/asm/hugetlb.h @@ -25,7 +25,11 @@ #include <asm/page.h> #include <asm-generic/hugetlb.h> +#ifdef CONFIG_ARM_LPAE #include <asm/hugetlb-3level.h> +#else +#include <asm/hugetlb-2level.h> +#endif static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h index 72410f92e30e..7423ecf31808 100644 --- a/arch/arm/include/asm/pgtable-2level.h +++ b/arch/arm/include/asm/pgtable-2level.h @@ -183,6 +183,108 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) + +#ifdef CONFIG_SYS_SUPPORTS_HUGETLBFS + +/* + * now follows some of the definitions to allow huge page support, we can't put + * these in the hugetlb source files as they are also required for transparent + * hugepage support. + */ + +#define HPAGE_SHIFT PMD_SHIFT +#define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) +#define HPAGE_MASK (~(HPAGE_SIZE - 1)) +#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) + +#define HUGE_LINUX_PTE_COUNT (PAGE_OFFSET >> HPAGE_SHIFT) +#define HUGE_LINUX_PTE_SIZE (HUGE_LINUX_PTE_COUNT * sizeof(pte_t *)) +#define HUGE_LINUX_PTE_INDEX(addr) (addr >> HPAGE_SHIFT) + +/* + * We re-purpose the following domain bits in the section descriptor + */ +#define PMD_DOMAIN_MASK (_AT(pmdval_t, 0xF) << 5) +#define PMD_DSECT_DIRTY (_AT(pmdval_t, 1) << 5) +#define PMD_DSECT_AF (_AT(pmdval_t, 1) << 6) + +#define PMD_BIT_FUNC(fn,op) \ +static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } + +static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, + pmd_t *pmdp, pmd_t pmd) +{ + /* + * we can sometimes be passed a pmd pointing to a level 2 descriptor + * from collapse_huge_page. + */ + if ((pmd_val(pmd) & PMD_TYPE_MASK) == PMD_TYPE_TABLE) { + pmdp[0] = __pmd(pmd_val(pmd)); + pmdp[1] = __pmd(pmd_val(pmd) + 256 * sizeof(pte_t)); + } else { + pmdp[0] = __pmd(pmd_val(pmd)); /* first 1M section */ + pmdp[1] = __pmd(pmd_val(pmd) + SECTION_SIZE); /* second 1M section */ + } + + flush_pmd_entry(pmdp); +} + +extern pmdval_t arm_hugepmdprotval; +extern pteval_t arm_hugepteprotval; + +#define pmd_mkhuge(pmd) (__pmd((pmd_val(pmd) & ~PMD_TYPE_MASK) | PMD_TYPE_SECT)) + +PMD_BIT_FUNC(mkold, &= ~PMD_DSECT_AF); +PMD_BIT_FUNC(mkdirty, |= PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkclean, &= ~PMD_DSECT_DIRTY); +PMD_BIT_FUNC(mkyoung, |= PMD_DSECT_AF); +PMD_BIT_FUNC(mkwrite, |= PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(wrprotect, &= ~PMD_SECT_AP_WRITE); +PMD_BIT_FUNC(mknotpresent, &= ~PMD_TYPE_MASK); +PMD_BIT_FUNC(mkexec, &= ~PMD_SECT_XN); +PMD_BIT_FUNC(mknexec, |= PMD_SECT_XN); + +#define pmd_young(pmd) (pmd_val(pmd) & PMD_DSECT_AF) +#define pmd_write(pmd) (pmd_val(pmd) & PMD_SECT_AP_WRITE) +#define pmd_exec(pmd) (!(pmd_val(pmd) & PMD_SECT_XN)) +#define pmd_dirty(pmd) (pmd_val(pmd) & PMD_DSECT_DIRTY) + +#define __HAVE_ARCH_PMD_WRITE + +#define pmd_modify(pmd, prot) \ +({ \ + pmd_t pmdret = __pmd((pmd_val(pmd) & (PMD_MASK | PMD_DOMAIN_MASK)) \ + | arm_hugepmdprotval); \ + pgprot_t inprot = prot; \ + pte_t newprot = __pte(pgprot_val(inprot)); \ + \ + if (pte_dirty(newprot)) \ + pmdret = pmd_mkdirty(pmdret); \ + else \ + pmdret = pmd_mkclean(pmdret); \ + \ + if (pte_exec(newprot)) \ + pmdret = pmd_mkexec(pmdret); \ + else \ + pmdret = pmd_mknexec(pmdret); \ + \ + if (pte_write(newprot)) \ + pmdret = pmd_mkwrite(pmdret); \ + else \ + pmdret = pmd_wrprotect(pmdret); \ + \ + if (pte_young(newprot)) \ + pmdret = pmd_mkyoung(pmdret); \ + else \ + pmdret = pmd_mkold(pmdret); \ + \ + pmdret; \ +}) + +#else +#define HPAGE_SIZE 0 +#endif /* CONFIG_SYS_SUPPORTS_HUGETLBFS */ + #endif /* __ASSEMBLY__ */ #endif /* _ASM_PGTABLE_2LEVEL_H */ diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index 2ed433d218d8..8e124ca284ba 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -253,6 +253,8 @@ PTE_BIT_FUNC(mkclean, &= ~L_PTE_DIRTY); PTE_BIT_FUNC(mkdirty, |= L_PTE_DIRTY); PTE_BIT_FUNC(mkold, &= ~L_PTE_YOUNG); PTE_BIT_FUNC(mkyoung, |= L_PTE_YOUNG); +PTE_BIT_FUNC(mkexec, &= ~L_PTE_XN); +PTE_BIT_FUNC(mknexec, |= L_PTE_XN); static inline pte_t pte_mkspecial(pte_t pte) { return pte; } diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index bdc62da2f212..0fc2d9d35031 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -92,10 +92,16 @@ static inline void tlb_flush(struct mmu_gather *tlb) static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) { if (!tlb->fullmm) { + unsigned long size = PAGE_SIZE; + if (addr < tlb->range_start) tlb->range_start = addr; - if (addr + PAGE_SIZE > tlb->range_end) - tlb->range_end = addr + PAGE_SIZE; + + if (tlb->vma && is_vm_hugetlb_page(tlb->vma)) + size = HPAGE_SIZE; + + if (addr + size > tlb->range_end) + tlb->range_end = addr + size; } } |