diff options
author | John Stultz <john.stultz@linaro.org> | 2011-11-11 15:29:25 -0800 |
---|---|---|
committer | John Stultz <john.stultz@linaro.org> | 2011-11-11 15:29:25 -0800 |
commit | c3e5127312f97482a6a09c0df602c726675eff5b (patch) | |
tree | 9082d8201a1daac5ff019ef50322d551711eb250 | |
parent | 5e537f041bd05cddee0d2a2fb3013ee642064cc6 (diff) | |
parent | f171dfebbdf1de4b6ca267285739e190f95f84cc (diff) |
Merge branch 'upstream/linaro-3.1' into linaro-android-3.1-agreen-rebaselinux-linaro-3.1-2011.11-0-android-0
52 files changed, 2116 insertions, 733 deletions
diff --git a/Documentation/devicetree/bindings/arm/l2cc.txt b/Documentation/devicetree/bindings/arm/l2cc.txt new file mode 100644 index 00000000000..7ca52161e7a --- /dev/null +++ b/Documentation/devicetree/bindings/arm/l2cc.txt @@ -0,0 +1,44 @@ +* ARM L2 Cache Controller + +ARM cores often have a separate level 2 cache controller. There are various +implementations of the L2 cache controller with compatible programming models. +The ARM L2 cache representation in the device tree should be done as follows: + +Required properties: + +- compatible : should be one of: + "arm,pl310-cache" + "arm,l220-cache" + "arm,l210-cache" +- cache-unified : Specifies the cache is a unified cache. +- cache-level : Should be set to 2 for a level 2 cache. +- reg : Physical base address and size of cache controller's memory mapped + registers. + +Optional properties: + +- arm,data-latency : Cycles of latency for Data RAM accesses. Specifies 3 cells of + read, write and setup latencies. Minimum valid values are 1. Controllers + without setup latency control should use a value of 0. +- arm,tag-latency : Cycles of latency for Tag RAM accesses. Specifies 3 cells of + read, write and setup latencies. Controllers without setup latency control + should use 0. Controllers without separate read and write Tag RAM latency + values should only use the first cell. +- arm,dirty-latency : Cycles of latency for Dirty RAMs. This is a single cell. +- arm,filter-ranges : <start length> Starting address and length of window to + filter. Addresses in the filter window are directed to the M1 port. Other + addresses will go to the M0 port. +- interrupts : 1 combined interrupt. + +Example: + +L2: cache-controller { + compatible = "arm,pl310-cache"; + reg = <0xfff12000 0x1000>; + arm,data-latency = <1 1 1>; + arm,tag-latency = <2 2 2>; + arm,filter-latency = <0x80000000 0x8000000>; + cache-unified; + cache-level = <2>; + interrupts = <45>; +}; diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 0874b160e59..b67d130e189 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1912,7 +1912,7 @@ endchoice config XIP_KERNEL bool "Kernel Execute-In-Place from ROM" - depends on !ZBOOT_ROM + depends on !ZBOOT_ROM && !ARM_LPAE help Execute-In-Place allows the kernel to run from non-volatile storage directly addressable by the CPU, such as NOR flash. This saves RAM diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index e45767851a4..fa1ebfb2129 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -658,6 +658,7 @@ __armv7_mmu_cache_on: mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer mcrne p15, 0, r1, c3, c0, 0 @ load domain access control #endif + mcr p15, 0, r0, c7, c5, 4 @ ISB mcr p15, 0, r0, c1, c0, 0 @ load control register mrc p15, 0, r0, c1, c0, 0 @ and read it back mov r0, #0 diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 29035e86a59..b6e65dedfd7 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -187,6 +187,17 @@ #endif /* + * Instruction barrier + */ + .macro instr_sync +#if __LINUX_ARM_ARCH__ >= 7 + isb +#elif __LINUX_ARM_ARCH__ == 6 + mcr p15, 0, r0, c7, c5, 4 +#endif + .endm + +/* * SMP data memory barrier */ .macro smp_dmb mode diff --git a/arch/arm/include/asm/dma-mapping.h b/arch/arm/include/asm/dma-mapping.h index 7f27fab9d40..cb3b7c981c4 100644 --- a/arch/arm/include/asm/dma-mapping.h +++ b/arch/arm/include/asm/dma-mapping.h @@ -32,7 +32,7 @@ static inline unsigned long dma_to_pfn(struct device *dev, dma_addr_t addr) static inline void *dma_to_virt(struct device *dev, dma_addr_t addr) { - return (void *)__bus_to_virt(addr); + return (void *)__bus_to_virt((unsigned long)addr); } static inline dma_addr_t virt_to_dma(struct device *dev, void *addr) diff --git a/arch/arm/include/asm/hardware/cache-l2x0.h b/arch/arm/include/asm/hardware/cache-l2x0.h index 99a6ed7e1bf..434edccdf7f 100644 --- a/arch/arm/include/asm/hardware/cache-l2x0.h +++ b/arch/arm/include/asm/hardware/cache-l2x0.h @@ -52,6 +52,8 @@ #define L2X0_LOCKDOWN_WAY_D_BASE 0x900 #define L2X0_LOCKDOWN_WAY_I_BASE 0x904 #define L2X0_LOCKDOWN_STRIDE 0x08 +#define L2X0_ADDR_FILTER_START 0xC00 +#define L2X0_ADDR_FILTER_END 0xC04 #define L2X0_TEST_OPERATION 0xF00 #define L2X0_LINE_DATA 0xF10 #define L2X0_LINE_TAG 0xF30 @@ -65,8 +67,23 @@ #define L2X0_CACHE_ID_PART_MASK (0xf << 6) #define L2X0_CACHE_ID_PART_L210 (1 << 6) #define L2X0_CACHE_ID_PART_L310 (3 << 6) +#define L2X0_CACHE_ID_RTL_MASK 0x3f +#define L2X0_CACHE_ID_RTL_R0P0 0x0 +#define L2X0_CACHE_ID_RTL_R1P0 0x2 +#define L2X0_CACHE_ID_RTL_R2P0 0x4 +#define L2X0_CACHE_ID_RTL_R3P0 0x5 +#define L2X0_CACHE_ID_RTL_R3P1 0x6 +#define L2X0_CACHE_ID_RTL_R3P2 0x8 #define L2X0_AUX_CTRL_MASK 0xc0000fff +#define L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT 0 +#define L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK 0x7 +#define L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT 3 +#define L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK (0x7 << 3) +#define L2X0_AUX_CTRL_TAG_LATENCY_SHIFT 6 +#define L2X0_AUX_CTRL_TAG_LATENCY_MASK (0x7 << 6) +#define L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT 9 +#define L2X0_AUX_CTRL_DIRTY_LATENCY_MASK (0x7 << 9) #define L2X0_AUX_CTRL_ASSOCIATIVITY_SHIFT 16 #define L2X0_AUX_CTRL_WAY_SIZE_SHIFT 17 #define L2X0_AUX_CTRL_WAY_SIZE_MASK (0x7 << 17) @@ -77,8 +94,33 @@ #define L2X0_AUX_CTRL_INSTR_PREFETCH_SHIFT 29 #define L2X0_AUX_CTRL_EARLY_BRESP_SHIFT 30 +#define L2X0_LATENCY_CTRL_SETUP_SHIFT 0 +#define L2X0_LATENCY_CTRL_RD_SHIFT 4 +#define L2X0_LATENCY_CTRL_WR_SHIFT 8 + +#define L2X0_ADDR_FILTER_EN 1 + #ifndef __ASSEMBLY__ extern void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask); +extern int l2x0_of_init(__u32 aux_val, __u32 aux_mask); + +struct l2x0_regs { + unsigned long phy_base; + unsigned long aux_ctrl; + /* + * Whether the following registers need to be saved/restored + * depends on platform + */ + unsigned long tag_latency; + unsigned long data_latency; + unsigned long filter_start; + unsigned long filter_end; + unsigned long prefetch_ctrl; + unsigned long pwr_ctrl; +}; + +extern struct l2x0_regs l2x0_saved_regs; + #endif #endif diff --git a/arch/arm/include/asm/outercache.h b/arch/arm/include/asm/outercache.h index d8387437ec5..53426c66352 100644 --- a/arch/arm/include/asm/outercache.h +++ b/arch/arm/include/asm/outercache.h @@ -34,6 +34,7 @@ struct outer_cache_fns { void (*sync)(void); #endif void (*set_debug)(unsigned long); + void (*resume)(void); }; #ifdef CONFIG_OUTER_CACHE @@ -74,6 +75,12 @@ static inline void outer_disable(void) outer_cache.disable(); } +static inline void outer_resume(void) +{ + if (outer_cache.resume) + outer_cache.resume(); +} + #else static inline void outer_inv_range(phys_addr_t start, phys_addr_t end) diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index ac75d084888..97b440c25c5 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -151,47 +151,11 @@ extern void __cpu_copy_user_highpage(struct page *to, struct page *from, #define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) extern void copy_page(void *to, const void *from); -typedef unsigned long pteval_t; - -#undef STRICT_MM_TYPECHECKS - -#ifdef STRICT_MM_TYPECHECKS -/* - * These are used to make use of C type-checking.. - */ -typedef struct { pteval_t pte; } pte_t; -typedef struct { unsigned long pmd; } pmd_t; -typedef struct { unsigned long pgd[2]; } pgd_t; -typedef struct { unsigned long pgprot; } pgprot_t; - -#define pte_val(x) ((x).pte) -#define pmd_val(x) ((x).pmd) -#define pgd_val(x) ((x).pgd[0]) -#define pgprot_val(x) ((x).pgprot) - -#define __pte(x) ((pte_t) { (x) } ) -#define __pmd(x) ((pmd_t) { (x) } ) -#define __pgprot(x) ((pgprot_t) { (x) } ) - +#ifdef CONFIG_ARM_LPAE +#include <asm/pgtable-3level-types.h> #else -/* - * .. while these make it easier on the compiler - */ -typedef pteval_t pte_t; -typedef unsigned long pmd_t; -typedef unsigned long pgd_t[2]; -typedef unsigned long pgprot_t; - -#define pte_val(x) (x) -#define pmd_val(x) (x) -#define pgd_val(x) ((x)[0]) -#define pgprot_val(x) (x) - -#define __pte(x) (x) -#define __pmd(x) (x) -#define __pgprot(x) (x) - -#endif /* STRICT_MM_TYPECHECKS */ +#include <asm/pgtable-2level-types.h> +#endif #endif /* CONFIG_MMU */ diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h index 22de005f159..943504f53f5 100644 --- a/arch/arm/include/asm/pgalloc.h +++ b/arch/arm/include/asm/pgalloc.h @@ -25,12 +25,34 @@ #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER)) #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL)) +#ifdef CONFIG_ARM_LPAE + +static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) +{ + return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); +} + +static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) +{ + BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); + free_page((unsigned long)pmd); +} + +static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) +{ + set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE)); +} + +#else /* !CONFIG_ARM_LPAE */ + /* * Since we have only two-level page tables, these are trivial */ #define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); }) #define pmd_free(mm, pmd) do { } while (0) -#define pgd_populate(mm,pmd,pte) BUG() +#define pud_populate(mm,pmd,pte) BUG() + +#endif /* CONFIG_ARM_LPAE */ extern pgd_t *pgd_alloc(struct mm_struct *mm); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); @@ -105,11 +127,13 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte) } static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte, - unsigned long prot) + pmdval_t prot) { - unsigned long pmdval = (pte + PTE_HWTABLE_OFF) | prot; + pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot; pmdp[0] = __pmd(pmdval); +#ifndef CONFIG_ARM_LPAE pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); +#endif flush_pmd_entry(pmdp); } diff --git a/arch/arm/include/asm/pgtable-2level-hwdef.h b/arch/arm/include/asm/pgtable-2level-hwdef.h new file mode 100644 index 00000000000..5cfba15cb40 --- /dev/null +++ b/arch/arm/include/asm/pgtable-2level-hwdef.h @@ -0,0 +1,93 @@ +/* + * arch/arm/include/asm/pgtable-2level-hwdef.h + * + * Copyright (C) 1995-2002 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _ASM_PGTABLE_2LEVEL_HWDEF_H +#define _ASM_PGTABLE_2LEVEL_HWDEF_H + +/* + * Hardware page table definitions. + * + * + Level 1 descriptor (PMD) + * - common + */ +#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0) +#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0) +#define PMD_TYPE_TABLE (_AT(pmdval_t, 1) << 0) +#define PMD_TYPE_SECT (_AT(pmdval_t, 2) << 0) +#define PMD_BIT4 (_AT(pmdval_t, 1) << 4) +#define PMD_DOMAIN(x) (_AT(pmdval_t, (x)) << 5) +#define PMD_PROTECTION (_AT(pmdval_t, 1) << 9) /* v5 */ +/* + * - section + */ +#define PMD_SECT_BUFFERABLE (_AT(pmdval_t, 1) << 2) +#define PMD_SECT_CACHEABLE (_AT(pmdval_t, 1) << 3) +#define PMD_SECT_XN (_AT(pmdval_t, 1) << 4) /* v6 */ +#define PMD_SECT_AP_WRITE (_AT(pmdval_t, 1) << 10) +#define PMD_SECT_AP_READ (_AT(pmdval_t, 1) << 11) +#define PMD_SECT_TEX(x) (_AT(pmdval_t, (x)) << 12) /* v5 */ +#define PMD_SECT_APX (_AT(pmdval_t, 1) << 15) /* v6 */ +#define PMD_SECT_S (_AT(pmdval_t, 1) << 16) /* v6 */ +#define PMD_SECT_nG (_AT(pmdval_t, 1) << 17) /* v6 */ +#define PMD_SECT_SUPER (_AT(pmdval_t, 1) << 18) /* v6 */ +#define PMD_SECT_AF (_AT(pmdval_t, 0)) + +#define PMD_SECT_UNCACHED (_AT(pmdval_t, 0)) +#define PMD_SECT_BUFFERED (PMD_SECT_BUFFERABLE) +#define PMD_SECT_WT (PMD_SECT_CACHEABLE) +#define PMD_SECT_WB (PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE) +#define PMD_SECT_MINICACHE (PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE) +#define PMD_SECT_WBWA (PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE) +#define PMD_SECT_NONSHARED_DEV (PMD_SECT_TEX(2)) + +/* + * - coarse table (not used) + */ + +/* + * + Level 2 descriptor (PTE) + * - common + */ +#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0) +#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0) +#define PTE_TYPE_LARGE (_AT(pteval_t, 1) << 0) +#define PTE_TYPE_SMALL (_AT(pteval_t, 2) << 0) +#define PTE_TYPE_EXT (_AT(pteval_t, 3) << 0) /* v5 */ +#define PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) +#define PTE_CACHEABLE (_AT(pteval_t, 1) << 3) + +/* + * - extended small page/tiny page + */ +#define PTE_EXT_XN (_AT(pteval_t, 1) << 0) /* v6 */ +#define PTE_EXT_AP_MASK (_AT(pteval_t, 3) << 4) +#define PTE_EXT_AP0 (_AT(pteval_t, 1) << 4) +#define PTE_EXT_AP1 (_AT(pteval_t, 2) << 4) +#define PTE_EXT_AP_UNO_SRO (_AT(pteval_t, 0) << 4) +#define PTE_EXT_AP_UNO_SRW (PTE_EXT_AP0) +#define PTE_EXT_AP_URO_SRW (PTE_EXT_AP1) +#define PTE_EXT_AP_URW_SRW (PTE_EXT_AP1|PTE_EXT_AP0) +#define PTE_EXT_TEX(x) (_AT(pteval_t, (x)) << 6) /* v5 */ +#define PTE_EXT_APX (_AT(pteval_t, 1) << 9) /* v6 */ +#define PTE_EXT_COHERENT (_AT(pteval_t, 1) << 9) /* XScale3 */ +#define PTE_EXT_SHARED (_AT(pteval_t, 1) << 10) /* v6 */ +#define PTE_EXT_NG (_AT(pteval_t, 1) << 11) /* v6 */ + +/* + * - small page + */ +#define PTE_SMALL_AP_MASK (_AT(pteval_t, 0xff) << 4) +#define PTE_SMALL_AP_UNO_SRO (_AT(pteval_t, 0x00) << 4) +#define PTE_SMALL_AP_UNO_SRW (_AT(pteval_t, 0x55) << 4) +#define PTE_SMALL_AP_URO_SRW (_AT(pteval_t, 0xaa) << 4) +#define PTE_SMALL_AP_URW_SRW (_AT(pteval_t, 0xff) << 4) + +#define PHYS_MASK (~0UL) + +#endif diff --git a/arch/arm/include/asm/pgtable-2level-types.h b/arch/arm/include/asm/pgtable-2level-types.h new file mode 100644 index 00000000000..66cb5b0e89c --- /dev/null +++ b/arch/arm/include/asm/pgtable-2level-types.h @@ -0,0 +1,67 @@ +/* + * arch/arm/include/asm/pgtable-2level-types.h + * + * Copyright (C) 1995-2003 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#ifndef _ASM_PGTABLE_2LEVEL_TYPES_H +#define _ASM_PGTABLE_2LEVEL_TYPES_H + +#include <asm/types.h> + +typedef u32 pteval_t; +typedef u32 pmdval_t; + +#undef STRICT_MM_TYPECHECKS + +#ifdef STRICT_MM_TYPECHECKS +/* + * These are used to make use of C type-checking.. + */ +typedef struct { pteval_t pte; } pte_t; +typedef struct { pmdval_t pmd; } pmd_t; +typedef struct { pmdval_t pgd[2]; } pgd_t; +typedef struct { pteval_t pgprot; } pgprot_t; + +#define pte_val(x) ((x).pte) +#define pmd_val(x) ((x).pmd) +#define pgd_val(x) ((x).pgd[0]) +#define pgprot_val(x) ((x).pgprot) + +#define __pte(x) ((pte_t) { (x) } ) +#define __pmd(x) ((pmd_t) { (x) } ) +#define __pgprot(x) ((pgprot_t) { (x) } ) + +#else +/* + * .. while these make it easier on the compiler + */ +typedef pteval_t pte_t; +typedef pmdval_t pmd_t; +typedef pmdval_t pgd_t[2]; +typedef pteval_t pgprot_t; + +#define pte_val(x) (x) +#define pmd_val(x) (x) +#define pgd_val(x) ((x)[0]) +#define pgprot_val(x) (x) + +#define __pte(x) (x) +#define __pmd(x) (x) +#define __pgprot(x) (x) + +#endif /* STRICT_MM_TYPECHECKS */ + +#endif /* _ASM_PGTABLE_2LEVEL_TYPES_H */ diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h new file mode 100644 index 00000000000..2317a71c8f8 --- /dev/null +++ b/arch/arm/include/asm/pgtable-2level.h @@ -0,0 +1,184 @@ +/* + * arch/arm/include/asm/pgtable-2level.h + * + * Copyright (C) 1995-2002 Russell King + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _ASM_PGTABLE_2LEVEL_H +#define _ASM_PGTABLE_2LEVEL_H + +/* + * Hardware-wise, we have a two level page table structure, where the first + * level has 4096 entries, and the second level has 256 entries. Each entry + * is one 32-bit word. Most of the bits in the second level entry are used + * by hardware, and there aren't any "accessed" and "dirty" bits. + * + * Linux on the other hand has a three level page table structure, which can + * be wrapped to fit a two level page table structure easily - using the PGD + * and PTE only. However, Linux also expects one "PTE" table per page, and + * at least a "dirty" bit. + * + * Therefore, we tweak the implementation slightly - we tell Linux that we + * have 2048 entries in the first level, each of which is 8 bytes (iow, two + * hardware pointers to the second level.) The second level contains two + * hardware PTE tables arranged contiguously, preceded by Linux versions + * which contain the state information Linux needs. We, therefore, end up + * with 512 entries in the "PTE" level. + * + * This leads to the page tables having the following layout: + * + * pgd pte + * | | + * +--------+ + * | | +------------+ +0 + * +- - - - + | Linux pt 0 | + * | | +------------+ +1024 + * +--------+ +0 | Linux pt 1 | + * | |-----> +------------+ +2048 + * +- - - - + +4 | h/w pt 0 | + * | |-----> +------------+ +3072 + * +--------+ +8 | h/w pt 1 | + * | | +------------+ +4096 + * + * See L_PTE_xxx below for definitions of bits in the "Linux pt", and + * PTE_xxx for definitions of bits appearing in the "h/w pt". + * + * PMD_xxx definitions refer to bits in the first level page table. + * + * The "dirty" bit is emulated by only granting hardware write permission + * iff the page is marked "writable" and "dirty" in the Linux PTE. This + * means that a write to a clean page will cause a permission fault, and + * the Linux MM layer will mark the page dirty via handle_pte_fault(). + * For the hardware to notice the permission change, the TLB entry must + * be flushed, and ptep_set_access_flags() does that for us. + * + * The "accessed" or "young" bit is emulated by a similar method; we only + * allow accesses to the page if the "young" bit is set. Accesses to the + * page will cause a fault, and handle_pte_fault() will set the young bit + * for us as long as the page is marked present in the corresponding Linux + * PTE entry. Again, ptep_set_access_flags() will ensure that the TLB is + * up to date. + * + * However, when the "young" bit is cleared, we deny access to the page + * by clearing the hardware PTE. Currently Linux does not flush the TLB + * for us in this case, which means the TLB will retain the transation + * until either the TLB entry is evicted under pressure, or a context + * switch which changes the user space mapping occurs. + */ +#define PTRS_PER_PTE 512 +#define PTRS_PER_PMD 1 +#define PTRS_PER_PGD 2048 + +#define PTE_HWTABLE_PTRS (PTRS_PER_PTE) +#define PTE_HWTABLE_OFF (PTE_HWTABLE_PTRS * sizeof(pte_t)) +#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u32)) + +/* + * PMD_SHIFT determines the size of the area a second-level page table can map + * PGDIR_SHIFT determines what a third-level page table entry can map + */ +#define PMD_SHIFT 21 +#define PGDIR_SHIFT 21 + +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +/* + * section address mask and size definitions. + */ +#define SECTION_SHIFT 20 +#define SECTION_SIZE (1UL << SECTION_SHIFT) +#define SECTION_MASK (~(SECTION_SIZE-1)) + +/* + * ARMv6 supersection address mask and size definitions. + */ +#define SUPERSECTION_SHIFT 24 +#define SUPERSECTION_SIZE (1UL << SUPERSECTION_SHIFT) +#define SUPERSECTION_MASK (~(SUPERSECTION_SIZE-1)) + +#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) + +/* + * "Linux" PTE definitions. + * + * We keep two sets of PTEs - the hardware and the linux version. + * This allows greater flexibility in the way we map the Linux bits + * onto the hardware tables, and allows us to have YOUNG and DIRTY + * bits. + * + * The PTE table pointer refers to the hardware entries; the "Linux" + * entries are stored 1024 bytes below. + */ +#define L_PTE_PRESENT (_AT(pteval_t, 1) << 0) +#define L_PTE_YOUNG (_AT(pteval_t, 1) << 1) +#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */ +#define L_PTE_DIRTY (_AT(pteval_t, 1) << 6) +#define L_PTE_RDONLY (_AT(pteval_t, 1) << 7) +#define L_PTE_USER (_AT(pteval_t, 1) << 8) +#define L_PTE_XN (_AT(pteval_t, 1) << 9) +#define L_PTE_SHARED (_AT(pteval_t, 1) << 10) /* shared(v6), coherent(xsc3) */ + +/* + * These are the memory types, defined to be compatible with + * pre-ARMv6 CPUs cacheable and bufferable bits: XXCB + */ +#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0x00) << 2) /* 0000 */ +#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 0x01) << 2) /* 0001 */ +#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 0x02) << 2) /* 0010 */ +#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 0x03) << 2) /* 0011 */ +#define L_PTE_MT_MINICACHE (_AT(pteval_t, 0x06) << 2) /* 0110 (sa1100, xscale) */ +#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 0x07) << 2) /* 0111 */ +#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 0x04) << 2) /* 0100 */ +#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 0x0c) << 2) /* 1100 */ +#define L_PTE_MT_DEV_WC (_AT(pteval_t, 0x09) << 2) /* 1001 */ +#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */ +#define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2) + +#ifndef __ASSEMBLY__ + +/* + * The "pud_xxx()" functions here are trivial when the pmd is folded into + * the pud: the pud entry is never bad, always exists, and can't be set or + * cleared. + */ +#define pud_none(pud) (0) +#define pud_bad(pud) (0) +#define pud_present(pud) (1) +#define pud_clear(pudp) do { } while (0) +#define set_pud(pud,pudp) do { } while (0) + +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) +{ + return (pmd_t *)pud; +} + +#define pmd_bad(pmd) (pmd_val(pmd) & 2) + +#define copy_pmd(pmdpd,pmdps) \ + do { \ + pmdpd[0] = pmdps[0]; \ + pmdpd[1] = pmdps[1]; \ + flush_pmd_entry(pmdpd); \ + } while (0) + +#define pmd_clear(pmdp) \ + do { \ + pmdp[0] = __pmd(0); \ + pmdp[1] = __pmd(0); \ + clean_pmd_entry(pmdp); \ + } while (0) + +/* we don't need complex calculations here as the pmd is folded into the pgd */ +#define pmd_addr_end(addr,end) (end) + +#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_PGTABLE_2LEVEL_H */ diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h new file mode 100644 index 00000000000..7c238a3b48c --- /dev/null +++ b/arch/arm/include/asm/pgtable-3level-hwdef.h @@ -0,0 +1,82 @@ +/* + * arch/arm/include/asm/pgtable-3level-hwdef.h + * + * Copyright (C) 2011 ARM Ltd. + * Author: Catalin Marinas <catalin.marinas@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#ifndef _ASM_PGTABLE_3LEVEL_HWDEF_H +#define _ASM_PGTABLE_3LEVEL_HWDEF_H + +/* + * Hardware page table definitions. + * + * + Level 1/2 descriptor + * - common + */ +#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0) +#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0) +#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0) +#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0) +#define PMD_BIT4 (_AT(pmdval_t, 0)) +#define PMD_DOMAIN(x) (_AT(pmdval_t, 0)) + +/* + * - section + */ +#define PMD_SECT_BUFFERABLE (_AT(pmdval_t, 1) << 2) +#define PMD_SECT_CACHEABLE (_AT(pmdval_t, 1) << 3) +#define PMD_SECT_S (_AT(pmdval_t, 3) << 8) +#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10) +#define PMD_SECT_nG (_AT(pmdval_t, 1) << 11) +#ifdef __ASSEMBLY__ +/* avoid 'shift count out of range' warning */ +#define PMD_SECT_XN (0) +#else +#define PMD_SECT_XN ((pmdval_t)1 << 54) +#endif +#define PMD_SECT_AP_WRITE (_AT(pmdval_t, 0)) +#define PMD_SECT_AP_READ (_AT(pmdval_t, 0)) +#define PMD_SECT_TEX(x) (_AT(pmdval_t, 0)) + +/* + * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers). + */ +#define PMD_SECT_UNCACHED (_AT(pmdval_t, 0) << 2) /* strongly ordered */ +#define PMD_SECT_BUFFERED (_AT(pmdval_t, 1) << 2) /* normal non-cacheable */ +#define PMD_SECT_WT (_AT(pmdval_t, 2) << 2) /* normal inner write-through */ +#define PMD_SECT_WB (_AT(pmdval_t, 3) << 2) /* normal inner write-back */ +#define PMD_SECT_WBWA (_AT(pmdval_t, 7) << 2) /* normal inner write-alloc */ + +/* + * + Level 3 descriptor (PTE) + */ +#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0) +#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0) +#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0) +#define PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */ +#define PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */ +#define PTE_EXT_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */ +#define PTE_EXT_AF (_AT(pteval_t, 1) << 10) /* Access Flag */ +#define PTE_EXT_NG (_AT(pteval_t, 1) << 11) /* nG */ +#define PTE_EXT_XN (_AT(pteval_t, 1) << 54) /* XN */ + +/* + * 40-bit physical address supported. + */ +#define PHYS_MASK_SHIFT (40) +#define PHYS_MASK ((1ULL << PHYS_MASK_SHIFT) - 1) + +#endif diff --git a/arch/arm/include/asm/pgtable-3level-types.h b/arch/arm/include/asm/pgtable-3level-types.h new file mode 100644 index 00000000000..921aa30259c --- /dev/null +++ b/arch/arm/include/asm/pgtable-3level-types.h @@ -0,0 +1,70 @@ +/* + * arch/arm/include/asm/pgtable-3level-types.h + * + * Copyright (C) 2011 ARM Ltd. + * Author: Catalin Marinas <catalin.marinas@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#ifndef _ASM_PGTABLE_3LEVEL_TYPES_H +#define _ASM_PGTABLE_3LEVEL_TYPES_H + +#include <asm/types.h> + +typedef u64 pteval_t; +typedef u64 pmdval_t; +typedef u64 pgdval_t; + +#undef STRICT_MM_TYPECHECKS + +#ifdef STRICT_MM_TYPECHECKS + +/* + * These are used to make use of C type-checking.. + */ +typedef struct { pteval_t pte; } pte_t; +typedef struct { pmdval_t pmd; } pmd_t; +typedef struct { pgdval_t pgd; } pgd_t; +typedef struct { pteval_t pgprot; } pgprot_t; + +#define pte_val(x) ((x).pte) +#define pmd_val(x) ((x).pmd) +#define pgd_val(x) ((x).pgd) +#define pgprot_val(x) ((x).pgprot) + +#define __pte(x) ((pte_t) { (x) } ) +#define __pmd(x) ((pmd_t) { (x) } ) +#define __pgd(x) ((pgd_t) { (x) } ) +#define __pgprot(x) ((pgprot_t) { (x) } ) + +#else /* !STRICT_MM_TYPECHECKS */ + +typedef pteval_t pte_t; +typedef pmdval_t pmd_t; +typedef pgdval_t pgd_t; +typedef pteval_t pgprot_t; + +#define pte_val(x) (x) +#define pmd_val(x) (x) +#define pgd_val(x) (x) +#define pgprot_val(x) (x) + +#define __pte(x) (x) +#define __pmd(x) (x) +#define __pgd(x) (x) +#define __pgprot(x) (x) + +#endif /* STRICT_MM_TYPECHECKS */ + +#endif /* _ASM_PGTABLE_3LEVEL_TYPES_H */ diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h new file mode 100644 index 00000000000..759af70f9a0 --- /dev/null +++ b/arch/arm/include/asm/pgtable-3level.h @@ -0,0 +1,155 @@ +/* + * arch/arm/include/asm/pgtable-3level.h + * + * Copyright (C) 2011 ARM Ltd. + * Author: Catalin Marinas <catalin.marinas@arm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ +#ifndef _ASM_PGTABLE_3LEVEL_H +#define _ASM_PGTABLE_3LEVEL_H + +/* + * With LPAE, there are 3 levels of page tables. Each level has 512 entries of + * 8 bytes each, occupying a 4K page. The first level table covers a range of + * 512GB, each entry representing 1GB. Since we are limited to 4GB input + * address range, only 4 entries in the PGD are used. + * + * There are enough spare bits in a page table entry for the kernel specific + * state. + */ +#define PTRS_PER_PTE 512 +#define PTRS_PER_PMD 512 +#define PTRS_PER_PGD 4 + +#define PTE_HWTABLE_PTRS (PTRS_PER_PTE) +#define PTE_HWTABLE_OFF (0) +#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64)) + +/* + * PGDIR_SHIFT determines the size a top-level page table entry can map. + */ +#define PGDIR_SHIFT 30 + +/* + * PMD_SHIFT determines the size a middle-level page table entry can map. + */ +#define PMD_SHIFT 21 + +#define PMD_SIZE (1UL << PMD_SHIFT) +#define PMD_MASK (~(PMD_SIZE-1)) +#define PGDIR_SIZE (1UL << PGDIR_SHIFT) +#define PGDIR_MASK (~(PGDIR_SIZE-1)) + +/* + * section address mask and size definitions. + */ +#define SECTION_SHIFT 21 +#define SECTION_SIZE (1UL << SECTION_SHIFT) +#define SECTION_MASK (~(SECTION_SIZE-1)) + +#define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE) + +/* + * "Linux" PTE definitions for LPAE. + * + * These bits overlap with the hardware bits but the naming is preserved for + * consistency with the classic page table format. + */ +#define L_PTE_PRESENT (_AT(pteval_t, 3) << 0) /* Valid */ +#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */ +#define L_PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */ +#define L_PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */ +#define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */ +#define L_PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */ +#define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */ +#define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */ +#define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */ +#define L_PTE_DIRTY (_AT(pteval_t, 1) << 55) /* unused */ +#define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56) /* unused */ + +/* + * To be used in assembly code with the upper page attributes. + */ +#define L_PTE_XN_HIGH (1 << (54 - 32)) +#define L_PTE_DIRTY_HIGH (1 << (55 - 32)) + +/* + * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers). + */ +#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0) << 2) /* strongly ordered */ +#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 1) << 2) /* normal non-cacheable */ +#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 2) << 2) /* normal inner write-through */ +#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 3) << 2) /* normal inner write-back */ +#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 7) << 2) /* normal inner write-alloc */ +#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 4) << 2) /* device */ +#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 4) << 2) /* device */ +#define L_PTE_MT_DEV_WC (_AT(pteval_t, 1) << 2) /* normal non-cacheable */ +#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */ +#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2) + +/* + * Software PGD flags. + */ +#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */ + +#ifndef __ASSEMBLY__ + +#define pud_none(pud) (!pud_val(pud)) +#define pud_bad(pud) (!(pud_val(pud) & 2)) +#define pud_present(pud) (pud_val(pud)) + +#define pud_clear(pudp) \ + do { \ + *pudp = __pud(0); \ + clean_pmd_entry(pudp); \ + } while (0) + +#define set_pud(pudp, pud) \ + do { \ + *pudp = pud; \ + flush_pmd_entry(pudp); \ + } while (0) + +static inline pmd_t *pud_page_vaddr(pud_t pud) +{ + return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK); +} + +/* Find an entry in the second-level page table.. */ +#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1)) +static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr) +{ + return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr); +} + +#define pmd_bad(pmd) (!(pmd_val(pmd) & 2)) + +#define copy_pmd(pmdpd,pmdps) \ + do { \ + *pmdpd = *pmdps; \ + flush_pmd_entry(pmdpd); \ + } while (0) + +#define pmd_clear(pmdp) \ + do { \ + *pmdp = __pmd(0); \ + clean_pmd_entry(pmdp); \ + } while (0) + +#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext))) + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_PGTABLE_3LEVEL_H */ diff --git a/arch/arm/include/asm/pgtable-hwdef.h b/arch/arm/include/asm/pgtable-hwdef.h index fd1521d5cb9..8426229ba29 100644 --- a/arch/arm/include/asm/pgtable-hwdef.h +++ b/arch/arm/include/asm/pgtable-hwdef.h @@ -10,81 +10,10 @@ #ifndef _ASMARM_PGTABLE_HWDEF_H #define _ASMARM_PGTABLE_HWDEF_H -/* - * Hardware page table definitions. - * - * + Level 1 descriptor (PMD) - * - common - */ -#define PMD_TYPE_MASK (3 << 0) -#define PMD_TYPE_FAULT (0 << 0) -#define PMD_TYPE_TABLE (1 << 0) -#define PMD_TYPE_SECT (2 << 0) -#define PMD_BIT4 (1 << 4) -#define PMD_DOMAIN(x) ((x) << 5) -#define PMD_PROTECTION (1 << 9) /* v5 */ -/* - * - section - */ -#define PMD_SECT_BUFFERABLE (1 << 2) -#define PMD_SECT_CACHEABLE (1 << 3) -#define PMD_SECT_XN (1 << 4) /* v6 */ -#define PMD_SECT_AP_WRITE (1 << 10) -#define PMD_SECT_AP_READ (1 << 11) -#define PMD_SECT_TEX(x) ((x) << 12) /* v5 */ -#define PMD_SECT_APX (1 << 15) /* v6 */ -#define PMD_SECT_S (1 << 16) /* v6 */ -#define PMD_SECT_nG (1 << 17) /* v6 */ -#define PMD_SECT_SUPER (1 << 18) /* v6 */ - -#define PMD_SECT_UNCACHED (0) -#define PMD_SECT_BUFFERED (PMD_SECT_BUFFERABLE) -#define PMD_SECT_WT (PMD_SECT_CACHEABLE) -#define PMD_SECT_WB (PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE) -#define PMD_SECT_MINICACHE (PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE) -#define PMD_SECT_WBWA (PMD_SECT_TEX(1) | PMD_SECT_CACHEABLE | PMD_SECT_BUFFERABLE) -#define PMD_SECT_NONSHARED_DEV (PMD_SECT_TEX(2)) - -/* - * - coarse table (not used) - */ - -/* - * + Level 2 descriptor (PTE) - * - common - */ -#define PTE_TYPE_MASK (3 << 0) -#define PTE_TYPE_FAULT (0 << 0) -#define PTE_TYPE_LARGE (1 << 0) -#define PTE_TYPE_SMALL (2 << 0) -#define PTE_TYPE_EXT (3 << 0) /* v5 */ -#define PTE_BUFFERABLE (1 << 2) -#define PTE_CACHEABLE (1 << 3) - -/* - * - extended small page/tiny page - */ -#define PTE_EXT_XN (1 << 0) /* v6 */ -#define PTE_EXT_AP_MASK (3 << 4) -#define PTE_EXT_AP0 (1 << 4) -#define PTE_EXT_AP1 (2 << 4) -#define PTE_EXT_AP_UNO_SRO (0 << 4) -#define PTE_EXT_AP_UNO_SRW (PTE_EXT_AP0) -#define PTE_EXT_AP_URO_SRW (PTE_EXT_AP1) -#define PTE_EXT_AP_URW_SRW (PTE_EXT_AP1|PTE_EXT_AP0) -#define PTE_EXT_TEX(x) ((x) << 6) /* v5 */ -#define PTE_EXT_APX (1 << 9) /* v6 */ -#define PTE_EXT_COHERENT (1 << 9) /* XScale3 */ -#define PTE_EXT_SHARED (1 << 10) /* v6 */ -#define PTE_EXT_NG (1 << 11) /* v6 */ - -/* - * - small page - */ -#define PTE_SMALL_AP_MASK (0xff << 4) -#define PTE_SMALL_AP_UNO_SRO (0x00 << 4) -#define PTE_SMALL_AP_UNO_SRW (0x55 << 4) -#define PTE_SMALL_AP_URO_SRW (0xaa << 4) -#define PTE_SMALL_AP_URW_SRW (0xff << 4) +#ifdef CONFIG_ARM_LPAE +#include <asm/pgtable-3level-hwdef.h> +#else +#include <asm/pgtable-2level-hwdef.h> +#endif #endif diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index f1956b27ae5..aec18abf4e6 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -11,19 +11,26 @@ #define _ASMARM_PGTABLE_H #include <linux/const.h> -#include <asm-generic/4level-fixup.h> #include <asm/proc-fns.h> #ifndef CONFIG_MMU +#include <asm-generic/4level-fixup.h> #include "pgtable-nommu.h" #else +#include <asm-generic/pgtable-nopud.h> #include <asm/memory.h> #include <mach/vmalloc.h> #include <asm/pgtable-hwdef.h> +#ifdef CONFIG_ARM_LPAE +#include <asm/pgtable-3level.h> +#else +#include <asm/pgtable-2level.h> +#endif + /* * Just any arbitrary offset to the start of the vmalloc VM area: the * current 8MB value just means that there will be a 8MB "hole" after the @@ -41,79 +48,6 @@ #define VMALLOC_START (((unsigned long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1)) #endif -/* - * Hardware-wise, we have a two level page table structure, where the first - * level has 4096 entries, and the second level has 256 entries. Each entry - * is one 32-bit word. Most of the bits in the second level entry are used - * by hardware, and there aren't any "accessed" and "dirty" bits. - * - * Linux on the other hand has a three level page table structure, which can - * be wrapped to fit a two level page table structure easily - using the PGD - * and PTE only. However, Linux also expects one "PTE" table per page, and - * at least a "dirty" bit. - * - * Therefore, we tweak the implementation slightly - we tell Linux that we - * have 2048 entries in the first level, each of which is 8 bytes (iow, two - * hardware pointers to the second level.) The second level contains two - * hardware PTE tables arranged contiguously, preceded by Linux versions - * which contain the state information Linux needs. We, therefore, end up - * with 512 entries in the "PTE" level. - * - * This leads to the page tables having the following layout: - * - * pgd pte - * | | - * +--------+ - * | | +------------+ +0 - * +- - - - + | Linux pt 0 | - * | | +------------+ +1024 - * +--------+ +0 | Linux pt 1 | - * | |-----> +------------+ +2048 - * +- - - - + +4 | h/w pt 0 | - * | |-----> +------------+ +3072 - * +--------+ +8 | h/w pt 1 | - * | | +------------+ +4096 - * - * See L_PTE_xxx below for definitions of bits in the "Linux pt", and - * PTE_xxx for definitions of bits appearing in the "h/w pt". - * - * PMD_xxx definitions refer to bits in the first level page table. - * - * The "dirty" bit is emulated by only granting hardware write permission - * iff the page is marked "writable" and "dirty" in the Linux PTE. This - * means that a write to a clean page will cause a permission fault, and - * the Linux MM layer will mark the page dirty via handle_pte_fault(). - * For the hardware to notice the permission change, the TLB entry must - * be flushed, and ptep_set_access_flags() does that for us. - * - * The "accessed" or "young" bit is emulated by a similar method; we only - * allow accesses to the page if the "young" bit is set. Accesses to the - * page will cause a fault, and handle_pte_fault() will set the young bit - * for us as long as the page is marked present in the corresponding Linux - * PTE entry. Again, ptep_set_access_flags() will ensure that the TLB is - * up to date. - * - * However, when the "young" bit is cleared, we deny access to the page - * by clearing the hardware PTE. Currently Linux does not flush the TLB - * for us in this case, which means the TLB will retain the transation - * until either the TLB entry is evicted under pressure, or a context - * switch which changes the user space mapping occurs. - */ -#define PTRS_PER_PTE 512 -#define PTRS_PER_PMD 1 -#define PTRS_PER_PGD 2048 - -#define PTE_HWTABLE_PTRS (PTRS_PER_PTE) -#define PTE_HWTABLE_OFF (PTE_HWTABLE_PTRS * sizeof(pte_t)) -#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u32)) - -/* - * PMD_SHIFT determines the size of the area a second-level page table can map - * PGDIR_SHIFT determines what a third-level page table entry can map - */ -#define PMD_SHIFT 21 -#define PGDIR_SHIFT 21 - #define LIBRARY_TEXT_START 0x0c000000 #ifndef __ASSEMBLY__ @@ -124,12 +58,6 @@ extern void __pgd_error(const char *file, int line, pgd_t); #define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte) #define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd) #define pgd_ERROR(pgd) __pgd_error(__FILE__, __LINE__, pgd) -#endif /* !__ASSEMBLY__ */ - -#define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) -#define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) /* * This is the lowest virtual address we can permit any user space @@ -138,60 +66,6 @@ extern void __pgd_error(const char *file, int line, pgd_t); */ #define FIRST_USER_ADDRESS PAGE_SIZE -#define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) - -/* - * section address mask and size definitions. - */ -#define SECTION_SHIFT 20 -#define SECTION_SIZE (1UL << SECTION_SHIFT) -#define SECTION_MASK (~(SECTION_SIZE-1)) - -/* - * ARMv6 supersection address mask and size definitions. - */ -#define SUPERSECTION_SHIFT 24 -#define SUPERSECTION_SIZE (1UL << SUPERSECTION_SHIFT) -#define SUPERSECTION_MASK (~(SUPERSECTION_SIZE-1)) - -/* - * "Linux" PTE definitions. - * - * We keep two sets of PTEs - the hardware and the linux version. - * This allows greater flexibility in the way we map the Linux bits - * onto the hardware tables, and allows us to have YOUNG and DIRTY - * bits. - * - * The PTE table pointer refers to the hardware entries; the "Linux" - * entries are stored 1024 bytes below. - */ -#define L_PTE_PRESENT (_AT(pteval_t, 1) << 0) -#define L_PTE_YOUNG (_AT(pteval_t, 1) << 1) -#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */ -#define L_PTE_DIRTY (_AT(pteval_t, 1) << 6) -#define L_PTE_RDONLY (_AT(pteval_t, 1) << 7) -#define L_PTE_USER (_AT(pteval_t, 1) << 8) -#define L_PTE_XN (_AT(pteval_t, 1) << 9) -#define L_PTE_SHARED (_AT(pteval_t, 1) << 10) /* shared(v6), coherent(xsc3) */ - -/* - * These are the memory types, defined to be compatible with - * pre-ARMv6 CPUs cacheable and bufferable bits: XXCB - */ -#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0x00) << 2) /* 0000 */ -#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 0x01) << 2) /* 0001 */ -#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 0x02) << 2) /* 0010 */ -#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 0x03) << 2) /* 0011 */ -#define L_PTE_MT_MINICACHE (_AT(pteval_t, 0x06) << 2) /* 0110 (sa1100, xscale) */ -#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 0x07) << 2) /* 0111 */ -#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 0x04) << 2) /* 0100 */ -#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 0x0c) << 2) /* 1100 */ -#define L_PTE_MT_DEV_WC (_AT(pteval_t, 0x09) << 2) /* 1001 */ -#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */ -#define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2) - -#ifndef __ASSEMBLY__ - /* * The pgprot_* and protection_map entries will be fixed up in runtime * to include the cachable and bufferable bits based on memory policy, @@ -294,50 +168,15 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; /* to find an entry in a kernel page-table-directory */ #define pgd_offset_k(addr) pgd_offset(&init_mm, addr) -/* - * The "pgd_xxx()" functions here are trivial for a folded two-level - * setup: the pgd is never bad, and a pmd always exists (as it's folded - * into the pgd entry) - */ -#define pgd_none(pgd) (0) -#define pgd_bad(pgd) (0) -#define pgd_present(pgd) (1) -#define pgd_clear(pgdp) do { } while (0) -#define set_pgd(pgd,pgdp) do { } while (0) -#define set_pud(pud,pudp) do { } while (0) - - -/* Find an entry in the second-level page table.. */ -#define pmd_offset(dir, addr) ((pmd_t *)(dir)) - #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_present(pmd) (pmd_val(pmd)) -#define pmd_bad(pmd) (pmd_val(pmd) & 2) - -#define copy_pmd(pmdpd,pmdps) \ - do { \ - pmdpd[0] = pmdps[0]; \ - pmdpd[1] = pmdps[1]; \ - flush_pmd_entry(pmdpd); \ - } while (0) - -#define pmd_clear(pmdp) \ - do { \ - pmdp[0] = __pmd(0); \ - pmdp[1] = __pmd(0); \ - clean_pmd_entry(pmdp); \ - } while (0) static inline pte_t *pmd_page_vaddr(pmd_t pmd) { - return __va(pmd_val(pmd) & PAGE_MASK); + return __va(pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK); } -#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd))) - -/* we don't need complex calculations here as the pmd is folded into the pgd */ -#define pmd_addr_end(addr,end) (end) - +#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) #ifndef CONFIG_HIGHPTE #define __pte_map(pmd) pmd_page_vaddr(*(pmd)) @@ -354,13 +193,12 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_offset_map(pmd,addr) (__pte_map(pmd) + pte_index(addr)) #define pte_unmap(pte) __pte_unmap(pte) -#define pte_pfn(pte) (pte_val(pte) >> PAGE_SHIFT) +#define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT) #define pfn_pte(pfn,prot) __pte(__pfn_to_phys(pfn) | pgprot_val(prot)) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) #define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot) -#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) #define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0) #if __LINUX_ARM_ARCH__ < 6 diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index 9e92cb205e6..f3628fb3d2b 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -65,7 +65,11 @@ extern struct processor { * Set a possibly extended PTE. Non-extended PTEs should * ignore 'ext'. */ +#ifdef CONFIG_ARM_LPAE + void (*set_pte_ext)(pte_t *ptep, pte_t pte); +#else void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext); +#endif /* Suspend/resume */ unsigned int suspend_size; @@ -79,7 +83,11 @@ extern void cpu_proc_fin(void); extern int cpu_do_idle(void); extern void cpu_dcache_clean_area(void *, int); extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); +#ifdef CONFIG_ARM_LPAE +extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte); +#else extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); +#endif extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); /* These three are private to arch/arm/kernel/suspend.c */ @@ -107,6 +115,18 @@ extern void cpu_resume(void); #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) +#ifdef CONFIG_ARM_LPAE +#define cpu_get_pgd() \ + ({ \ + unsigned long pg, pg2; \ + __asm__("mrrc p15, 0, %0, %1, c2" \ + : "=r" (pg), "=r" (pg2) \ + : \ + : "cc"); \ + pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \ + (pgd_t *)phys_to_virt(pg); \ + }) +#else #define cpu_get_pgd() \ ({ \ unsigned long pg; \ @@ -115,6 +135,7 @@ extern void cpu_resume(void); pg &= ~0x3fff; \ (pgd_t *)phys_to_virt(pg); \ }) +#endif #endif diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h index 4adf71b2b54..d4ced6df39f 100644 --- a/arch/arm/include/asm/system.h +++ b/arch/arm/include/asm/system.h @@ -87,6 +87,14 @@ struct siginfo; void arm_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info, unsigned long err, unsigned long trap); +#ifdef CONFIG_ARM_LPAE +#define FAULT_CODE_ALIGNMENT 33 +#define FAULT_CODE_DEBUG 34 +#else +#define FAULT_CODE_ALIGNMENT 1 +#define FAULT_CODE_DEBUG 2 +#endif + void hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), int sig, int code, const char *name); diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h index 265f908c4a6..5d3ed7e3856 100644 --- a/arch/arm/include/asm/tlb.h +++ b/arch/arm/include/asm/tlb.h @@ -202,8 +202,18 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, tlb_remove_page(tlb, pte); } +static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, + unsigned long addr) +{ +#ifdef CONFIG_ARM_LPAE + tlb_add_flush(tlb, addr); + tlb_remove_page(tlb, virt_to_page(pmdp)); +#endif +} + #define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) -#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) +#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr) +#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) #define tlb_migrate_finish(mm) do { } while (0) diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index 8077145698f..02b2f820398 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -471,7 +471,7 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr) * these operations. This is typically used when we are removing * PMD entries. */ -static inline void flush_pmd_entry(pmd_t *pmd) +static inline void flush_pmd_entry(void *pmd) { const unsigned int __tlb_flag = __cpu_tlb_flags; @@ -487,7 +487,7 @@ static inline void flush_pmd_entry(pmd_t *pmd) dsb(); } -static inline void clean_pmd_entry(pmd_t *pmd) +static inline void clean_pmd_entry(void *pmd) { const unsigned int __tlb_flag = __cpu_tlb_flags; diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 927522cfc12..69391bff2b9 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -20,6 +20,7 @@ #include <asm/thread_info.h> #include <asm/memory.h> #include <asm/procinfo.h> +#include <asm/hardware/cache-l2x0.h> #include <linux/kbuild.h> /* @@ -89,6 +90,17 @@ int main(void) DEFINE(S_OLD_R0, offsetof(struct pt_regs, ARM_ORIG_r0)); DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs)); BLANK(); +#ifdef CONFIG_CACHE_L2X0 + DEFINE(L2X0_R_PHY_BASE, offsetof(struct l2x0_regs, phy_base)); + DEFINE(L2X0_R_AUX_CTRL, offsetof(struct l2x0_regs, aux_ctrl)); + DEFINE(L2X0_R_TAG_LATENCY, offsetof(struct l2x0_regs, tag_latency)); + DEFINE(L2X0_R_DATA_LATENCY, offsetof(struct l2x0_regs, data_latency)); + DEFINE(L2X0_R_FILTER_START, offsetof(struct l2x0_regs, filter_start)); + DEFINE(L2X0_R_FILTER_END, offsetof(struct l2x0_regs, filter_end)); + DEFINE(L2X0_R_PREFETCH_CTRL, offsetof(struct l2x0_regs, prefetch_ctrl)); + DEFINE(L2X0_R_PWR_CTRL, offsetof(struct l2x0_regs, pwr_ctrl)); + BLANK(); +#endif #ifdef CONFIG_CPU_HAS_ASID DEFINE(MM_CONTEXT_ID, offsetof(struct mm_struct, context.id)); BLANK(); diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 742b6108a00..a400a4dc57c 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -21,6 +21,7 @@ #include <asm/memory.h> #include <asm/thread_info.h> #include <asm/system.h> +#include <asm/pgtable.h> #ifdef CONFIG_DEBUG_LL #include <mach/debug-macro.S> @@ -38,11 +39,20 @@ #error KERNEL_RAM_VADDR must start at 0xXXXX8000 #endif +#ifdef CONFIG_ARM_LPAE + /* LPAE requires an additional page for the PGD */ +#define PG_DIR_SIZE 0x5000 +#define PMD_ORDER 3 +#else +#define PG_DIR_SIZE 0x4000 +#define PMD_ORDER 2 +#endif + .globl swapper_pg_dir - .equ swapper_pg_dir, KERNEL_RAM_VADDR - 0x4000 + .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE .macro pgtbl, rd, phys - add \rd, \phys, #TEXT_OFFSET - 0x4000 + add \rd, \phys, #TEXT_OFFSET - PG_DIR_SIZE .endm #ifdef CONFIG_XIP_KERNEL @@ -148,11 +158,11 @@ __create_page_tables: pgtbl r4, r8 @ page table address /* - * Clear the 16K level 1 swapper page table + * Clear the swapper page table */ mov r0, r4 mov r3, #0 - add r6, r0, #0x4000 + add r6, r0, #PG_DIR_SIZE 1: str r3, [r0], #4 str r3, [r0], #4 str r3, [r0], #4 @@ -160,6 +170,25 @@ __create_page_tables: teq r0, r6 bne 1b +#ifdef CONFIG_ARM_LPAE + /* + * Build the PGD table (first level) to point to the PMD table. A PGD + * entry is 64-bit wide. + */ + mov r0, r4 + add r3, r4, #0x1000 @ first PMD table address + orr r3, r3, #3 @ PGD block type + mov r6, #4 @ PTRS_PER_PGD + mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER +1: str r3, [r0], #4 @ set bottom PGD entry bits + str r7, [r0], #4 @ set top PGD entry bits + add r3, r3, #0x1000 @ next PMD table + subs r6, r6, #1 + bne 1b + + add r4, r4, #0x1000 @ point to the PMD tables +#endif + ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags /* @@ -171,30 +200,30 @@ __create_page_tables: sub r0, r0, r3 @ virt->phys offset add r5, r5, r0 @ phys __enable_mmu add r6, r6, r0 @ phys __enable_mmu_end - mov r5, r5, lsr #20 - mov r6, r6, lsr #20 + mov r5, r5, lsr #SECTION_SHIFT + mov r6, r6, lsr #SECTION_SHIFT -1: orr r3, r7, r5, lsl #20 @ flags + kernel base - str r3, [r4, r5, lsl #2] @ identity mapping - teq r5, r6 - addne r5, r5, #1 @ next section - bne 1b +1: orr r3, r7, r5, lsl #SECTION_SHIFT @ flags + kernel base + str r3, [r4, r5, lsl #PMD_ORDER] @ identity mapping + cmp r5, r6 + addlo r5, r5, #1 @ next section + blo 1b /* * Now setup the pagetables for our kernel direct * mapped region. */ mov r3, pc - mov r3, r3, lsr #20 - orr r3, r7, r3, lsl #20 - add r0, r4, #(KERNEL_START & 0xff000000) >> 18 - str r3, [r0, #(KERNEL_START & 0x00f00000) >> 18]! + mov r3, r3, lsr #SECTION_SHIFT + orr r3, r7, r3, lsl #SECTION_SHIFT + add r0, r4, #(KERNEL_START & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) + str r3, [r0, #((KERNEL_START & 0x00f00000) >> SECTION_SHIFT) << PMD_ORDER]! ldr r6, =(KERNEL_END - 1) - add r0, r0, #4 - add r6, r4, r6, lsr #18 + add r0, r0, #1 << PMD_ORDER + add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 1: cmp r0, r6 - add r3, r3, #1 << 20 - strls r3, [r0], #4 + add r3, r3, #1 << SECTION_SHIFT + strls r3, [r0], #1 << PMD_ORDER bls 1b #ifdef CONFIG_XIP_KERNEL @@ -203,11 +232,11 @@ __create_page_tables: */ add r3, r8, #TEXT_OFFSET orr r3, r3, r7 - add r0, r4, #(KERNEL_RAM_VADDR & 0xff000000) >> 18 - str r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> 18]! + add r0, r4, #(KERNEL_RAM_VADDR & 0xff000000) >> (SECTION_SHIFT - PMD_ORDER) + str r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> (SECTION_SHIFT - PMD_ORDER)]! ldr r6, =(_end - 1) add r0, r0, #4 - add r6, r4, r6, lsr #18 + add r6, r4, r6, lsr #(SECTION_SHIFT - PMD_ORDER) 1: cmp r0, r6 add r3, r3, #1 << 20 strls r3, [r0], #4 @@ -215,15 +244,15 @@ __create_page_tables: #endif /* - * Then map boot params address in r2 or - * the first 1MB of ram if boot params address is not specified. + * Then map boot params address in r2 or the first 1MB (2MB with LPAE) + * of ram if boot params address is not specified. */ - mov r0, r2, lsr #20 - movs r0, r0, lsl #20 + mov r0, r2, lsr #SECTION_SHIFT + movs r0, r0, lsl #SECTION_SHIFT moveq r0, r8 sub r3, r0, r8 add r3, r3, #PAGE_OFFSET - add r3, r4, r3, lsr #18 + add r3, r4, r3, lsr #(SECTION_SHIFT - PMD_ORDER) orr r6, r7, r0 str r6, [r3] @@ -236,21 +265,27 @@ __create_page_tables: */ addruart r7, r3 - mov r3, r3, lsr #20 - mov r3, r3, lsl #2 + mov r3, r3, lsr #SECTION_SHIFT + mov r3, r3, lsl #PMD_ORDER add r0, r4, r3 rsb r3, r3, #0x4000 @ PTRS_PER_PGD*sizeof(long) cmp r3, #0x0800 @ limit to 512MB movhi r3, #0x0800 add r6, r0, r3 - mov r3, r7, lsr #20 + mov r3, r7, lsr #SECTION_SHIFT ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags - orr r3, r7, r3, lsl #20 + orr r3, r7, r3, lsl #SECTION_SHIFT +#ifdef CONFIG_ARM_LPAE + mov r7, #1 << (54 - 32) @ XN +#endif 1: str r3, [r0], #4 - add r3, r3, #1 << 20 - teq r0, r6 - bne 1b +#ifdef CONFIG_ARM_LPAE + str r7, [r0], #4 +#endif + add r3, r3, #1 << SECTION_SHIFT + cmp r0, r6 + blo 1b #else /* CONFIG_DEBUG_ICEDCC */ /* we don't need any serial debugging mappings for ICEDCC */ @@ -262,7 +297,7 @@ __create_page_tables: * If we're using the NetWinder or CATS, we also need to map * in the 16550-type serial port for the debug messages */ - add r0, r4, #0xff000000 >> 18 + add r0, r4, #0xff000000 >> (SECTION_SHIFT - PMD_ORDER) orr r3, r7, #0x7c000000 str r3, [r0] #endif @@ -272,13 +307,16 @@ __create_page_tables: * Similar reasons here - for debug. This is * only for Acorn RiscPC architectures. */ - add r0, r4, #0x02000000 >> 18 + add r0, r4, #0x02000000 >> (SECTION_SHIFT - PMD_ORDER) orr r3, r7, #0x02000000 str r3, [r0] - add r0, r4, #0xd8000000 >> 18 + add r0, r4, #0xd8000000 >> (SECTION_SHIFT - PMD_ORDER) str r3, [r0] #endif #endif +#ifdef CONFIG_ARM_LPAE + sub r4, r4, #0x1000 @ point to the PGD table +#endif mov pc, lr ENDPROC(__create_page_tables) .ltorg @@ -370,12 +408,17 @@ __enable_mmu: #ifdef CONFIG_CPU_ICACHE_DISABLE bic r0, r0, #CR_I #endif +#ifdef CONFIG_ARM_LPAE + mov r5, #0 + mcrr p15, 0, r4, r5, c2 @ load TTBR0 +#else mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ domain_val(DOMAIN_IO, DOMAIN_CLIENT)) mcr p15, 0, r5, c3, c0, 0 @ load domain access register mcr p15, 0, r4, c2, c0, 0 @ load page table pointer +#endif b __turn_mmu_on ENDPROC(__enable_mmu) @@ -396,8 +439,10 @@ ENDPROC(__enable_mmu) .align 5 __turn_mmu_on: mov r0, r0 + instr_sync mcr p15, 0, r0, c1, c0, 0 @ write control reg mrc p15, 0, r3, c0, c0, 0 @ read id reg + instr_sync mov r3, r3 mov r3, r13 mov pc, r3 diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c index 814a52a9dc3..d6a95ef9131 100644 --- a/arch/arm/kernel/hw_breakpoint.c +++ b/arch/arm/kernel/hw_breakpoint.c @@ -1016,10 +1016,10 @@ static int __init arch_hw_breakpoint_init(void) } /* Register debug fault handler. */ - hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, - "watchpoint debug exception"); - hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, - "breakpoint debug exception"); + hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, + TRAP_HWBKPT, "watchpoint debug exception"); + hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP, + TRAP_HWBKPT, "breakpoint debug exception"); /* Register hotplug notifier. */ register_cpu_notifier(&dbg_reset_nb); diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c index cc2020c2c70..1e9be5d25e5 100644 --- a/arch/arm/kernel/module.c +++ b/arch/arm/kernel/module.c @@ -33,7 +33,7 @@ * recompiling the whole kernel when CONFIG_XIP_KERNEL is turned on/off. */ #undef MODULES_VADDR -#define MODULES_VADDR (((unsigned long)_etext + ~PGDIR_MASK) & PGDIR_MASK) +#define MODULES_VADDR (((unsigned long)_etext + ~PMD_MASK) & PMD_MASK) #endif #ifdef CONFIG_MMU diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S index 020e99c845e..4ac45fe955f 100644 --- a/arch/arm/kernel/sleep.S +++ b/arch/arm/kernel/sleep.S @@ -56,8 +56,10 @@ ENDPROC(cpu_suspend_abort) .align 5 ENTRY(cpu_resume_mmu) ldr r3, =cpu_resume_after_mmu + instr_sync mcr p15, 0, r0, c1, c0, 0 @ turn on MMU, I-cache, etc mrc p15, 0, r0, c0, c0, 0 @ read id reg + instr_sync mov r0, r0 mov r0, r0 mov pc, r3 @ jump to virtual address diff --git a/arch/arm/kernel/smp_scu.c b/arch/arm/kernel/smp_scu.c index 7fcddb75c87..8f5dd796335 100644 --- a/arch/arm/kernel/smp_scu.c +++ b/arch/arm/kernel/smp_scu.c @@ -34,7 +34,7 @@ unsigned int __init scu_get_core_count(void __iomem *scu_base) /* * Enable the SCU */ -void __init scu_enable(void __iomem *scu_base) +void scu_enable(void __iomem *scu_base) { u32 scu_ctrl; diff --git a/arch/arm/mach-exynos4/cpu.c b/arch/arm/mach-exynos4/cpu.c index 746d6fc6d39..c1cb39df589 100644 --- a/arch/arm/mach-exynos4/cpu.c +++ b/arch/arm/mach-exynos4/cpu.c @@ -11,6 +11,7 @@ #include <linux/sched.h> #include <linux/sysdev.h> +#include <asm/cacheflush.h> #include <asm/mach/map.h> #include <asm/mach/irq.h> @@ -31,6 +32,7 @@ #include <mach/regs-irq.h> #include <mach/regs-pmu.h> +#include <mach/pmu.h> extern int combiner_init(unsigned int combiner_nr, void __iomem *base, unsigned int irq_start); @@ -221,16 +223,34 @@ core_initcall(exynos4_core_init); #ifdef CONFIG_CACHE_L2X0 static int __init exynos4_l2x0_cache_init(void) { - /* TAG, Data Latency Control: 2cycle */ - __raw_writel(0x110, S5P_VA_L2CC + L2X0_TAG_LATENCY_CTRL); - __raw_writel(0x110, S5P_VA_L2CC + L2X0_DATA_LATENCY_CTRL); + if (!(__raw_readl(S5P_VA_L2CC + L2X0_CTRL) & 0x1)) { - /* L2X0 Prefetch Control */ - __raw_writel(0x30000007, S5P_VA_L2CC + L2X0_PREFETCH_CTRL); + l2x0_saved_regs.phy_base = EXYNOS4_PA_L2CC; + /* TAG, Data Latency Control: 2 cycles */ + l2x0_saved_regs.tag_latency = 0x110; + l2x0_saved_regs.data_latency = 0x110; + l2x0_saved_regs.prefetch_ctrl = 0x30000007; + l2x0_saved_regs.pwr_ctrl = + (L2X0_DYNAMIC_CLK_GATING_EN | L2X0_STNDBY_MODE_EN); - /* L2X0 Power Control */ - __raw_writel(L2X0_DYNAMIC_CLK_GATING_EN | L2X0_STNDBY_MODE_EN, - S5P_VA_L2CC + L2X0_POWER_CTRL); + l2x0_regs_phys = virt_to_phys(&l2x0_saved_regs); + + __raw_writel(l2x0_saved_regs.tag_latency, + S5P_VA_L2CC + L2X0_TAG_LATENCY_CTRL); + __raw_writel(l2x0_saved_regs.data_latency, + S5P_VA_L2CC + L2X0_DATA_LATENCY_CTRL); + + /* L2X0 Prefetch Control */ + __raw_writel(l2x0_saved_regs.prefetch_ctrl, + S5P_VA_L2CC + L2X0_PREFETCH_CTRL); + + /* L2X0 Power Control */ + __raw_writel(l2x0_saved_regs.pwr_ctrl, + S5P_VA_L2CC + L2X0_POWER_CTRL); + + clean_dcache_area(&l2x0_regs_phys, sizeof(unsigned long)); + clean_dcache_area(&l2x0_saved_regs, sizeof(struct l2x0_regs)); + } l2x0_init(S5P_VA_L2CC, 0x7C470001, 0xC200ffff); @@ -240,6 +260,15 @@ static int __init exynos4_l2x0_cache_init(void) early_initcall(exynos4_l2x0_cache_init); #endif +int exynos4_subrev(void) +{ + static int subrev = -1; + if (unlikely(subrev < 0)) + subrev = readl(S5P_VA_CHIPID) & 0xf; + + return subrev; +} + int __init exynos4_init(void) { printk(KERN_INFO "EXYNOS4: Initializing architecture\n"); diff --git a/arch/arm/mach-exynos4/cpuidle.c b/arch/arm/mach-exynos4/cpuidle.c index bf7e96f2793..6d527610c5a 100644 --- a/arch/arm/mach-exynos4/cpuidle.c +++ b/arch/arm/mach-exynos4/cpuidle.c @@ -11,22 +11,48 @@ #include <linux/kernel.h> #include <linux/init.h> #include <linux/cpuidle.h> +#include <linux/cpu_pm.h> #include <linux/io.h> - +#include <linux/suspend.h> +#include <linux/err.h> #include <asm/proc-fns.h> +#include <asm/smp_scu.h> +#include <asm/suspend.h> +#include <asm/unified.h> +#include <mach/regs-pmu.h> +#include <mach/pmu.h> + +#include <plat/exynos4.h> +#include <plat/cpu.h> + +#define REG_DIRECTGO_ADDR (exynos4_subrev() == 0 ?\ + (S5P_VA_SYSRAM + 0x24) : S5P_INFORM7) +#define REG_DIRECTGO_FLAG (exynos4_subrev() == 0 ?\ + (S5P_VA_SYSRAM + 0x20) : S5P_INFORM6) static int exynos4_enter_idle(struct cpuidle_device *dev, struct cpuidle_state *state); +static int exynos4_enter_lowpower(struct cpuidle_device *dev, + struct cpuidle_state *state); + static struct cpuidle_state exynos4_cpuidle_set[] = { [0] = { .enter = exynos4_enter_idle, .exit_latency = 1, .target_residency = 100000, .flags = CPUIDLE_FLAG_TIME_VALID, - .name = "IDLE", + .name = "C0", .desc = "ARM clock gating(WFI)", }, + [1] = { + .enter = exynos4_enter_lowpower, + .exit_latency = 300, + .target_residency = 100000, + .flags = CPUIDLE_FLAG_TIME_VALID, + .name = "C1", + .desc = "ARM power down", + }, }; static DEFINE_PER_CPU(struct cpuidle_device, exynos4_cpuidle_device); @@ -36,6 +62,96 @@ static struct cpuidle_driver exynos4_idle_driver = { .owner = THIS_MODULE, }; +/* Ext-GIC nIRQ/nFIQ is the only wakeup source in AFTR */ +static void exynos4_set_wakeupmask(void) +{ + __raw_writel(0x0000ff3e, S5P_WAKEUP_MASK); +} + +static unsigned int g_pwr_ctrl, g_diag_reg; + +static void save_cpu_arch_register(void) +{ + /*read power control register*/ + asm("mrc p15, 0, %0, c15, c0, 0" : "=r"(g_pwr_ctrl) : : "cc"); + /*read diagnostic register*/ + asm("mrc p15, 0, %0, c15, c0, 1" : "=r"(g_diag_reg) : : "cc"); + return; +} + +static void restore_cpu_arch_register(void) +{ + /*write power control register*/ + asm("mcr p15, 0, %0, c15, c0, 0" : : "r"(g_pwr_ctrl) : "cc"); + /*write diagnostic register*/ + asm("mcr p15, 0, %0, c15, c0, 1" : : "r"(g_diag_reg) : "cc"); + return; +} + +static int idle_finisher(unsigned long flags) +{ + cpu_do_idle(); + return 1; +} + +static int exynos4_enter_core0_aftr(struct cpuidle_device *dev, + struct cpuidle_state *state) +{ + struct timeval before, after; + int idle_time; + unsigned long tmp; + + local_irq_disable(); + do_gettimeofday(&before); + + exynos4_set_wakeupmask(); + + /* Set value of power down register for aftr mode */ + exynos4_sys_powerdown_conf(SYS_AFTR); + + save_cpu_arch_register(); + + /* Setting Central Sequence Register for power down mode */ + tmp = __raw_readl(S5P_CENTRAL_SEQ_CONFIGURATION); + tmp &= ~S5P_CENTRAL_LOWPWR_CFG; + __raw_writel(tmp, S5P_CENTRAL_SEQ_CONFIGURATION); + + cpu_pm_enter(); + cpu_cluster_pm_enter(); + + cpu_suspend(0, idle_finisher); + + scu_enable(S5P_VA_SCU); + + cpu_cluster_pm_exit(); + cpu_pm_exit(); + + restore_cpu_arch_register(); + + /* + * If PMU failed while entering sleep mode, WFI will be + * ignored by PMU and then exiting cpu_do_idle(). + * S5P_CENTRAL_LOWPWR_CFG bit will not be set automatically + * in this situation. + */ + tmp = __raw_readl(S5P_CENTRAL_SEQ_CONFIGURATION); + if (!(tmp & S5P_CENTRAL_LOWPWR_CFG)) { + tmp |= S5P_CENTRAL_LOWPWR_CFG; + __raw_writel(tmp, S5P_CENTRAL_SEQ_CONFIGURATION); + } + + /* Clear wakeup state register */ + __raw_writel(0x0, S5P_WAKEUP_STAT); + + do_gettimeofday(&after); + + local_irq_enable(); + idle_time = (after.tv_sec - before.tv_sec) * USEC_PER_SEC + + (after.tv_usec - before.tv_usec); + + return idle_time; +} + static int exynos4_enter_idle(struct cpuidle_device *dev, struct cpuidle_state *state) { @@ -55,6 +171,26 @@ static int exynos4_enter_idle(struct cpuidle_device *dev, return idle_time; } +static int exynos4_enter_lowpower(struct cpuidle_device *dev, + struct cpuidle_state *state) +{ + struct cpuidle_state *new_state = state; + + /* This mode only can be entered when Core1 is offline */ + if (num_online_cpus() > 1) { + BUG_ON(!dev->safe_state); + new_state = dev->safe_state; + } + dev->last_state = new_state; + + if (new_state == &dev->states[0]) + return exynos4_enter_idle(dev, new_state); + else + return exynos4_enter_core0_aftr(dev, new_state); + + return exynos4_enter_idle(dev, new_state); +} + static int __init exynos4_init_cpuidle(void) { int i, max_cpuidle_state, cpu_id; @@ -66,8 +202,11 @@ static int __init exynos4_init_cpuidle(void) device = &per_cpu(exynos4_cpuidle_device, cpu_id); device->cpu = cpu_id; - device->state_count = (sizeof(exynos4_cpuidle_set) / + if (cpu_id == 0) + device->state_count = (sizeof(exynos4_cpuidle_set) / sizeof(struct cpuidle_state)); + else + device->state_count = 1; /* Support IDLE only */ max_cpuidle_state = device->state_count; @@ -76,11 +215,18 @@ static int __init exynos4_init_cpuidle(void) sizeof(struct cpuidle_state)); } + device->safe_state = &device->states[0]; + if (cpuidle_register_device(device)) { printk(KERN_ERR "CPUidle register device failed\n,"); return -EIO; } } + + __raw_writel(BSYM(virt_to_phys(s3c_cpu_resume)), + REG_DIRECTGO_ADDR); + __raw_writel(0xfcba0d10, REG_DIRECTGO_FLAG); + return 0; } device_initcall(exynos4_init_cpuidle); diff --git a/arch/arm/mach-exynos4/include/mach/pmu.h b/arch/arm/mach-exynos4/include/mach/pmu.h index a952904b010..31d1ed8e3c6 100644 --- a/arch/arm/mach-exynos4/include/mach/pmu.h +++ b/arch/arm/mach-exynos4/include/mach/pmu.h @@ -20,6 +20,8 @@ enum sys_powerdown { NUM_SYS_POWERDOWN, }; +extern unsigned long l2x0_regs_phys; extern void exynos4_sys_powerdown_conf(enum sys_powerdown mode); +extern void s3c_cpu_resume(void); #endif /* __ASM_ARCH_PMU_H */ diff --git a/arch/arm/mach-exynos4/platsmp.c b/arch/arm/mach-exynos4/platsmp.c index df6ef1b2f98..ca01370840d 100644 --- a/arch/arm/mach-exynos4/platsmp.c +++ b/arch/arm/mach-exynos4/platsmp.c @@ -30,9 +30,11 @@ #include <mach/regs-clock.h> #include <mach/regs-pmu.h> +#include <plat/exynos4.h> + extern void exynos4_secondary_startup(void); -#define CPU1_BOOT_REG S5P_VA_SYSRAM +#define CPU1_BOOT_REG (exynos4_subrev() == 0 ? S5P_VA_SYSRAM : S5P_INFORM5) /* * control for which core is the next to come out of the secondary @@ -218,5 +220,6 @@ void __init platform_smp_prepare_cpus(unsigned int max_cpus) * until it receives a soft interrupt, and then the * secondary CPU branches to this address. */ - __raw_writel(BSYM(virt_to_phys(exynos4_secondary_startup)), S5P_VA_SYSRAM); + __raw_writel(BSYM(virt_to_phys(exynos4_secondary_startup)), + CPU1_BOOT_REG); } diff --git a/arch/arm/mach-exynos4/pm.c b/arch/arm/mach-exynos4/pm.c index bc6ca9482de..47d5428887d 100644 --- a/arch/arm/mach-exynos4/pm.c +++ b/arch/arm/mach-exynos4/pm.c @@ -131,78 +131,6 @@ static struct sleep_save exynos4_core_save[] = { SAVE_ITEM(S5P_CLKGATE_SCLKCPU), SAVE_ITEM(S5P_CLKGATE_IP_CPU), - /* GIC side */ - SAVE_ITEM(S5P_VA_GIC_CPU + 0x000), - SAVE_ITEM(S5P_VA_GIC_CPU + 0x004), - SAVE_ITEM(S5P_VA_GIC_CPU + 0x008), - SAVE_ITEM(S5P_VA_GIC_CPU + 0x00C), - SAVE_ITEM(S5P_VA_GIC_CPU + 0x014), - SAVE_ITEM(S5P_VA_GIC_CPU + 0x018), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x000), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x004), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x100), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x104), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x108), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x300), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x304), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x308), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x400), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x404), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x408), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x40C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x410), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x414), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x418), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x41C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x420), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x424), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x428), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x42C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x430), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x434), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x438), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x43C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x440), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x444), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x448), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x44C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x450), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x454), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x458), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x45C), - - SAVE_ITEM(S5P_VA_GIC_DIST + 0x800), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x804), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x808), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x80C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x810), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x814), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x818), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x81C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x820), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x824), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x828), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x82C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x830), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x834), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x838), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x83C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x840), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x844), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x848), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x84C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x850), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x854), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x858), - SAVE_ITEM(S5P_VA_GIC_DIST + 0x85C), - - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC00), - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC04), - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC08), - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC0C), - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC10), - SAVE_ITEM(S5P_VA_GIC_DIST + 0xC14), - SAVE_ITEM(S5P_VA_COMBINER_BASE + 0x000), SAVE_ITEM(S5P_VA_COMBINER_BASE + 0x010), SAVE_ITEM(S5P_VA_COMBINER_BASE + 0x020), @@ -222,13 +150,6 @@ static struct sleep_save exynos4_core_save[] = { SAVE_ITEM(S5P_SROM_BC3), }; -static struct sleep_save exynos4_l2cc_save[] = { - SAVE_ITEM(S5P_VA_L2CC + L2X0_TAG_LATENCY_CTRL), - SAVE_ITEM(S5P_VA_L2CC + L2X0_DATA_LATENCY_CTRL), - SAVE_ITEM(S5P_VA_L2CC + L2X0_PREFETCH_CTRL), - SAVE_ITEM(S5P_VA_L2CC + L2X0_POWER_CTRL), - SAVE_ITEM(S5P_VA_L2CC + L2X0_AUX_CTRL), -}; /* For Cortex-A9 Diagnostic and Power control register */ static unsigned int save_arm_register[2]; @@ -249,7 +170,6 @@ static void exynos4_pm_prepare(void) u32 tmp; s3c_pm_do_save(exynos4_core_save, ARRAY_SIZE(exynos4_core_save)); - s3c_pm_do_save(exynos4_l2cc_save, ARRAY_SIZE(exynos4_l2cc_save)); s3c_pm_do_save(exynos4_epll_save, ARRAY_SIZE(exynos4_epll_save)); s3c_pm_do_save(exynos4_vpll_save, ARRAY_SIZE(exynos4_vpll_save)); @@ -462,13 +382,6 @@ static void exynos4_pm_resume(void) exynos4_scu_enable(S5P_VA_SCU); -#ifdef CONFIG_CACHE_L2X0 - s3c_pm_do_restore_core(exynos4_l2cc_save, ARRAY_SIZE(exynos4_l2cc_save)); - outer_inv_all(); - /* enable L2X0*/ - writel_relaxed(1, S5P_VA_L2CC + L2X0_CTRL); -#endif - early_wakeup: return; } diff --git a/arch/arm/mach-exynos4/sleep.S b/arch/arm/mach-exynos4/sleep.S index 0984078f1eb..3284213a1a3 100644 --- a/arch/arm/mach-exynos4/sleep.S +++ b/arch/arm/mach-exynos4/sleep.S @@ -27,8 +27,8 @@ */ #include <linux/linkage.h> -#include <asm/assembler.h> -#include <asm/memory.h> +#include <asm/asm-offsets.h> +#include <asm/hardware/cache-l2x0.h> .text @@ -49,6 +49,31 @@ * other way of restoring the stack pointer after sleep, and we * must not write to the code segment (code is read-only) */ + .align + .data ENTRY(s3c_cpu_resume) + adr r0, l2x0_regs_phys + ldr r0, [r0] + ldr r1, [r0, #L2X0_R_PHY_BASE] + ldr r2, [r1, #L2X0_CTRL] + tst r2, #0x1 + bne resume_l2on + ldr r2, [r0, #L2X0_R_AUX_CTRL] + str r2, [r1, #L2X0_AUX_CTRL] + ldr r2, [r0, #L2X0_R_TAG_LATENCY] + str r2, [r1, #L2X0_TAG_LATENCY_CTRL] + ldr r2, [r0, #L2X0_R_DATA_LATENCY] + str r2, [r1, #L2X0_DATA_LATENCY_CTRL] + ldr r2, [r0, #L2X0_R_PREFETCH_CTRL] + str r2, [r1, #L2X0_PREFETCH_CTRL] + ldr r2, [r0, #L2X0_R_PWR_CTRL] + str r2, [r1, #L2X0_POWER_CTRL] + mov r2, #1 + str r2, [r1, #L2X0_CTRL] +resume_l2on: b cpu_resume +ENDPROC(s3c_cpu_resume) + .globl l2x0_regs_phys +l2x0_regs_phys: + .long 0 diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 88633fe01a5..0e15dd9fbee 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -629,6 +629,23 @@ config IO_36 comment "Processor Features" +config ARM_LPAE + bool "Support for the Large Physical Address Extension" + depends on MMU && CPU_V7 + help + Say Y if you have an ARMv7 processor supporting the LPAE page + table format and you would like to access memory beyond the + 4GB limit. The resulting kernel image will not run on + processors without the LPA extension. + + If unsure, say N. + +config ARCH_PHYS_ADDR_T_64BIT + def_bool ARM_LPAE + +config ARCH_DMA_ADDR_T_64BIT + def_bool ARM_LPAE + config ARM_THUMB bool "Support Thumb user binaries" depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V6K || CPU_V7 || CPU_FEROCEON diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index cfbcf8b9559..715eb1dbebb 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -968,7 +968,7 @@ static int __init alignment_init(void) ai_usermode = safe_usermode(ai_usermode, false); } - hook_fault_code(1, do_alignment, SIGBUS, BUS_ADRALN, + hook_fault_code(FAULT_CODE_ALIGNMENT, do_alignment, SIGBUS, BUS_ADRALN, "alignment exception"); /* diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 9ecfdb51195..3f9b9980478 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -16,9 +16,12 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ +#include <linux/err.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/io.h> +#include <linux/of.h> +#include <linux/of_address.h> #include <asm/cacheflush.h> #include <asm/hardware/cache-l2x0.h> @@ -30,11 +33,19 @@ static DEFINE_SPINLOCK(l2x0_lock); static uint32_t l2x0_way_mask; /* Bitmask of active ways */ static uint32_t l2x0_size; +struct l2x0_regs l2x0_saved_regs; + +struct l2x0_of_data { + void (*setup)(const struct device_node *, __u32 *, __u32 *); + void (*save)(void); + void (*resume)(void); +}; + static inline void cache_wait_way(void __iomem *reg, unsigned long mask) { /* wait for cache operation by line or way to complete */ while (readl_relaxed(reg) & mask) - ; + cpu_relax(); } #ifdef CONFIG_CACHE_PL310 @@ -277,7 +288,7 @@ static void l2x0_disable(void) spin_unlock_irqrestore(&l2x0_lock, flags); } -static void __init l2x0_unlock(__u32 cache_id) +static void l2x0_unlock(__u32 cache_id) { int lockregs; int i; @@ -353,6 +364,8 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) /* l2x0 controller is disabled */ writel_relaxed(aux, l2x0_base + L2X0_AUX_CTRL); + l2x0_saved_regs.aux_ctrl = aux; + l2x0_inv_all(); /* enable L2X0 */ @@ -372,3 +385,202 @@ void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) printk(KERN_INFO "l2x0: %d ways, CACHE_ID 0x%08x, AUX_CTRL 0x%08x, Cache size: %d B\n", ways, cache_id, aux, l2x0_size); } + +#ifdef CONFIG_OF +static void __init l2x0_of_setup(const struct device_node *np, + __u32 *aux_val, __u32 *aux_mask) +{ + u32 data[2] = { 0, 0 }; + u32 tag = 0; + u32 dirty = 0; + u32 val = 0, mask = 0; + + of_property_read_u32(np, "arm,tag-latency", &tag); + if (tag) { + mask |= L2X0_AUX_CTRL_TAG_LATENCY_MASK; + val |= (tag - 1) << L2X0_AUX_CTRL_TAG_LATENCY_SHIFT; + } + + of_property_read_u32_array(np, "arm,data-latency", + data, ARRAY_SIZE(data)); + if (data[0] && data[1]) { + mask |= L2X0_AUX_CTRL_DATA_RD_LATENCY_MASK | + L2X0_AUX_CTRL_DATA_WR_LATENCY_MASK; + val |= ((data[0] - 1) << L2X0_AUX_CTRL_DATA_RD_LATENCY_SHIFT) | + ((data[1] - 1) << L2X0_AUX_CTRL_DATA_WR_LATENCY_SHIFT); + } + + of_property_read_u32(np, "arm,dirty-latency", &dirty); + if (dirty) { + mask |= L2X0_AUX_CTRL_DIRTY_LATENCY_MASK; + val |= (dirty - 1) << L2X0_AUX_CTRL_DIRTY_LATENCY_SHIFT; + } + + *aux_val &= ~mask; + *aux_val |= val; + *aux_mask &= ~mask; +} + +static void __init pl310_of_setup(const struct device_node *np, + __u32 *aux_val, __u32 *aux_mask) +{ + u32 data[3] = { 0, 0, 0 }; + u32 tag[3] = { 0, 0, 0 }; + u32 filter[2] = { 0, 0 }; + + of_property_read_u32_array(np, "arm,tag-latency", tag, ARRAY_SIZE(tag)); + if (tag[0] && tag[1] && tag[2]) + writel_relaxed( + ((tag[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) | + ((tag[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) | + ((tag[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT), + l2x0_base + L2X0_TAG_LATENCY_CTRL); + + of_property_read_u32_array(np, "arm,data-latency", + data, ARRAY_SIZE(data)); + if (data[0] && data[1] && data[2]) + writel_relaxed( + ((data[0] - 1) << L2X0_LATENCY_CTRL_RD_SHIFT) | + ((data[1] - 1) << L2X0_LATENCY_CTRL_WR_SHIFT) | + ((data[2] - 1) << L2X0_LATENCY_CTRL_SETUP_SHIFT), + l2x0_base + L2X0_DATA_LATENCY_CTRL); + + of_property_read_u32_array(np, "arm,filter-ranges", + filter, ARRAY_SIZE(filter)); + if (filter[1]) { + writel_relaxed(ALIGN(filter[0] + filter[1], SZ_1M), + l2x0_base + L2X0_ADDR_FILTER_END); + writel_relaxed((filter[0] & ~(SZ_1M - 1)) | L2X0_ADDR_FILTER_EN, + l2x0_base + L2X0_ADDR_FILTER_START); + } +} + +static void __init pl310_save(void) +{ + u32 l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) & + L2X0_CACHE_ID_RTL_MASK; + + l2x0_saved_regs.tag_latency = readl_relaxed(l2x0_base + + L2X0_TAG_LATENCY_CTRL); + l2x0_saved_regs.data_latency = readl_relaxed(l2x0_base + + L2X0_DATA_LATENCY_CTRL); + l2x0_saved_regs.filter_end = readl_relaxed(l2x0_base + + L2X0_ADDR_FILTER_END); + l2x0_saved_regs.filter_start = readl_relaxed(l2x0_base + + L2X0_ADDR_FILTER_START); + + if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) { + /* + * From r2p0, there is Prefetch offset/control register + */ + l2x0_saved_regs.prefetch_ctrl = readl_relaxed(l2x0_base + + L2X0_PREFETCH_CTRL); + /* + * From r3p0, there is Power control register + */ + if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0) + l2x0_saved_regs.pwr_ctrl = readl_relaxed(l2x0_base + + L2X0_POWER_CTRL); + } +} + +static void l2x0_resume(void) +{ + if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { + /* restore aux ctrl and enable l2 */ + l2x0_unlock(readl_relaxed(l2x0_base + L2X0_CACHE_ID)); + + writel_relaxed(l2x0_saved_regs.aux_ctrl, l2x0_base + + L2X0_AUX_CTRL); + + l2x0_inv_all(); + + writel_relaxed(1, l2x0_base + L2X0_CTRL); + } +} + +static void pl310_resume(void) +{ + u32 l2x0_revision; + + if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { + /* restore pl310 setup */ + writel_relaxed(l2x0_saved_regs.tag_latency, + l2x0_base + L2X0_TAG_LATENCY_CTRL); + writel_relaxed(l2x0_saved_regs.data_latency, + l2x0_base + L2X0_DATA_LATENCY_CTRL); + writel_relaxed(l2x0_saved_regs.filter_end, + l2x0_base + L2X0_ADDR_FILTER_END); + writel_relaxed(l2x0_saved_regs.filter_start, + l2x0_base + L2X0_ADDR_FILTER_START); + + l2x0_revision = readl_relaxed(l2x0_base + L2X0_CACHE_ID) & + L2X0_CACHE_ID_RTL_MASK; + + if (l2x0_revision >= L2X0_CACHE_ID_RTL_R2P0) { + writel_relaxed(l2x0_saved_regs.prefetch_ctrl, + l2x0_base + L2X0_PREFETCH_CTRL); + if (l2x0_revision >= L2X0_CACHE_ID_RTL_R3P0) + writel_relaxed(l2x0_saved_regs.pwr_ctrl, + l2x0_base + L2X0_POWER_CTRL); + } + } + + l2x0_resume(); +} + +static const struct l2x0_of_data pl310_data = { + pl310_of_setup, + pl310_save, + pl310_resume, +}; + +static const struct l2x0_of_data l2x0_data = { + l2x0_of_setup, + NULL, + l2x0_resume, +}; + +static const struct of_device_id l2x0_ids[] __initconst = { + { .compatible = "arm,pl310-cache", .data = (void *)&pl310_data }, + { .compatible = "arm,l220-cache", .data = (void *)&l2x0_data }, + { .compatible = "arm,l210-cache", .data = (void *)&l2x0_data }, + {} +}; + +int __init l2x0_of_init(__u32 aux_val, __u32 aux_mask) +{ + struct device_node *np; + struct l2x0_of_data *data; + struct resource res; + + np = of_find_matching_node(NULL, l2x0_ids); + if (!np) + return -ENODEV; + + if (of_address_to_resource(np, 0, &res)) + return -ENODEV; + + l2x0_base = ioremap(res.start, resource_size(&res)); + if (!l2x0_base) + return -ENOMEM; + + l2x0_saved_regs.phy_base = res.start; + + data = of_match_node(l2x0_ids, np)->data; + + /* L2 configuration can only be changed if the cache is disabled */ + if (!(readl_relaxed(l2x0_base + L2X0_CTRL) & 1)) { + if (data->setup) + data->setup(np, &aux_val, &aux_mask); + } + + if (data->save) + data->save(); + + l2x0_init(l2x0_base, aux_val, aux_mask); + + outer_cache.resume = data->resume; + return 0; +} +#endif diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index b0ee9ba3cfa..fcdb1017cdc 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -22,6 +22,21 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION; DEFINE_PER_CPU(struct mm_struct *, current_mm); #endif +#ifdef CONFIG_ARM_LPAE +#define cpu_set_asid(asid) { \ + unsigned long ttbl, ttbh; \ + asm volatile( \ + " mrrc p15, 0, %0, %1, c2 @ read TTBR0\n" \ + " mov %1, %2, lsl #(48 - 32) @ set ASID\n" \ + " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" \ + : "=&r" (ttbl), "=&r" (ttbh) \ + : "r" (asid & ~ASID_MASK)); \ +} +#else +#define cpu_set_asid(asid) \ + asm(" mcr p15, 0, %0, c13, c0, 1\n" : : "r" (asid)) +#endif + /* * We fork()ed a process, and we need a new context for the child * to run in. We reserve version 0 for initial tasks so we will @@ -37,7 +52,7 @@ void __init_new_context(struct task_struct *tsk, struct mm_struct *mm) static void flush_context(void) { /* set the reserved ASID before flushing the TLB */ - asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (0)); + cpu_set_asid(0); isb(); local_flush_tlb_all(); if (icache_is_vivt_asid_tagged()) { @@ -99,7 +114,7 @@ static void reset_context(void *info) set_mm_context(mm, asid); /* set the new ASID */ - asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id)); + cpu_set_asid(mm->context.id); isb(); } diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 01f5987eb1a..e4e7f6cba1a 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -120,9 +120,8 @@ static void __dma_free_buffer(struct page *page, size_t size) #ifdef CONFIG_MMU - #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - consistent_base) >> PAGE_SHIFT) -#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PGDIR_SHIFT) +#define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - consistent_base) >> PMD_SHIFT) /* * These are the page tables (2MB each) covering uncached, DMA consistent allocations @@ -206,7 +205,7 @@ static int __init consistent_init(void) } consistent_pte[i++] = pte; - base += (1 << PGDIR_SHIFT); + base += PMD_SIZE; } while (base < CONSISTENT_END); return ret; diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 3b5ea68acbb..05588bc1d22 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -26,19 +26,6 @@ #include "fault.h" -/* - * Fault status register encodings. We steal bit 31 for our own purposes. - */ -#define FSR_LNX_PF (1 << 31) -#define FSR_WRITE (1 << 11) -#define FSR_FS4 (1 << 10) -#define FSR_FS3_0 (15) - -static inline int fsr_fs(unsigned int fsr) -{ - return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6; -} - #ifdef CONFIG_MMU #ifdef CONFIG_KPROBES @@ -122,8 +109,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr) pte = pte_offset_map(pmd, addr); printk(", *pte=%08llx", (long long)pte_val(*pte)); +#ifndef CONFIG_ARM_LPAE printk(", *ppte=%08llx", (long long)pte_val(pte[PTE_HWTABLE_PTRS])); +#endif pte_unmap(pte); } while(0); @@ -440,6 +429,12 @@ do_translation_fault(unsigned long addr, unsigned int fsr, pmd = pmd_offset(pud, addr); pmd_k = pmd_offset(pud_k, addr); +#ifdef CONFIG_ARM_LPAE + /* + * Only one hardware entry per PMD with LPAE. + */ + index = 0; +#else /* * On ARM one Linux PGD entry contains two hardware entries (see page * tables layout in pgtable.h). We normally guarantee that we always @@ -449,6 +444,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr, * for the first of pair. */ index = (addr >> SECTION_SHIFT) & 1; +#endif if (pmd_none(pmd_k[index])) goto bad_area; @@ -488,55 +484,20 @@ do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) return 1; } -static struct fsr_info { +struct fsr_info { int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs); int sig; int code; const char *name; -} fsr_info[] = { - /* - * The following are the standard ARMv3 and ARMv4 aborts. ARMv5 - * defines these to be "precise" aborts. - */ - { do_bad, SIGSEGV, 0, "vector exception" }, - { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" }, - { do_bad, SIGKILL, 0, "terminal exception" }, - { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" }, - { do_bad, SIGBUS, 0, "external abort on linefetch" }, - { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, - { do_bad, SIGBUS, 0, "external abort on linefetch" }, - { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, - { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" }, - { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, - { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, - { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, - /* - * The following are "imprecise" aborts, which are signalled by bit - * 10 of the FSR, and may not be recoverable. These are only - * supported if the CPU abort handler supports bit 10. - */ - { do_bad, SIGBUS, 0, "unknown 16" }, - { do_bad, SIGBUS, 0, "unknown 17" }, - { do_bad, SIGBUS, 0, "unknown 18" }, - { do_bad, SIGBUS, 0, "unknown 19" }, - { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */ - { do_bad, SIGBUS, 0, "unknown 21" }, - { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */ - { do_bad, SIGBUS, 0, "unknown 23" }, - { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */ - { do_bad, SIGBUS, 0, "unknown 25" }, - { do_bad, SIGBUS, 0, "unknown 26" }, - { do_bad, SIGBUS, 0, "unknown 27" }, - { do_bad, SIGBUS, 0, "unknown 28" }, - { do_bad, SIGBUS, 0, "unknown 29" }, - { do_bad, SIGBUS, 0, "unknown 30" }, - { do_bad, SIGBUS, 0, "unknown 31" } }; +/* FSR definition */ +#ifdef CONFIG_ARM_LPAE +#include "fsr-3level.c" +#else +#include "fsr-2level.c" +#endif + void __init hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), int sig, int code, const char *name) @@ -572,42 +533,6 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) arm_notify_die("", regs, &info, fsr, 0); } - -static struct fsr_info ifsr_info[] = { - { do_bad, SIGBUS, 0, "unknown 0" }, - { do_bad, SIGBUS, 0, "unknown 1" }, - { do_bad, SIGBUS, 0, "debug event" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" }, - { do_bad, SIGBUS, 0, "unknown 4" }, - { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" }, - { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, - { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" }, - { do_bad, SIGBUS, 0, "unknown 10" }, - { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, - { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, - { do_bad, SIGBUS, 0, "external abort on translation" }, - { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, - { do_bad, SIGBUS, 0, "unknown 16" }, - { do_bad, SIGBUS, 0, "unknown 17" }, - { do_bad, SIGBUS, 0, "unknown 18" }, - { do_bad, SIGBUS, 0, "unknown 19" }, - { do_bad, SIGBUS, 0, "unknown 20" }, - { do_bad, SIGBUS, 0, "unknown 21" }, - { do_bad, SIGBUS, 0, "unknown 22" }, - { do_bad, SIGBUS, 0, "unknown 23" }, - { do_bad, SIGBUS, 0, "unknown 24" }, - { do_bad, SIGBUS, 0, "unknown 25" }, - { do_bad, SIGBUS, 0, "unknown 26" }, - { do_bad, SIGBUS, 0, "unknown 27" }, - { do_bad, SIGBUS, 0, "unknown 28" }, - { do_bad, SIGBUS, 0, "unknown 29" }, - { do_bad, SIGBUS, 0, "unknown 30" }, - { do_bad, SIGBUS, 0, "unknown 31" }, -}; - void __init hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), int sig, int code, const char *name) @@ -640,6 +565,7 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) arm_notify_die("", regs, &info, ifsr, 0); } +#ifndef CONFIG_ARM_LPAE static int __init exceptions_init(void) { if (cpu_architecture() >= CPU_ARCH_ARMv6) { @@ -662,3 +588,4 @@ static int __init exceptions_init(void) } arch_initcall(exceptions_init); +#endif diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h index 49e9e3804de..cf08bdfbe0d 100644 --- a/arch/arm/mm/fault.h +++ b/arch/arm/mm/fault.h @@ -1,3 +1,28 @@ -void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs); +#ifndef __ARCH_ARM_FAULT_H +#define __ARCH_ARM_FAULT_H + +/* + * Fault status register encodings. We steal bit 31 for our own purposes. + */ +#define FSR_LNX_PF (1 << 31) +#define FSR_WRITE (1 << 11) +#define FSR_FS4 (1 << 10) +#define FSR_FS3_0 (15) +#define FSR_FS5_0 (0x3f) + +#ifdef CONFIG_ARM_LPAE +static inline int fsr_fs(unsigned int fsr) +{ + return fsr & FSR_FS5_0; +} +#else +static inline int fsr_fs(unsigned int fsr) +{ + return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6; +} +#endif +void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs); unsigned long search_exception_table(unsigned long addr); + +#endif /* __ARCH_ARM_FAULT_H */ diff --git a/arch/arm/mm/fsr-2level.c b/arch/arm/mm/fsr-2level.c new file mode 100644 index 00000000000..9260894e394 --- /dev/null +++ b/arch/arm/mm/fsr-2level.c @@ -0,0 +1,78 @@ +static struct fsr_info fsr_info[] = { + /* + * The following are the standard ARMv3 and ARMv4 aborts. ARMv5 + * defines these to be "precise" aborts. + */ + { do_bad, SIGSEGV, 0, "vector exception" }, + { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" }, + { do_bad, SIGKILL, 0, "terminal exception" }, + { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" }, + { do_bad, SIGBUS, 0, "external abort on linefetch" }, + { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, + { do_bad, SIGBUS, 0, "external abort on linefetch" }, + { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, + { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" }, + { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, + { do_bad, SIGBUS, 0, "external abort on translation" }, + { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_bad, SIGBUS, 0, "external abort on translation" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, + /* + * The following are "imprecise" aborts, which are signalled by bit + * 10 of the FSR, and may not be recoverable. These are only + * supported if the CPU abort handler supports bit 10. + */ + { do_bad, SIGBUS, 0, "unknown 16" }, + { do_bad, SIGBUS, 0, "unknown 17" }, + { do_bad, SIGBUS, 0, "unknown 18" }, + { do_bad, SIGBUS, 0, "unknown 19" }, + { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */ + { do_bad, SIGBUS, 0, "unknown 21" }, + { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */ + { do_bad, SIGBUS, 0, "unknown 23" }, + { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */ + { do_bad, SIGBUS, 0, "unknown 25" }, + { do_bad, SIGBUS, 0, "unknown 26" }, + { do_bad, SIGBUS, 0, "unknown 27" }, + { do_bad, SIGBUS, 0, "unknown 28" }, + { do_bad, SIGBUS, 0, "unknown 29" }, + { do_bad, SIGBUS, 0, "unknown 30" }, + { do_bad, SIGBUS, 0, "unknown 31" } +}; + +static struct fsr_info ifsr_info[] = { + { do_bad, SIGBUS, 0, "unknown 0" }, + { do_bad, SIGBUS, 0, "unknown 1" }, + { do_bad, SIGBUS, 0, "debug event" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" }, + { do_bad, SIGBUS, 0, "unknown 4" }, + { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" }, + { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" }, + { do_bad, SIGBUS, 0, "external abort on non-linefetch" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" }, + { do_bad, SIGBUS, 0, "unknown 10" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" }, + { do_bad, SIGBUS, 0, "external abort on translation" }, + { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" }, + { do_bad, SIGBUS, 0, "external abort on translation" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" }, + { do_bad, SIGBUS, 0, "unknown 16" }, + { do_bad, SIGBUS, 0, "unknown 17" }, + { do_bad, SIGBUS, 0, "unknown 18" }, + { do_bad, SIGBUS, 0, "unknown 19" }, + { do_bad, SIGBUS, 0, "unknown 20" }, + { do_bad, SIGBUS, 0, "unknown 21" }, + { do_bad, SIGBUS, 0, "unknown 22" }, + { do_bad, SIGBUS, 0, "unknown 23" }, + { do_bad, SIGBUS, 0, "unknown 24" }, + { do_bad, SIGBUS, 0, "unknown 25" }, + { do_bad, SIGBUS, 0, "unknown 26" }, + { do_bad, SIGBUS, 0, "unknown 27" }, + { do_bad, SIGBUS, 0, "unknown 28" }, + { do_bad, SIGBUS, 0, "unknown 29" }, + { do_bad, SIGBUS, 0, "unknown 30" }, + { do_bad, SIGBUS, 0, "unknown 31" }, +}; diff --git a/arch/arm/mm/fsr-3level.c b/arch/arm/mm/fsr-3level.c new file mode 100644 index 00000000000..05a4e943183 --- /dev/null +++ b/arch/arm/mm/fsr-3level.c @@ -0,0 +1,68 @@ +static struct fsr_info fsr_info[] = { + { do_bad, SIGBUS, 0, "unknown 0" }, + { do_bad, SIGBUS, 0, "unknown 1" }, + { do_bad, SIGBUS, 0, "unknown 2" }, + { do_bad, SIGBUS, 0, "unknown 3" }, + { do_bad, SIGBUS, 0, "reserved translation fault" }, + { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, + { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, + { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, + { do_bad, SIGBUS, 0, "reserved access flag fault" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, + { do_bad, SIGBUS, 0, "reserved permission fault" }, + { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, + { do_sect_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, + { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, + { do_bad, SIGBUS, 0, "synchronous external abort" }, + { do_bad, SIGBUS, 0, "asynchronous external abort" }, + { do_bad, SIGBUS, 0, "unknown 18" }, + { do_bad, SIGBUS, 0, "unknown 19" }, + { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" }, + { do_bad, SIGBUS, 0, "synchronous parity error" }, + { do_bad, SIGBUS, 0, "asynchronous parity error" }, + { do_bad, SIGBUS, 0, "unknown 26" }, + { do_bad, SIGBUS, 0, "unknown 27" }, + { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" }, + { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" }, + { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" }, + { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" }, + { do_bad, SIGBUS, 0, "unknown 32" }, + { do_bad, SIGBUS, BUS_ADRALN, "alignment fault" }, + { do_bad, SIGBUS, 0, "debug event" }, + { do_bad, SIGBUS, 0, "unknown 35" }, + { do_bad, SIGBUS, 0, "unknown 36" }, + { do_bad, SIGBUS, 0, "unknown 37" }, + { do_bad, SIGBUS, 0, "unknown 38" }, + { do_bad, SIGBUS, 0, "unknown 39" }, + { do_bad, SIGBUS, 0, "unknown 40" }, + { do_bad, SIGBUS, 0, "unknown 41" }, + { do_bad, SIGBUS, 0, "unknown 42" }, + { do_bad, SIGBUS, 0, "unknown 43" }, + { do_bad, SIGBUS, 0, "unknown 44" }, + { do_bad, SIGBUS, 0, "unknown 45" }, + { do_bad, SIGBUS, 0, "unknown 46" }, + { do_bad, SIGBUS, 0, "unknown 47" }, + { do_bad, SIGBUS, 0, "unknown 48" }, + { do_bad, SIGBUS, 0, "unknown 49" }, + { do_bad, SIGBUS, 0, "unknown 50" }, + { do_bad, SIGBUS, 0, "unknown 51" }, + { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" }, + { do_bad, SIGBUS, 0, "unknown 53" }, + { do_bad, SIGBUS, 0, "unknown 54" }, + { do_bad, SIGBUS, 0, "unknown 55" }, + { do_bad, SIGBUS, 0, "unknown 56" }, + { do_bad, SIGBUS, 0, "unknown 57" }, + { do_bad, SIGBUS, 0, "implementation fault (coprocessor abort)" }, + { do_bad, SIGBUS, 0, "unknown 59" }, + { do_bad, SIGBUS, 0, "unknown 60" }, + { do_bad, SIGBUS, 0, "unknown 61" }, + { do_bad, SIGBUS, 0, "unknown 62" }, + { do_bad, SIGBUS, 0, "unknown 63" }, +}; + +#define ifsr_info fsr_info diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index 2be9139a4ef..24e06555bee 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -1,9 +1,36 @@ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/kernel.h> #include <asm/cputype.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> +#ifdef CONFIG_ARM_LPAE +static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, + unsigned long prot) +{ + pmd_t *pmd; + unsigned long next; + + if (pud_none_or_clear_bad(pud) || (pud_val(*pud) & L_PGD_SWAPPER)) { + pmd = pmd_alloc_one(NULL, addr); + if (!pmd) { + pr_warning("Failed to allocate identity pmd.\n"); + return; + } + pud_populate(NULL, pud, pmd); + pmd += pmd_index(addr); + } else + pmd = pmd_offset(pud, addr); + + do { + next = pmd_addr_end(addr, end); + *pmd = __pmd((addr & PMD_MASK) | prot); + flush_pmd_entry(pmd); + } while (pmd++, addr = next, addr != end); +} +#else /* !CONFIG_ARM_LPAE */ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, unsigned long prot) { @@ -15,6 +42,7 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, pmd[1] = __pmd(addr); flush_pmd_entry(pmd); } +#endif /* CONFIG_ARM_LPAE */ static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, unsigned long prot) @@ -32,7 +60,7 @@ void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) { unsigned long prot, next; - prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE; + prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF; if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) prot |= PMD_BIT4; @@ -46,7 +74,11 @@ void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long end) #ifdef CONFIG_SMP static void idmap_del_pmd(pud_t *pud, unsigned long addr, unsigned long end) { - pmd_t *pmd = pmd_offset(pud, addr); + pmd_t *pmd; + + if (pud_none_or_clear_bad(pud)) + return; + pmd = pmd_offset(pud, addr); pmd_clear(pmd); } diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index ab506272b2d..645011bf8d4 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -64,7 +64,7 @@ void __check_kvm_seq(struct mm_struct *mm) } while (seq != init_mm.context.kvm_seq); } -#ifndef CONFIG_SMP +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) /* * Section support is unsafe on SMP - If you iounmap and ioremap a region, * the other CPUs will not see this change until their next context switch. @@ -79,13 +79,16 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) { unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1)); pgd_t *pgd; + pud_t *pud; + pmd_t *pmdp; flush_cache_vunmap(addr, end); pgd = pgd_offset_k(addr); + pud = pud_offset(pgd, addr); + pmdp = pmd_offset(pud, addr); do { - pmd_t pmd, *pmdp = pmd_offset(pgd, addr); + pmd_t pmd = *pmdp; - pmd = *pmdp; if (!pmd_none(pmd)) { /* * Clear the PMD from the page table, and @@ -104,8 +107,8 @@ static void unmap_area_sections(unsigned long virt, unsigned long size) pte_free_kernel(&init_mm, pmd_page_vaddr(pmd)); } - addr += PGDIR_SIZE; - pgd++; + addr += PMD_SIZE; + pmdp += 2; } while (addr < end); /* @@ -124,6 +127,8 @@ remap_area_sections(unsigned long virt, unsigned long pfn, { unsigned long addr = virt, end = virt + size; pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; /* * Remove and free any PTE-based mapping, and @@ -132,17 +137,17 @@ remap_area_sections(unsigned long virt, unsigned long pfn, unmap_area_sections(virt, size); pgd = pgd_offset_k(addr); + pud = pud_offset(pgd, addr); + pmd = pmd_offset(pud, addr); do { - pmd_t *pmd = pmd_offset(pgd, addr); - pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); pfn += SZ_1M >> PAGE_SHIFT; pmd[1] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); pfn += SZ_1M >> PAGE_SHIFT; flush_pmd_entry(pmd); - addr += PGDIR_SIZE; - pgd++; + addr += PMD_SIZE; + pmd += 2; } while (addr < end); return 0; @@ -154,6 +159,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, { unsigned long addr = virt, end = virt + size; pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; /* * Remove and free any PTE-based mapping, and @@ -162,6 +169,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, unmap_area_sections(virt, size); pgd = pgd_offset_k(virt); + pud = pud_offset(pgd, addr); + pmd = pmd_offset(pud, addr); do { unsigned long super_pmd_val, i; @@ -170,14 +179,12 @@ remap_area_supersections(unsigned long virt, unsigned long pfn, super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20; for (i = 0; i < 8; i++) { - pmd_t *pmd = pmd_offset(pgd, addr); - pmd[0] = __pmd(super_pmd_val); pmd[1] = __pmd(super_pmd_val); flush_pmd_entry(pmd); - addr += PGDIR_SIZE; - pgd++; + addr += PMD_SIZE; + pmd += 2; } pfn += SUPERSECTION_SIZE >> PAGE_SHIFT; @@ -195,11 +202,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, unsigned long addr; struct vm_struct * area; +#ifndef CONFIG_ARM_LPAE /* * High mappings must be supersection aligned */ if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK)) return NULL; +#endif /* * Don't allow RAM to be mapped - this causes problems with ARMv6+ @@ -221,7 +230,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, return NULL; addr = (unsigned long)area->addr; -#ifndef CONFIG_SMP +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) if (DOMAIN_IO == 0 && (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) || cpu_is_xsc3()) && pfn >= 0x100000 && @@ -292,7 +301,7 @@ EXPORT_SYMBOL(__arm_ioremap); void __iounmap(volatile void __iomem *io_addr) { void *addr = (void *)(PAGE_MASK & (unsigned long)io_addr); -#ifndef CONFIG_SMP +#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE) struct vm_struct **p, *tmp; /* diff --git a/arch/arm/mm/mm.h b/arch/arm/mm/mm.h index 010566799c8..ad7cce3bc43 100644 --- a/arch/arm/mm/mm.h +++ b/arch/arm/mm/mm.h @@ -12,8 +12,8 @@ static inline pmd_t *pmd_off_k(unsigned long virt) struct mem_type { pteval_t prot_pte; - unsigned int prot_l1; - unsigned int prot_sect; + pmdval_t prot_l1; + pmdval_t prot_sect; unsigned int domain; }; diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ea9c9f3e48b..b836d6b2258 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -60,7 +60,7 @@ EXPORT_SYMBOL(pgprot_kernel); struct cachepolicy { const char policy[16]; unsigned int cr_mask; - unsigned int pmd; + pmdval_t pmd; pteval_t pte; }; @@ -150,6 +150,7 @@ static int __init early_nowrite(char *__unused) } early_param("nowb", early_nowrite); +#ifndef CONFIG_ARM_LPAE static int __init early_ecc(char *p) { if (memcmp(p, "on", 2) == 0) @@ -159,6 +160,7 @@ static int __init early_ecc(char *p) return 0; } early_param("ecc", early_ecc); +#endif static int __init noalign_setup(char *__unused) { @@ -228,10 +230,12 @@ static struct mem_type mem_types[] = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, .domain = DOMAIN_KERNEL, }, +#ifndef CONFIG_ARM_LPAE [MT_MINICLEAN] = { .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE, .domain = DOMAIN_KERNEL, }, +#endif [MT_LOW_VECTORS] = { .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_RDONLY, @@ -296,7 +300,7 @@ static void __init build_mem_type_table(void) { struct cachepolicy *cp; unsigned int cr = get_cr(); - unsigned int user_pgprot, kern_pgprot, vecs_pgprot; + pteval_t user_pgprot, kern_pgprot, vecs_pgprot; int cpu_arch = cpu_architecture(); int i; @@ -429,6 +433,7 @@ static void __init build_mem_type_table(void) * ARMv6 and above have extended page tables. */ if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { +#ifndef CONFIG_ARM_LPAE /* * Mark cache clean areas and XIP ROM read only * from SVC mode and no access from userspace. @@ -436,6 +441,7 @@ static void __init build_mem_type_table(void) mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; +#endif if (is_smp()) { /* @@ -474,6 +480,18 @@ static void __init build_mem_type_table(void) mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE; } +#ifdef CONFIG_ARM_LPAE + /* + * Do not generate access flag faults for the kernel mappings. + */ + for (i = 0; i < ARRAY_SIZE(mem_types); i++) { + mem_types[i].prot_pte |= PTE_EXT_AF; + mem_types[i].prot_sect |= PMD_SECT_AF; + } + kern_pgprot |= PTE_EXT_AF; + vecs_pgprot |= PTE_EXT_AF; +#endif + for (i = 0; i < 16; i++) { unsigned long v = pgprot_val(protection_map[i]); protection_map[i] = __pgprot(v | user_pgprot); @@ -572,8 +590,10 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr, if (((addr | end | phys) & ~SECTION_MASK) == 0) { pmd_t *p = pmd; +#ifndef CONFIG_ARM_LPAE if (addr & SECTION_SIZE) pmd++; +#endif do { *pmd = __pmd(phys | type->prot_sect); @@ -603,6 +623,7 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end, } while (pud++, addr = next, addr != end); } +#ifndef CONFIG_ARM_LPAE static void __init create_36bit_mapping(struct map_desc *md, const struct mem_type *type) { @@ -662,6 +683,7 @@ static void __init create_36bit_mapping(struct map_desc *md, pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT; } while (addr != end); } +#endif /* !CONFIG_ARM_LPAE */ /* * Create the page directory entries and any necessary @@ -693,6 +715,7 @@ static void __init create_mapping(struct map_desc *md) type = &mem_types[md->type]; +#ifndef CONFIG_ARM_LPAE /* * Catch 36-bit addresses */ @@ -700,6 +723,7 @@ static void __init create_mapping(struct map_desc *md) create_36bit_mapping(md, type); return; } +#endif addr = md->virtual & PAGE_MASK; phys = __pfn_to_phys(md->pfn); @@ -775,6 +799,9 @@ void __init sanity_check_meminfo(void) struct membank *bank = &meminfo.bank[j]; *bank = meminfo.bank[i]; + if (bank->start > ULONG_MAX) + highmem = 1; + #ifdef CONFIG_HIGHMEM if (__va(bank->start) >= vmalloc_min || __va(bank->start) < (void *)PAGE_OFFSET) @@ -786,7 +813,7 @@ void __init sanity_check_meminfo(void) * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. */ - if (__va(bank->start) < vmalloc_min && + if (!highmem && __va(bank->start) < vmalloc_min && bank->size > vmalloc_min - __va(bank->start)) { if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " @@ -807,6 +834,17 @@ void __init sanity_check_meminfo(void) bank->highmem = highmem; /* + * Highmem banks not allowed with !CONFIG_HIGHMEM. + */ + if (highmem) { + printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " + "(!CONFIG_HIGHMEM).\n", + (unsigned long long)bank->start, + (unsigned long long)bank->start + bank->size - 1); + continue; + } + + /* * Check whether this memory bank would entirely overlap * the vmalloc area. */ @@ -871,14 +909,14 @@ static inline void prepare_page_table(void) /* * Clear out all the mappings below the kernel image. */ - for (addr = 0; addr < MODULES_VADDR; addr += PGDIR_SIZE) + for (addr = 0; addr < MODULES_VADDR; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); #ifdef CONFIG_XIP_KERNEL /* The XIP kernel is mapped in the module area -- skip over it */ - addr = ((unsigned long)_etext + PGDIR_SIZE - 1) & PGDIR_MASK; + addr = ((unsigned long)_etext + PMD_SIZE - 1) & PMD_MASK; #endif - for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE) + for ( ; addr < PAGE_OFFSET; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); /* @@ -893,10 +931,18 @@ static inline void prepare_page_table(void) * memory bank, up to the end of the vmalloc region. */ for (addr = __phys_to_virt(end); - addr < VMALLOC_END; addr += PGDIR_SIZE) + addr < VMALLOC_END; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); } +#ifdef CONFIG_ARM_LPAE +/* the first page is reserved for pgd */ +#define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \ + PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t)) +#else +#define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) +#endif + /* * Reserve the special regions of memory */ @@ -906,7 +952,7 @@ void __init arm_mm_memblock_reserve(void) * Reserve the page tables. These are already in use, * and can only be in node 0. */ - memblock_reserve(__pa(swapper_pg_dir), PTRS_PER_PGD * sizeof(pgd_t)); + memblock_reserve(__pa(swapper_pg_dir), SWAPPER_PG_DIR_SIZE); #ifdef CONFIG_SA1111 /* @@ -934,7 +980,7 @@ static void __init devicemaps_init(struct machine_desc *mdesc) */ vectors_page = early_alloc(PAGE_SIZE); - for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE) + for (addr = VMALLOC_END; addr; addr += PMD_SIZE) pmd_clear(pmd_off_k(addr)); /* diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index b2027c154b2..a3e78ccabd6 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -10,6 +10,7 @@ #include <linux/mm.h> #include <linux/gfp.h> #include <linux/highmem.h> +#include <linux/slab.h> #include <asm/pgalloc.h> #include <asm/page.h> @@ -17,6 +18,14 @@ #include "mm.h" +#ifdef CONFIG_ARM_LPAE +#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL) +#define __pgd_free(pgd) kfree(pgd) +#else +#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2) +#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) +#endif + /* * need to get a 16k page for level 1 */ @@ -27,7 +36,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm) pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; - new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2); + new_pgd = __pgd_alloc(); if (!new_pgd) goto no_pgd; @@ -42,10 +51,25 @@ pgd_t *pgd_alloc(struct mm_struct *mm) clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); +#ifdef CONFIG_ARM_LPAE + /* + * Allocate PMD table for modules and pkmap mappings. + */ + new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR), + MODULES_VADDR); + if (!new_pud) + goto no_pud; + + new_pmd = pmd_alloc(mm, new_pud, 0); + if (!new_pmd) + goto no_pmd; +#endif + if (!vectors_high()) { /* * On ARM, first page must always be allocated since it - * contains the machine vectors. + * contains the machine vectors. The vectors are always high + * with LPAE. */ new_pud = pud_alloc(mm, new_pgd, 0); if (!new_pud) @@ -74,7 +98,7 @@ no_pte: no_pmd: pud_free(mm, new_pud); no_pud: - free_pages((unsigned long)new_pgd, 2); + __pgd_free(new_pgd); no_pgd: return NULL; } @@ -111,5 +135,24 @@ no_pud: pgd_clear(pgd); pud_free(mm, pud); no_pgd: - free_pages((unsigned long) pgd_base, 2); +#ifdef CONFIG_ARM_LPAE + /* + * Free modules/pkmap or identity pmd tables. + */ + for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) { + if (pgd_none_or_clear_bad(pgd)) + continue; + if (pgd_val(*pgd) & L_PGD_SWAPPER) + continue; + pud = pud_offset(pgd, 0); + if (pud_none_or_clear_bad(pud)) + continue; + pmd = pmd_offset(pud, 0); + pud_clear(pud); + pmd_free(mm, pmd); + pgd_clear(pgd); + pud_free(mm, pud); + } +#endif + __pgd_free(pgd_base); } diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S index 307a4def8d3..2d8ff3ad86d 100644 --- a/arch/arm/mm/proc-macros.S +++ b/arch/arm/mm/proc-macros.S @@ -91,8 +91,9 @@ #if L_PTE_SHARED != PTE_EXT_SHARED #error PTE shared bit mismatch #endif -#if (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\ - L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED +#if !defined (CONFIG_ARM_LPAE) && \ + (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\ + L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED #error Invalid Linux PTE bit settings #endif #endif /* CONFIG_MMU */ diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S new file mode 100644 index 00000000000..3a4b3e7b888 --- /dev/null +++ b/arch/arm/mm/proc-v7-2level.S @@ -0,0 +1,171 @@ +/* + * arch/arm/mm/proc-v7-2level.S + * + * Copyright (C) 2001 Deep Blue Solutions Ltd. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#define TTB_S (1 << 1) +#define TTB_RGN_NC (0 << 3) +#define TTB_RGN_OC_WBWA (1 << 3) +#define TTB_RGN_OC_WT (2 << 3) +#define TTB_RGN_OC_WB (3 << 3) +#define TTB_NOS (1 << 5) +#define TTB_IRGN_NC ((0 << 0) | (0 << 6)) +#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6)) +#define TTB_IRGN_WT ((1 << 0) | (0 << 6)) +#define TTB_IRGN_WB ((1 << 0) | (1 << 6)) + +/* PTWs cacheable, inner WB not shareable, outer WB not shareable */ +#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB +#define PMD_FLAGS_UP PMD_SECT_WB + +/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */ +#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA +#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S + +/* + * cpu_v7_switch_mm(pgd_phys, tsk) + * + * Set the translation table base pointer to be pgd_phys + * + * - pgd_phys - physical address of new TTB + * + * It is assumed that: + * - we are not using split page tables + */ +ENTRY(cpu_v7_switch_mm) +#ifdef CONFIG_MMU + mov r2, #0 + ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id + ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) + ALT_UP(orr r0, r0, #TTB_FLAGS_UP) +#ifdef CONFIG_ARM_ERRATA_430973 + mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB +#endif +#ifdef CONFIG_ARM_ERRATA_754322 + dsb +#endif + mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID + isb +1: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0 + isb +#ifdef CONFIG_ARM_ERRATA_754322 + dsb +#endif + mcr p15, 0, r1, c13, c0, 1 @ set context ID + isb +#endif + mov pc, lr +ENDPROC(cpu_v7_switch_mm) + +/* + * cpu_v7_set_pte_ext(ptep, pte) + * + * Set a level 2 translation table entry. + * + * - ptep - pointer to level 2 translation table entry + * (hardware version is stored at +2048 bytes) + * - pte - PTE value to store + * - ext - value for extended PTE bits + */ +ENTRY(cpu_v7_set_pte_ext) +#ifdef CONFIG_MMU + str r1, [r0] @ linux version + + bic r3, r1, #0x000003f0 + bic r3, r3, #PTE_TYPE_MASK + orr r3, r3, r2 + orr r3, r3, #PTE_EXT_AP0 | 2 + + tst r1, #1 << 4 + orrne r3, r3, #PTE_EXT_TEX(1) + + eor r1, r1, #L_PTE_DIRTY + tst r1, #L_PTE_RDONLY | L_PTE_DIRTY + orrne r3, r3, #PTE_EXT_APX + + tst r1, #L_PTE_USER + orrne r3, r3, #PTE_EXT_AP1 +#ifdef CONFIG_CPU_USE_DOMAINS + @ allow kernel read/write access to read-only user pages + tstne r3, #PTE_EXT_APX + bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 +#endif + + tst r1, #L_PTE_XN + orrne r3, r3, #PTE_EXT_XN + + tst r1, #L_PTE_YOUNG + tstne r1, #L_PTE_PRESENT + moveq r3, #0 + + ARM( str r3, [r0, #2048]! ) + THUMB( add r0, r0, #2048 ) + THUMB( str r3, [r0] ) + mcr p15, 0, r0, c7, c10, 1 @ flush_pte +#endif + mov pc, lr +ENDPROC(cpu_v7_set_pte_ext) + + /* + * Memory region attributes with SCTLR.TRE=1 + * + * n = TEX[0],C,B + * TR = PRRR[2n+1:2n] - memory type + * IR = NMRR[2n+1:2n] - inner cacheable property + * OR = NMRR[2n+17:2n+16] - outer cacheable property + * + * n TR IR OR + * UNCACHED 000 00 + * BUFFERABLE 001 10 00 00 + * WRITETHROUGH 010 10 10 10 + * WRITEBACK 011 10 11 11 + * reserved 110 + * WRITEALLOC 111 10 01 01 + * DEV_SHARED 100 01 + * DEV_NONSHARED 100 01 + * DEV_WC 001 10 + * DEV_CACHED 011 10 + * + * Other attributes: + * + * DS0 = PRRR[16] = 0 - device shareable property + * DS1 = PRRR[17] = 1 - device shareable property + * NS0 = PRRR[18] = 0 - normal shareable property + * NS1 = PRRR[19] = 1 - normal shareable property + * NOS = PRRR[24+n] = 1 - not outer shareable + */ +.equ PRRR, 0xff0a81a8 +.equ NMRR, 0x40e040e0 + + /* + * Macro for setting up the TTBRx and TTBCR registers. + * - \ttb0 and \ttb1 updated with the corresponding flags. + */ + .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp + mcr p15, 0, \zero, c2, c0, 2 @ TTB control register + ALT_SMP(orr \ttbr0, \ttbr0, #TTB_FLAGS_SMP) + ALT_UP(orr \ttbr0, \ttbr0, #TTB_FLAGS_UP) + ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP) + ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP) + mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1 + .endm + + __CPUINIT + + /* AT + * TFR EV X F I D LR S + * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM + * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced + * 1 0 110 0011 1100 .111 1101 < we want + */ + .align 2 + .type v7_crval, #object +v7_crval: + crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c + + .previous diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S new file mode 100644 index 00000000000..79cad358ebe --- /dev/null +++ b/arch/arm/mm/proc-v7-3level.S @@ -0,0 +1,149 @@ +/* + * arch/arm/mm/proc-v7-3level.S + * + * Copyright (C) 2001 Deep Blue Solutions Ltd. + * Copyright (C) 2011 ARM Ltd. + * Author: Catalin Marinas <catalin.marinas@arm.com> + * based on arch/arm/mm/proc-v7-2level.S + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + */ + +#define TTB_IRGN_NC (0 << 8) +#define TTB_IRGN_WBWA (1 << 8) +#define TTB_IRGN_WT (2 << 8) +#define TTB_IRGN_WB (3 << 8) +#define TTB_RGN_NC (0 << 10) +#define TTB_RGN_OC_WBWA (1 << 10) +#define TTB_RGN_OC_WT (2 << 10) +#define TTB_RGN_OC_WB (3 << 10) +#define TTB_S (3 << 12) +#define TTB_EAE (1 << 31) + +/* PTWs cacheable, inner WB not shareable, outer WB not shareable */ +#define TTB_FLAGS_UP (TTB_IRGN_WB|TTB_RGN_OC_WB) +#define PMD_FLAGS_UP (PMD_SECT_WB) + +/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */ +#define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA) +#define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S) + +/* + * cpu_v7_switch_mm(pgd_phys, tsk) + * + * Set the translation table base pointer to be pgd_phys (physical address of + * the new TTB). + */ +ENTRY(cpu_v7_switch_mm) +#ifdef CONFIG_MMU + ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id + mov r2, #0 + and r3, r1, #0xff + mov r3, r3, lsl #(48 - 32) @ ASID + mcrr p15, 0, r0, r3, c2 @ set TTB 0 + isb +#endif + mov pc, lr +ENDPROC(cpu_v7_switch_mm) + +/* + * cpu_v7_set_pte_ext(ptep, pte) + * + * Set a level 2 translation table entry. + * - ptep - pointer to level 3 translation table entry + * - pte - PTE value to store (64-bit in r2 and r3) + */ +ENTRY(cpu_v7_set_pte_ext) +#ifdef CONFIG_MMU + tst r2, #L_PTE_PRESENT + beq 1f + tst r3, #1 << (55 - 32) @ L_PTE_DIRTY + orreq r2, #L_PTE_RDONLY +1: strd r2, r3, [r0] + mcr p15, 0, r0, c7, c10, 1 @ flush_pte +#endif + mov pc, lr +ENDPROC(cpu_v7_set_pte_ext) + + /* + * Memory region attributes for LPAE (defined in pgtable-3level.h): + * + * n = AttrIndx[2:0] + * + * n MAIR + * UNCACHED 000 00000000 + * BUFFERABLE 001 01000100 + * DEV_WC 001 01000100 + * WRITETHROUGH 010 10101010 + * WRITEBACK 011 11101110 + * DEV_CACHED 011 11101110 + * DEV_SHARED 100 00000100 + * DEV_NONSHARED 100 00000100 + * unused 101 + * unused 110 + * WRITEALLOC 111 11111111 + */ +.equ PRRR, 0xeeaa4400 @ MAIR0 +.equ NMRR, 0xff000004 @ MAIR1 + + /* + * Macro for setting up the TTBRx and TTBCR registers. + * - \ttbr1 updated. + */ + .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp + mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register + orr \tmp, \tmp, #TTB_EAE + ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP) + ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP) + ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16) + ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16) +#if PHYS_OFFSET <= PAGE_OFFSET + /* + * TTBR0/TTBR1 split (PAGE_OFFSET): + * 0x40000000: T0SZ = 2, T1SZ = 0 (not used) + * 0x80000000: T0SZ = 0, T1SZ = 1 + * 0xc0000000: T0SZ = 0, T1SZ = 2 + * + * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise + * booting secondary CPUs would end up using TTBR1 for the identity + * mapping set up in TTBR0. + */ + orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ +#if defined CONFIG_VMSPLIT_2G + /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */ + add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries +#elif defined CONFIG_VMSPLIT_3G + /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */ + add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd +#endif +#endif /* PHYS_OFFSET <= PAGE_OFFSET */ + mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register + mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 + .endm + + __CPUINIT + + /* + * AT + * TFR EV X F IHD LR S + * .EEE ..EE PUI. .TAT 4RVI ZWRS BLDP WCAM + * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced + * 11 0 110 1 0011 1100 .111 1101 < we want + */ + .align 2 + .type v7_crval, #object +v7_crval: + crval clear=0x0120c302, mmuset=0x30c23c7d, ucset=0x00c01c7c + + .previous diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S index 6af366ce016..163f844811b 100644 --- a/arch/arm/mm/proc-v7.S +++ b/arch/arm/mm/proc-v7.S @@ -19,24 +19,11 @@ #include "proc-macros.S" -#define TTB_S (1 << 1) -#define TTB_RGN_NC (0 << 3) -#define TTB_RGN_OC_WBWA (1 << 3) -#define TTB_RGN_OC_WT (2 << 3) -#define TTB_RGN_OC_WB (3 << 3) -#define TTB_NOS (1 << 5) -#define TTB_IRGN_NC ((0 << 0) | (0 << 6)) -#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6)) -#define TTB_IRGN_WT ((1 << 0) | (0 << 6)) -#define TTB_IRGN_WB ((1 << 0) | (1 << 6)) - -/* PTWs cacheable, inner WB not shareable, outer WB not shareable */ -#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB -#define PMD_FLAGS_UP PMD_SECT_WB - -/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */ -#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA -#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S +#ifdef CONFIG_ARM_LPAE +#include "proc-v7-3level.S" +#else +#include "proc-v7-2level.S" +#endif ENTRY(cpu_v7_proc_init) mov pc, lr @@ -97,127 +84,12 @@ ENTRY(cpu_v7_dcache_clean_area) mov pc, lr ENDPROC(cpu_v7_dcache_clean_area) -/* - * cpu_v7_switch_mm(pgd_phys, tsk) - * - * Set the translation table base pointer to be pgd_phys - * - * - pgd_phys - physical address of new TTB - * - * It is assumed that: - * - we are not using split page tables - */ -ENTRY(cpu_v7_switch_mm) -#ifdef CONFIG_MMU - mov r2, #0 - ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id - ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP) - ALT_UP(orr r0, r0, #TTB_FLAGS_UP) -#ifdef CONFIG_ARM_ERRATA_430973 - mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB -#endif -#ifdef CONFIG_ARM_ERRATA_754322 - dsb -#endif - mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID - isb -1: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0 - isb -#ifdef CONFIG_ARM_ERRATA_754322 - dsb -#endif - mcr p15, 0, r1, c13, c0, 1 @ set context ID - isb -#endif - mov pc, lr -ENDPROC(cpu_v7_switch_mm) - -/* - * cpu_v7_set_pte_ext(ptep, pte) - * - * Set a level 2 translation table entry. - * - * - ptep - pointer to level 2 translation table entry - * (hardware version is stored at +2048 bytes) - * - pte - PTE value to store - * - ext - value for extended PTE bits - */ -ENTRY(cpu_v7_set_pte_ext) -#ifdef CONFIG_MMU - str r1, [r0] @ linux version - - bic r3, r1, #0x000003f0 - bic r3, r3, #PTE_TYPE_MASK - orr r3, r3, r2 - orr r3, r3, #PTE_EXT_AP0 | 2 - - tst r1, #1 << 4 - orrne r3, r3, #PTE_EXT_TEX(1) - - eor r1, r1, #L_PTE_DIRTY - tst r1, #L_PTE_RDONLY | L_PTE_DIRTY - orrne r3, r3, #PTE_EXT_APX - - tst r1, #L_PTE_USER - orrne r3, r3, #PTE_EXT_AP1 -#ifdef CONFIG_CPU_USE_DOMAINS - @ allow kernel read/write access to read-only user pages - tstne r3, #PTE_EXT_APX - bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0 -#endif - - tst r1, #L_PTE_XN - orrne r3, r3, #PTE_EXT_XN - - tst r1, #L_PTE_YOUNG - tstne r1, #L_PTE_PRESENT - moveq r3, #0 - - ARM( str r3, [r0, #2048]! ) - THUMB( add r0, r0, #2048 ) - THUMB( str r3, [r0] ) - mcr p15, 0, r0, c7, c10, 1 @ flush_pte -#endif - mov pc, lr -ENDPROC(cpu_v7_set_pte_ext) - string cpu_v7_name, "ARMv7 Processor" .align - /* - * Memory region attributes with SCTLR.TRE=1 - * - * n = TEX[0],C,B - * TR = PRRR[2n+1:2n] - memory type - * IR = NMRR[2n+1:2n] - inner cacheable property - * OR = NMRR[2n+17:2n+16] - outer cacheable property - * - * n TR IR OR - * UNCACHED 000 00 - * BUFFERABLE 001 10 00 00 - * WRITETHROUGH 010 10 10 10 - * WRITEBACK 011 10 11 11 - * reserved 110 - * WRITEALLOC 111 10 01 01 - * DEV_SHARED 100 01 - * DEV_NONSHARED 100 01 - * DEV_WC 001 10 - * DEV_CACHED 011 10 - * - * Other attributes: - * - * DS0 = PRRR[16] = 0 - device shareable property - * DS1 = PRRR[17] = 1 - device shareable property - * NS0 = PRRR[18] = 0 - normal shareable property - * NS1 = PRRR[19] = 1 - normal shareable property - * NOS = PRRR[24+n] = 1 - not outer shareable - */ -.equ PRRR, 0xff0a81a8 -.equ NMRR, 0x40e040e0 - /* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */ .globl cpu_v7_suspend_size -.equ cpu_v7_suspend_size, 4 * 7 +.equ cpu_v7_suspend_size, 4 * 8 #ifdef CONFIG_PM_SLEEP ENTRY(cpu_v7_do_suspend) stmfd sp!, {r4 - r10, lr} @@ -226,10 +98,11 @@ ENTRY(cpu_v7_do_suspend) stmia r0!, {r4 - r5} mrc p15, 0, r6, c3, c0, 0 @ Domain ID mrc p15, 0, r7, c2, c0, 1 @ TTB 1 + mrc p15, 0, r11, c2, c0, 2 @ TTB control register mrc p15, 0, r8, c1, c0, 0 @ Control register mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control - stmia r0, {r6 - r10} + stmia r0, {r6 - r11} ldmfd sp!, {r4 - r10, pc} ENDPROC(cpu_v7_do_suspend) @@ -241,13 +114,15 @@ ENTRY(cpu_v7_do_resume) ldmia r0!, {r4 - r5} mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID - ldmia r0, {r6 - r10} + ldmia r0, {r6 - r11} mcr p15, 0, r6, c3, c0, 0 @ Domain ID +#ifndef CONFIG_ARM_LPAE ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP) ALT_UP(orr r1, r1, #TTB_FLAGS_UP) +#endif mcr p15, 0, r1, c2, c0, 0 @ TTB 0 mcr p15, 0, r7, c2, c0, 1 @ TTB 1 - mcr p15, 0, ip, c2, c0, 2 @ TTB control register + mcr p15, 0, r11, c2, c0, 2 @ TTB control register mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register teq r4, r9 @ Is it already set? mcrne p15, 0, r9, c1, c0, 1 @ No, so write it @@ -377,12 +252,7 @@ __v7_setup: dsb #ifdef CONFIG_MMU mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs - mcr p15, 0, r10, c2, c0, 2 @ TTB control register - ALT_SMP(orr r4, r4, #TTB_FLAGS_SMP) - ALT_UP(orr r4, r4, #TTB_FLAGS_UP) - ALT_SMP(orr r8, r8, #TTB_FLAGS_SMP) - ALT_UP(orr r8, r8, #TTB_FLAGS_UP) - mcr p15, 0, r8, c2, c0, 1 @ load TTB1 + v7_ttb_setup r10, r4, r8, r5 @ TTBCR, TTBRx setup ldr r5, =PRRR @ PRRR ldr r6, =NMRR @ NMRR mcr p15, 0, r5, c10, c2, 0 @ write PRRR @@ -404,16 +274,7 @@ __v7_setup: mov pc, lr @ return to head.S:__ret ENDPROC(__v7_setup) - /* AT - * TFR EV X F I D LR S - * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM - * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced - * 1 0 110 0011 1100 .111 1101 < we want - */ - .type v7_crval, #object -v7_crval: - crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c - + .align 2 __v7_setup_stack: .space 4 * 11 @ 11 registers @@ -435,11 +296,11 @@ __v7_setup_stack: */ .macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ - PMD_FLAGS_SMP | \mm_mmuflags) + PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags) ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ - PMD_FLAGS_UP | \mm_mmuflags) + PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags) .long PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_AP_WRITE | \ - PMD_SECT_AP_READ | \io_mmuflags + PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags W(b) \initfunc .long cpu_arch_name .long cpu_elf_name @@ -452,6 +313,7 @@ __v7_setup_stack: .long v7_cache_fns .endm +#ifndef CONFIG_ARM_LPAE /* * ARM Ltd. Cortex A5 processor. */ @@ -471,6 +333,7 @@ __v7_ca9mp_proc_info: .long 0xff0ffff0 __v7_proc __v7_ca9mp_setup .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info +#endif /* CONFIG_ARM_LPAE */ /* * ARM Ltd. Cortex A15 processor. diff --git a/arch/arm/plat-s5p/include/plat/exynos4.h b/arch/arm/plat-s5p/include/plat/exynos4.h index 907caab53dc..d62f7f74a80 100644 --- a/arch/arm/plat-s5p/include/plat/exynos4.h +++ b/arch/arm/plat-s5p/include/plat/exynos4.h @@ -15,6 +15,7 @@ extern void exynos4_common_init_uarts(struct s3c2410_uartcfg *cfg, int no); extern void exynos4_register_clocks(void); extern void exynos4_setup_clocks(void); +extern int exynos4_subrev(void); #ifdef CONFIG_CPU_EXYNOS4210 |