diff options
author | Peter Zijlstra <a.p.zijlstra@chello.nl> | 2011-08-11 15:31:31 +0200 |
---|---|---|
committer | Anders Roxell <anders.roxell@linaro.org> | 2015-06-26 22:30:03 +0200 |
commit | 9317fbeb698968af10833ca1756271aa02d9f549 (patch) | |
tree | bbc069d68f3c9fed5deca80b9b664ddb4906303b | |
parent | 737cf72fada6c30ba86a4207d5e072574a2b7a6c (diff) |
mm: pagefault_disabled()
Wrap the test for pagefault_disabled() into a helper, this allows us
to remove the need for current->pagefault_disabled on !-rt kernels.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/n/tip-3yy517m8zsi9fpsf14xfaqkw@git.kernel.org
-rw-r--r-- | arch/alpha/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/arm/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/avr32/mm/fault.c | 3 | ||||
-rw-r--r-- | arch/cris/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/frv/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/ia64/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/m32r/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/m68k/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/microblaze/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/mips/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/mn10300/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/parisc/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/powerpc/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/s390/mm/fault.c | 4 | ||||
-rw-r--r-- | arch/score/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/sh/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/sparc/mm/fault_32.c | 2 | ||||
-rw-r--r-- | arch/sparc/mm/fault_64.c | 2 | ||||
-rw-r--r-- | arch/tile/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/um/kernel/trap.c | 2 | ||||
-rw-r--r-- | arch/x86/mm/fault.c | 2 | ||||
-rw-r--r-- | arch/xtensa/mm/fault.c | 2 | ||||
-rw-r--r-- | include/linux/sched.h | 14 | ||||
-rw-r--r-- | kernel/fork.c | 2 |
24 files changed, 39 insertions, 24 deletions
diff --git a/arch/alpha/mm/fault.c b/arch/alpha/mm/fault.c index 5b59e11a96d8..870f626674e8 100644 --- a/arch/alpha/mm/fault.c +++ b/arch/alpha/mm/fault.c @@ -107,7 +107,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr, /* If we're in an interrupt context, or have no user context, we must not take the fault. */ - if (!mm || in_atomic() || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 9e92a6ca6137..b40d4bab8e07 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -277,7 +277,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/avr32/mm/fault.c b/arch/avr32/mm/fault.c index ddb8290e8378..8a7fb9343cbf 100644 --- a/arch/avr32/mm/fault.c +++ b/arch/avr32/mm/fault.c @@ -81,8 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs) * If we're in an interrupt or have no user context, we must * not take the fault... */ - if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM) || - current->pagefault_disabled) + if (!mm || regs->sr & SYSREG_BIT(GM) || pagefault_disabled()) goto no_context; local_irq_enable(); diff --git a/arch/cris/mm/fault.c b/arch/cris/mm/fault.c index 544670a37dea..b1a5631d95bf 100644 --- a/arch/cris/mm/fault.c +++ b/arch/cris/mm/fault.c @@ -113,7 +113,7 @@ do_page_fault(unsigned long address, struct pt_regs *regs, * user context, we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/frv/mm/fault.c b/arch/frv/mm/fault.c index 45204939aabc..c90d23f50579 100644 --- a/arch/frv/mm/fault.c +++ b/arch/frv/mm/fault.c @@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(__frame)) diff --git a/arch/ia64/mm/fault.c b/arch/ia64/mm/fault.c index c4fb062736dd..f9f7d083cfb4 100644 --- a/arch/ia64/mm/fault.c +++ b/arch/ia64/mm/fault.c @@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re /* * If we're in an interrupt or have no user context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP diff --git a/arch/m32r/mm/fault.c b/arch/m32r/mm/fault.c index ec9ef1059ff4..7069ad371f72 100644 --- a/arch/m32r/mm/fault.c +++ b/arch/m32r/mm/fault.c @@ -114,7 +114,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, * If we're in an interrupt or have no user context or are running in an * atomic region then we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (error_code & ACE_USERMODE) diff --git a/arch/m68k/mm/fault.c b/arch/m68k/mm/fault.c index af31d699b026..5de2621aab37 100644 --- a/arch/m68k/mm/fault.c +++ b/arch/m68k/mm/fault.c @@ -81,7 +81,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if (user_mode(regs)) diff --git a/arch/microblaze/mm/fault.c b/arch/microblaze/mm/fault.c index eb12f4b61798..914ae86b8c96 100644 --- a/arch/microblaze/mm/fault.c +++ b/arch/microblaze/mm/fault.c @@ -107,7 +107,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long address, if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { if (kernel_mode(regs)) goto bad_area_nosemaphore; diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c index 66d4a13d914e..5b41b84d9092 100644 --- a/arch/mips/mm/fault.c +++ b/arch/mips/mm/fault.c @@ -89,7 +89,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (user_mode(regs)) diff --git a/arch/mn10300/mm/fault.c b/arch/mn10300/mm/fault.c index 160b3b2af9f9..54305d255ced 100644 --- a/arch/mn10300/mm/fault.c +++ b/arch/mn10300/mm/fault.c @@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) diff --git a/arch/parisc/mm/fault.c b/arch/parisc/mm/fault.c index fe15ee910e1a..adb466f324c2 100644 --- a/arch/parisc/mm/fault.c +++ b/arch/parisc/mm/fault.c @@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code, int fault; unsigned int flags; - if (in_atomic() || current->pagefault_disabled) + if (pagefault_disabled()) goto no_context; tsk = current; diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 0abe4466e848..018224a542cf 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -261,7 +261,7 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, if (!arch_irq_disabled_regs(regs)) local_irq_enable(); - if (in_atomic() || mm == NULL || current->pagefault_disabled) { + if (in_atomic() || mm == NULL || pagefault_disabled()) { if (!user_mode(regs)) { rc = SIGSEGV; goto bail; diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 3ae32ed14d12..665af67ca8da 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -297,8 +297,8 @@ static inline int do_exception(struct pt_regs *regs, int access) * user context. */ fault = VM_FAULT_BADCONTEXT; - if (unlikely(!user_space_fault(trans_exc_code) || in_atomic() || !mm || - tsk->pagefault_disabled)) + if (unlikely(!user_space_fault(trans_exc_code) || + !mm || pagefault_disabled())) goto out; address = trans_exc_code & __FAIL_ADDR_MASK; diff --git a/arch/score/mm/fault.c b/arch/score/mm/fault.c index dae63aec5422..b946291e60d1 100644 --- a/arch/score/mm/fault.c +++ b/arch/score/mm/fault.c @@ -73,7 +73,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto bad_area_nosemaphore; if (user_mode(regs)) diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault.c index 204624f71a4b..364cae5806ce 100644 --- a/arch/sh/mm/fault.c +++ b/arch/sh/mm/fault.c @@ -440,7 +440,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs, * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index 0550c5c1bf4f..d9613e9a06bd 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -199,7 +199,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write, * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) + if (!mm || pagefault_disabled()) goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index f68b3d17cc78..58c2edf0fcab 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -332,7 +332,7 @@ asmlinkage void __kprobes do_sparc64_fault(struct pt_regs *regs) * If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_enabled) + if (!mm || pagefault_disabled()) goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff --git a/arch/tile/mm/fault.c b/arch/tile/mm/fault.c index 925b686f27a0..4cdab266bcc3 100644 --- a/arch/tile/mm/fault.c +++ b/arch/tile/mm/fault.c @@ -357,7 +357,7 @@ static int handle_page_fault(struct pt_regs *regs, * If we're in an interrupt, have no user context or are running in an * atomic region then we must not take the fault. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } diff --git a/arch/um/kernel/trap.c b/arch/um/kernel/trap.c index db1381b79a79..79910daee6a8 100644 --- a/arch/um/kernel/trap.c +++ b/arch/um/kernel/trap.c @@ -38,7 +38,7 @@ int handle_page_fault(unsigned long address, unsigned long ip, * If the fault was during atomic operation, don't take the fault, just * fail. */ - if (in_atomic() || current->pagefault_disabled) + if (pagefault_disabled()) goto out_nosemaphore; if (is_user) diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index c3cf5641f473..94df0d051e96 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1098,7 +1098,7 @@ __do_page_fault(struct pt_regs *regs, unsigned long error_code, * If we're in an interrupt, have no user context or are running * in an atomic region then we must not take the fault: */ - if (unlikely(in_atomic() || !mm || current->pagefault_disabled)) { + if (unlikely(!mm || pagefault_disabled())) { bad_area_nosemaphore(regs, error_code, address); return; } diff --git a/arch/xtensa/mm/fault.c b/arch/xtensa/mm/fault.c index 27f563e5c8fe..9696ab258ee6 100644 --- a/arch/xtensa/mm/fault.c +++ b/arch/xtensa/mm/fault.c @@ -57,7 +57,7 @@ void do_page_fault(struct pt_regs *regs) /* If we're in an interrupt or have no user * context, we must not take the fault.. */ - if (in_atomic() || !mm || current->pagefault_disabled) { + if (!mm || pagefault_disabled()) { bad_page_fault(regs, address, SIGSEGV); return; } diff --git a/include/linux/sched.h b/include/linux/sched.h index 71c5b6870513..abf781fcc649 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -54,6 +54,7 @@ struct sched_param { #include <linux/llist.h> #include <linux/uidgid.h> #include <linux/gfp.h> +#include <linux/hardirq.h> #include <asm/processor.h> @@ -1395,7 +1396,9 @@ struct task_struct { /* mutex deadlock detection */ struct mutex_waiter *blocked_on; #endif +#ifdef CONFIG_PREEMPT_RT_FULL int pagefault_disabled; +#endif #ifdef CONFIG_TRACE_IRQFLAGS unsigned int irq_events; unsigned long hardirq_enable_ip; @@ -1629,6 +1632,17 @@ static inline void task_numa_free(struct task_struct *p) } #endif +#ifdef CONFIG_PREEMPT_RT_FULL +static inline bool cur_pf_disabled(void) { return current->pagefault_disabled; } +#else +static inline bool cur_pf_disabled(void) { return false; } +#endif + +static inline bool pagefault_disabled(void) +{ + return in_atomic() || cur_pf_disabled(); +} + static inline struct pid *task_pid(struct task_struct *task) { return task->pids[PIDTYPE_PID].pid; diff --git a/kernel/fork.c b/kernel/fork.c index 96947e32d5a0..f2f3af28c4d1 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1303,7 +1303,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, p->hardirq_context = 0; p->softirq_context = 0; #endif +#ifdef CONFIG_PREEMPT_RT_FULL p->pagefault_disabled = 0; +#endif #ifdef CONFIG_LOCKDEP p->lockdep_depth = 0; /* no locks held yet */ p->curr_chain_key = 0; |