diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index aef63c8..9a69b1d 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -235,6 +235,8 @@ config ARCH_VERSATILE config ARCH_AT91 bool "Atmel AT91" select GENERIC_GPIO + select GENERIC_TIME if IPIPE + select GENERIC_CLOCKEVENTS if IPIPE select ARCH_REQUIRE_GPIOLIB select HAVE_CLK help @@ -962,6 +964,8 @@ config LOCAL_TIMERS accounting to be spread across the timer interval, preventing a "thundering herd" at every timer tick. +source "kernel/ipipe/Kconfig" + config PREEMPT bool "Preemptible Kernel (EXPERIMENTAL)" depends on EXPERIMENTAL diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index 4515728..0a4c8e5 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -1012,6 +1012,15 @@ memdump: mov r12, r0 mov pc, r10 #endif +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + .text + .align 0 + .type mcount %function + .global mcount +mcount: + mov pc, lr @ just return +#endif + .ltorg reloc_end: diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c index 2793447..5ec1999 100644 --- a/arch/arm/common/it8152.c +++ b/arch/arm/common/it8152.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -120,21 +121,21 @@ void it8152_irq_demux(unsigned int irq, struct irq_desc *desc) bits_pd &= ((1 << IT8152_PD_IRQ_COUNT) - 1); while (bits_pd) { i = __ffs(bits_pd); - generic_handle_irq(IT8152_PD_IRQ(i)); + ipipe_handle_irq_cond(IT8152_PD_IRQ(i)); bits_pd &= ~(1 << i); } bits_lp &= ((1 << IT8152_LP_IRQ_COUNT) - 1); while (bits_lp) { i = __ffs(bits_lp); - generic_handle_irq(IT8152_LP_IRQ(i)); + ipipe_handle_irq_cond(IT8152_LP_IRQ(i)); bits_lp &= ~(1 << i); } bits_ld &= ((1 << IT8152_LD_IRQ_COUNT) - 1); while (bits_ld) { i = __ffs(bits_ld); - generic_handle_irq(IT8152_LD_IRQ(i)); + ipipe_handle_irq_cond(IT8152_LD_IRQ(i)); bits_ld &= ~(1 << i); } } diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 15f8a09..6865028 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -81,6 +81,18 @@ .macro enable_irq cpsie i .endm + + .macro disable_irq_cond +#ifdef CONFIG_IPIPE + cpsid i +#endif /* CONFIG_IPIPE */ + .endm + + .macro enable_irq_cond +#ifdef CONFIG_IPIPE + cpsie i +#endif /* CONFIG_IPIPE */ + .endm #else .macro disable_irq msr cpsr_c, #PSR_I_BIT | SVC_MODE @@ -89,6 +101,18 @@ .macro enable_irq msr cpsr_c, #SVC_MODE .endm + + .macro disable_irq_cond +#ifdef CONFIG_IPIPE + msr cpsr_c, #PSR_I_BIT | SVC_MODE +#endif /* CONFIG_IPIPE */ + .endm + + .macro enable_irq_cond +#ifdef CONFIG_IPIPE + msr cpsr_c, #SVC_MODE +#endif /* CONFIG_IPIPE */ + .endm #endif /* diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index 9ed2377..0b92cde 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -170,10 +170,10 @@ static inline int atomic_add_return(int i, atomic_t *v) unsigned long flags; int val; - raw_local_irq_save(flags); + local_irq_save_hw(flags); val = v->counter; v->counter = val += i; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return val; } @@ -184,10 +184,10 @@ static inline int atomic_sub_return(int i, atomic_t *v) unsigned long flags; int val; - raw_local_irq_save(flags); + local_irq_save_hw(flags); val = v->counter; v->counter = val -= i; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return val; } @@ -198,11 +198,11 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) int ret; unsigned long flags; - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = v->counter; if (likely(ret == old)) v->counter = new; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return ret; } @@ -211,9 +211,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) { unsigned long flags; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *addr &= ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } #endif /* __LINUX_ARM_ARCH__ */ diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 63a481f..9a1f7b8 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -41,9 +41,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long * p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p |= mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p) @@ -53,9 +53,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p &= ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p) @@ -65,9 +65,9 @@ static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned lon p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); *p ^= mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); } static inline int @@ -79,10 +79,10 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res | mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } @@ -96,10 +96,10 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res & ~mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } @@ -113,10 +113,10 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + local_irq_save_hw(flags); res = *p; *p = res ^ mask; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); return res & mask; } diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 1a711ea..4503226 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -15,6 +15,7 @@ #include #include #include +#include #define CACHE_COLOUR(vaddr) ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT) @@ -334,23 +335,29 @@ static inline void outer_flush_range(unsigned long start, unsigned long end) #ifndef CONFIG_CPU_CACHE_VIPT static inline void flush_cache_mm(struct mm_struct *mm) { - if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) + if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) { + fcse_notify_flush_all(); __cpuc_flush_user_all(); + } } static inline void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { - if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) + if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { + start = fcse_va_to_mva(vma->vm_mm, start); + end = fcse_va_to_mva(vma->vm_mm, end); __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end), vma->vm_flags); + } } static inline void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsigned long pfn) { if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { - unsigned long addr = user_addr & PAGE_MASK; + unsigned long addr; + addr = fcse_va_to_mva(vma->vm_mm, user_addr) & PAGE_MASK; __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); } } @@ -381,14 +388,22 @@ extern void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, * Harvard caches are synchronised for the user space address range. * This is used for the ARM private sys_cacheflush system call. */ -#define flush_cache_user_range(vma,start,end) \ - __cpuc_coherent_user_range((start) & PAGE_MASK, PAGE_ALIGN(end)) +#define flush_cache_user_range(vma, start, end) \ + ({ \ + struct mm_struct *_mm = (vma)->vm_mm; \ + unsigned long _start, _end; \ + _start = fcse_va_to_mva(_mm, start) & PAGE_MASK; \ + _end = PAGE_ALIGN(fcse_va_to_mva(_mm, end)); \ + __cpuc_coherent_user_range(_start, _end); \ + }) /* * Perform necessary cache operations to ensure that data previously * stored within this range of addresses can be executed by the CPU. */ -#define flush_icache_range(s,e) __cpuc_coherent_kern_range(s,e) +#define flush_icache_range(s,e) \ + __cpuc_coherent_kern_range(fcse_va_to_mva(current->mm, (s)), \ + fcse_va_to_mva(current->mm, (e))) /* * Perform necessary cache operations to ensure that the TLB will @@ -426,7 +441,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma, extern void __flush_anon_page(struct vm_area_struct *vma, struct page *, unsigned long); if (PageAnon(page)) - __flush_anon_page(vma, page, vmaddr); + __flush_anon_page(vma, page, + fcse_va_to_mva(vma->vm_mm, vmaddr)); } #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE diff --git a/arch/arm/include/asm/cpu-multi32.h b/arch/arm/include/asm/cpu-multi32.h index e2b5b0b..82768d7 100644 --- a/arch/arm/include/asm/cpu-multi32.h +++ b/arch/arm/include/asm/cpu-multi32.h @@ -52,7 +52,8 @@ extern struct processor { /* * Set the page table */ - void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm); + void (*switch_mm)(unsigned long pgd_phys, + struct mm_struct *mm, unsigned flush); /* * Set a possibly extended PTE. Non-extended PTEs should * ignore 'ext'. @@ -66,4 +67,4 @@ extern struct processor { #define cpu_do_idle() processor._do_idle() #define cpu_dcache_clean_area(addr,sz) processor.dcache_clean_area(addr,sz) #define cpu_set_pte_ext(ptep,pte,ext) processor.set_pte_ext(ptep,pte,ext) -#define cpu_do_switch_mm(pgd,mm) processor.switch_mm(pgd,mm) +#define cpu_do_switch_mm(pgd,mm,flush) processor.switch_mm(pgd,mm,flush) diff --git a/arch/arm/include/asm/cpu-single.h b/arch/arm/include/asm/cpu-single.h index f073a6d..b9add09 100644 --- a/arch/arm/include/asm/cpu-single.h +++ b/arch/arm/include/asm/cpu-single.h @@ -39,6 +39,7 @@ extern void cpu_proc_init(void); extern void cpu_proc_fin(void); extern int cpu_do_idle(void); extern void cpu_dcache_clean_area(void *, int); -extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); +extern void cpu_do_switch_mm(unsigned long pgd_phys, + struct mm_struct *mm, unsigned flush); extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); diff --git a/arch/arm/include/asm/fcse.h b/arch/arm/include/asm/fcse.h new file mode 100644 index 0000000..d97bb40 --- /dev/null +++ b/arch/arm/include/asm/fcse.h @@ -0,0 +1,90 @@ +/* + * Filename: arch/arm/include/asm/fcse.h + * Description: ARM Process ID (PID) includes for Fast Address Space Switching + * (FASS) in ARM Linux. + * Created: 14/10/2001 + * Changes: 19/02/2002 - Macros added. + * 03/08/2007 - Adapted to kernel 2.6.21 (ssm) + * Feb 2008 - Simplified a bit (rco) + * + * Copyright: (C) 2001, 2002 Adam Wiggins + * (C) 2007 Sebastian Smolorz + * (C) 2008 Richard Cochran + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of teh GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_ARM_FCSE_H +#define __ASM_ARM_FCSE_H + +#ifdef CONFIG_ARM_FCSE + +#include /* For struct mm_struct */ + +#define FCSE_PID_SHIFT 25 + +/* Size of PID relocation area */ +#define FCSE_PID_TASK_SIZE (1UL << FCSE_PID_SHIFT) + +/* Mask to get rid of PID from relocated address */ +#define FCSE_PID_MASK (FCSE_PID_TASK_SIZE - 1) + +#define fcse_tlb_mask(mm) ((mm)->context.cpu_tlb_mask) +#define fcse_cpu_set_vm_mask(cpu, mm) cpu_set(cpu, (mm)->cpu_vm_mask) + +/* Sets the CPU's PID Register */ +static inline void fcse_pid_set(unsigned long pid) +{ + __asm__ __volatile__ ("mcr p15, 0, %0, c13, c0, 0" + : /* */ : "r" (pid) : "memory", "cc"); +} + +/* Returns the state of the CPU's PID Register */ +static inline unsigned long fcse_pid_get(void) +{ + unsigned long pid; + __asm__ __volatile__("mrc p15, 0, %0, c13, c0, 0" + : "=&r" (pid) : /* */ : "cc"); + return pid & ~FCSE_PID_MASK; +} + +static inline unsigned long fcse_mva_to_va(unsigned long mva) +{ + unsigned long pid = fcse_pid_get(); + if (pid && (pid == (mva & ~FCSE_PID_MASK))) + return mva & FCSE_PID_MASK; + return mva; +} + +static inline unsigned long +fcse_va_to_mva(struct mm_struct *mm, unsigned long va) +{ + if (va < FCSE_PID_TASK_SIZE) + return mm->context.pid | va; + return va; +} + +int fcse_pid_alloc(void); +void fcse_pid_free(unsigned pid); +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +int fcse_needs_flush(struct mm_struct *prev, struct mm_struct *next); +void fcse_notify_flush_all(void); +void fcse_pid_reference(unsigned pid); +void fcse_relocate_mm_to_null_pid(struct mm_struct *mm); +#else /* CONFIG_ARM_FCSE_GUARANTEED */ +#define fcse_needs_flush(prev, next) (0) +#define fcse_notify_flush_all() do { } while (0) +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + +#else /* ! CONFIG_ARM_FCSE */ +#define fcse_pid_set(pid) do { } while (0) +#define fcse_mva_to_va(x) (x) +#define fcse_va_to_mva(mm, x) ({ (void)(mm); (x); }) +#define fcse_tlb_mask(mm) ((mm)->cpu_vm_mask) +#define fcse_cpu_set_vm_mask(cpu, mm) do { } while (0) +#define fcse_needs_flush(prev, next) (1) +#define fcse_notify_flush_all() do { } while (0) +#endif /* ! CONFIG_ARM_FCSE */ + +#endif /* __ASM_ARM_FCSE_H */ diff --git a/arch/arm/include/asm/ipipe.h b/arch/arm/include/asm/ipipe.h new file mode 100644 index 0000000..24770d6 --- /dev/null +++ b/arch/arm/include/asm/ipipe.h @@ -0,0 +1,274 @@ +/* -*- linux-c -*- + * arch/arm/include/asm/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2005 Stelian Pop. + * Copyright (C) 2006-2008 Gilles Chanteperdrix. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ARM_IPIPE_H +#define __ARM_IPIPE_H + +#ifdef CONFIG_IPIPE + +#include +#include /* For __IPIPE_FEATURE_PIC_MUTE */ + +#define IPIPE_ARCH_STRING "1.15-01" +#define IPIPE_MAJOR_NUMBER 1 +#define IPIPE_MINOR_NUMBER 15 +#define IPIPE_PATCH_NUMBER 1 + +#ifdef CONFIG_SMP +#error "I-pipe/arm: SMP not yet implemented" +#define ipipe_processor_id() (current_thread_info()->cpu) +#else /* !CONFIG_SMP */ +#define ipipe_processor_id() 0 +#endif /* CONFIG_SMP */ + +#define smp_processor_id_hw() ipipe_processor_id() + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + +#define prepare_arch_switch(next) \ + do { \ + local_irq_enable_hw(); \ + ipipe_schedule_notify(current, next); \ + } while(0) + +#define task_hijacked(p) \ + ({ \ + int x = !ipipe_root_domain_p; \ + clear_bit(IPIPE_SYNC_FLAG, &ipipe_root_cpudom_var(status)); \ + x; \ + }) + +#define ipipe_mm_switch_protect(flags) \ + do { \ + (void) (flags); \ + } while (0) + +#define ipipe_mm_switch_unprotect(flags) \ + do { \ + (void) (flags); \ + } while (0) + +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +#define prepare_arch_switch(next) \ + do { \ + ipipe_schedule_notify(current ,next); \ + local_irq_disable_hw(); \ + } while(0) + +#define task_hijacked(p) \ + ({ \ + int x = !ipipe_root_domain_p; \ + __clear_bit(IPIPE_SYNC_FLAG, &ipipe_root_cpudom_var(status)); \ + if (!x) local_irq_enable_hw(); x; \ + }) + +#define ipipe_mm_switch_protect(flags) \ + local_irq_save_hw_cond(flags) + +#define ipipe_mm_switch_unprotect(flags) \ + local_irq_restore_hw_cond(flags) + +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +extern unsigned long arm_return_addr(int level); + +#define BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0) +#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1) + + +struct ipipe_domain; + +#define IPIPE_TSC_TYPE_NONE 0 +#define IPIPE_TSC_TYPE_FREERUNNING 1 +#define IPIPE_TSC_TYPE_DECREMENTER 2 + +struct __ipipe_tscinfo { + unsigned type; + union { + struct { + unsigned *counter; /* Hw counter physical address */ + unsigned mask; /* Significant bits in the hw counter. */ + unsigned long long *tsc; /* 64 bits tsc value. */ + } fr; + struct { + unsigned *counter; /* Hw counter physical address */ + unsigned mask; /* Significant bits in the hw counter. */ + unsigned *last_cnt; /* Counter value when updating + tsc value. */ + unsigned long long *tsc; /* 64 bits tsc value. */ + } dec; + } u; +}; + +struct ipipe_sysinfo { + + int ncpus; /* Number of CPUs on board */ + u64 cpufreq; /* CPU frequency (in Hz) */ + + /* Arch-dependent block */ + + struct { + unsigned tmirq; /* Timer tick IRQ */ + u64 tmfreq; /* Timer frequency */ + struct __ipipe_tscinfo tsc; /* exported data for u.s. tsc */ + } archdep; +}; + +DECLARE_PER_CPU(struct mm_struct *,ipipe_active_mm); +/* arch specific stuff */ +extern void *__ipipe_tsc_area; +extern int __ipipe_mach_timerint; +extern int __ipipe_mach_timerstolen; +extern unsigned int __ipipe_mach_ticks_per_jiffy; +extern void __ipipe_mach_acktimer(void); +extern unsigned long long __ipipe_mach_get_tsc(void); +extern void __ipipe_mach_set_dec(unsigned long); +extern void __ipipe_mach_release_timer(void); +extern unsigned long __ipipe_mach_get_dec(void); +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info); +int __ipipe_check_tickdev(const char *devname); + +#define ipipe_read_tsc(t) do { t = __ipipe_mach_get_tsc(); } while (0) +#define __ipipe_read_timebase() __ipipe_mach_get_tsc() + +#define ipipe_cpu_freq() (HZ * __ipipe_mach_ticks_per_jiffy) +#define ipipe_tsc2ns(t) \ +({ \ + unsigned long long delta = (t)*1000; \ + do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \ + (unsigned long)delta; \ +}) +#define ipipe_tsc2us(t) \ +({ \ + unsigned long long delta = (t); \ + do_div(delta, ipipe_cpu_freq() / 1000000 + 1); \ + (unsigned long)delta; \ +}) + +#define ipipe_handle_irq_cond(irq) \ + __ipipe_handle_irq(irq, (struct pt_regs *)1) + +/* Private interface -- Internal use only */ + +#define __ipipe_check_platform() do { } while(0) + +#define __ipipe_init_platform() do { } while(0) + +#define __ipipe_enable_irq(irq) irq_desc[irq].chip->enable(irq) + +#define __ipipe_disable_irq(irq) irq_desc[irq].chip->disable(irq) + +#define __ipipe_hook_critical_ipi(ipd) do { } while(0) + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq); + +#ifndef __IPIPE_FEATURE_PIC_MUTE +#define __ipipe_disable_irqdesc(ipd, irq) do { } while (0) +#else /* __IPIPE_FEATURE_PIC_MUTE */ + +typedef unsigned long +__ipipe_irqbits_t[(NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG]; +extern __ipipe_irqbits_t __ipipe_irqbits; + +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq); + +void ipipe_mute_pic(void); + +void ipipe_unmute_pic(void); +#endif /* __IPIPE_FEATURE_PIC_MUTE */ + +void __ipipe_enable_pipeline(void); + +void __ipipe_do_critical_sync(unsigned irq, + void *cookie); + +DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs); + +int __ipipe_handle_irq(int irq, + struct pt_regs *regs); + +#define ipipe_update_tick_evtdev(evtdev) do { } while (0) + +#define __ipipe_tick_irq __ipipe_mach_timerint + +static inline unsigned long __ipipe_ffnz(unsigned long ul) +{ + return ffs(ul) - 1; +} + +/* When running handlers, enable hw interrupts for all domains but the + * one heading the pipeline, so that IRQs can never be significantly + * deferred for the latter. */ +#define __ipipe_run_isr(ipd, irq) \ +do { \ + if (!__ipipe_pipeline_head_p(ipd)) \ + local_irq_enable_hw(); \ + if (ipd == ipipe_root_domain) { \ + if (likely(!ipipe_virtual_irq_p(irq))) \ + ((void (*)(unsigned, struct pt_regs *)) \ + ipd->irqs[irq].handler) (irq, \ + &__raw_get_cpu_var(__ipipe_tick_regs)); \ + else { \ + irq_enter(); \ + ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \ + irq_exit(); \ + } \ + } else { \ + __clear_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \ + ipd->irqs[irq].handler(irq,ipd->irqs[irq].cookie); \ + __set_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \ + } \ + local_irq_disable_hw(); \ +} while(0) + +#define __ipipe_syscall_watched_p(p, sc) \ + (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= __ARM_NR_BASE + 64) + +#define __ipipe_root_tick_p(regs) (!raw_irqs_disabled_flags(regs->ARM_cpsr)) + +#else /* !CONFIG_IPIPE */ + +#define task_hijacked(p) 0 + +#define ipipe_update_tick_evtdev(evtdev) do { } while (0) + +#define smp_processor_id_hw() smp_processor_id() + +#define ipipe_handle_irq_cond(irq) \ + generic_handle_irq(irq) + +#define ipipe_mm_switch_protect(flags) \ + do { \ + (void) (flags); \ + } while (0) + +#define ipipe_mm_switch_unprotect(flags) \ + do { \ + (void) (flags); \ + } while (0) + +#endif /* CONFIG_IPIPE */ + +#endif /* !__ARM_IPIPE_H */ diff --git a/arch/arm/include/asm/ipipe_base.h b/arch/arm/include/asm/ipipe_base.h new file mode 100644 index 0000000..20e97a4 --- /dev/null +++ b/arch/arm/include/asm/ipipe_base.h @@ -0,0 +1,108 @@ +/* -*- linux-c -*- + * arch/arm/include/asm/ipipe_base.h + * + * Copyright (C) 2007 Gilles Chanteperdrix. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ARM_IPIPE_BASE_H +#define __ARM_IPIPE_BASE_H + +#include +#include +#include + +#define IPIPE_NR_XIRQS NR_IRQS +#define IPIPE_IRQ_ISHIFT 5 /* 25 for 32bits arch. */ + +/* ARM traps */ +#define IPIPE_TRAP_ACCESS 0 /* Data or instruction access exception */ +#define IPIPE_TRAP_SECTION 1 /* Section fault */ +#define IPIPE_TRAP_DABT 2 /* Generic data abort */ +#define IPIPE_TRAP_UNKNOWN 3 /* Unknown exception */ +#define IPIPE_TRAP_BREAK 4 /* Instruction breakpoint */ +#define IPIPE_TRAP_FPU 5 /* Floating point exception */ +#define IPIPE_TRAP_VFP 6 /* VFP floating point exception */ +#define IPIPE_TRAP_UNDEFINSTR 7 /* Undefined instruction */ +#define IPIPE_TRAP_ALIGNMENT 8 /* Unaligned access exception */ +#define IPIPE_NR_FAULTS 9 + +/* Pseudo-vectors used for kernel events */ +#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS +#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT) +#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1) +#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2) +#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3) +#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4) +#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5) +#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6) +#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP +#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1) + +#ifndef __ASSEMBLY__ + +#include + +#ifdef CONFIG_SMP +#error "SMP not implemented." +#define __ipipe_root_status ipipe_root_cpudom_var(status) + +#else /* !CONFIG_SMP */ + +#ifdef CONFIG_VFP +#define __IPIPE_FEATURE_VFP_SAFE 1 +#endif + +#if __GNUC__ >= 4 +/* Alias to ipipe_root_cpudom_var(status) */ +extern unsigned long __ipipe_root_status; +#else +extern unsigned long *const __ipipe_root_status_addr; +#define __ipipe_root_status (*__ipipe_root_status_addr) +#endif + +static inline void __ipipe_stall_root(void) +{ + unsigned long flags; + + local_irq_save_hw(flags); + __ipipe_root_status |= 1; + local_irq_restore_hw(flags); +} + +static inline unsigned __ipipe_test_root(void) +{ + return __ipipe_root_status & 1; +} + +static inline unsigned __ipipe_test_and_stall_root(void) +{ + unsigned long flags, res; + + local_irq_save_hw(flags); + res = __ipipe_root_status; + __ipipe_root_status = res | 1; + local_irq_restore_hw(flags); + + return res & 1; +} + +#endif /* CONFIG_SMP */ + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ARM_IPIPE_BASE_H */ diff --git a/arch/arm/include/asm/irqflags.h b/arch/arm/include/asm/irqflags.h index 6d09974..0aa4056 100644 --- a/arch/arm/include/asm/irqflags.h +++ b/arch/arm/include/asm/irqflags.h @@ -10,30 +10,30 @@ */ #if __LINUX_ARM_ARCH__ >= 6 -#define raw_local_irq_save(x) \ +#define local_irq_save_hw_notrace(x) \ ({ \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_save\n" \ + "mrs %0, cpsr @ local_irq_save_hw\n" \ "cpsid i" \ : "=r" (x) : : "memory", "cc"); \ }) -#define raw_local_irq_enable() __asm__("cpsie i @ __sti" : : : "memory", "cc") -#define raw_local_irq_disable() __asm__("cpsid i @ __cli" : : : "memory", "cc") -#define local_fiq_enable() __asm__("cpsie f @ __stf" : : : "memory", "cc") -#define local_fiq_disable() __asm__("cpsid f @ __clf" : : : "memory", "cc") +#define local_irq_enable_hw_notrace() __asm__("cpsie i @ __sti" : : : "memory", "cc") +#define local_irq_disable_hw_notrace() __asm__("cpsid i @ __cli" : : : "memory", "cc") +#define local_fiq_enable_hw_notrace() __asm__("cpsie f @ __stf" : : : "memory", "cc") +#define local_fiq_disable_hw_notrace() __asm__("cpsid f @ __clf" : : : "memory", "cc") #else /* * Save the current interrupt enable state & disable IRQs */ -#define raw_local_irq_save(x) \ +#define local_irq_save_hw_notrace(x) \ ({ \ unsigned long temp; \ (void) (&temp == &x); \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_save\n" \ + "mrs %0, cpsr @ local_irq_save_hw\n" \ " orr %1, %0, #128\n" \ " msr cpsr_c, %1" \ : "=r" (x), "=r" (temp) \ @@ -44,11 +44,11 @@ /* * Enable IRQs */ -#define raw_local_irq_enable() \ +#define local_irq_enable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_enable\n" \ + "mrs %0, cpsr @ local_irq_enable_hw\n" \ " bic %0, %0, #128\n" \ " msr cpsr_c, %0" \ : "=r" (temp) \ @@ -59,11 +59,11 @@ /* * Disable IRQs */ -#define raw_local_irq_disable() \ +#define local_irq_disable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_irq_disable\n" \ + "mrs %0, cpsr @ local_irq_disable_hw\n" \ " orr %0, %0, #128\n" \ " msr cpsr_c, %0" \ : "=r" (temp) \ @@ -74,7 +74,7 @@ /* * Enable FIQs */ -#define local_fiq_enable() \ +#define local_fiq_enable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ @@ -89,7 +89,7 @@ /* * Disable FIQs */ -#define local_fiq_disable() \ +#define local_fiq_disable_hw_notrace() \ ({ \ unsigned long temp; \ __asm__ __volatile__( \ @@ -106,19 +106,19 @@ /* * Save the current interrupt enable state. */ -#define raw_local_save_flags(x) \ +#define local_save_flags_hw(x) \ ({ \ __asm__ __volatile__( \ - "mrs %0, cpsr @ local_save_flags" \ + "mrs %0, cpsr @ local_save_flags_hw" \ : "=r" (x) : : "memory", "cc"); \ }) /* * restore saved IRQ & FIQ state */ -#define raw_local_irq_restore(x) \ +#define local_irq_restore_hw_notrace(x) \ __asm__ __volatile__( \ - "msr cpsr_c, %0 @ local_irq_restore\n" \ + "msr cpsr_c, %0 @ local_irq_restore_hw\n" \ : \ : "r" (x) \ : "memory", "cc") @@ -128,5 +128,99 @@ (int)((flags) & PSR_I_BIT); \ }) +#define irqs_disabled_hw() \ +({ \ + unsigned long flags; \ + local_save_flags_hw(flags); \ + raw_irqs_disabled_flags(flags); \ +}) + +static inline unsigned long raw_mangle_irq_bits(int virt, unsigned long real) +{ + /* Merge virtual and real interrupt mask bits into a single + 32bit word. */ + return (real & ~(1L << 8)) | ((virt != 0) << 8); +} + +static inline int raw_demangle_irq_bits(unsigned long *x) +{ + int virt = (*x & (1 << 8)) != 0; + *x &= ~(1L << 8); + return virt; +} + +#ifdef CONFIG_IPIPE + +void __ipipe_unstall_root(void); +void __ipipe_restore_root(unsigned long flags); + +/* PSR_I_BIT is bit no. 7 and is set if interrupts are _disabled_ */ +#define raw_local_irq_save(flags) ((flags) = __ipipe_test_and_stall_root() << 7) +#define raw_local_irq_enable() __ipipe_unstall_root() +#define raw_local_irq_disable() __ipipe_stall_root() +#define local_fiq_enable() __ipipe_unstall_root() +#define local_fiq_disable() __ipipe_stall_root() +#define raw_local_save_flags(flags) ((flags) = __ipipe_test_root() << 7) +#define raw_local_irq_restore(flags) __ipipe_restore_root(flags & (1 << 7)) + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + +#include + +#define local_irq_disable_hw() do { \ + if (!irqs_disabled_hw()) { \ + local_irq_disable_hw_notrace(); \ + ipipe_trace_begin(0x80000000); \ + } \ +} while (0) +#define local_irq_enable_hw() do { \ + if (irqs_disabled_hw()) { \ + ipipe_trace_end(0x80000000); \ + local_irq_enable_hw_notrace(); \ + } \ +} while (0) +#define local_irq_save_hw(x) do { \ + local_save_flags_hw(x); \ + if (!raw_irqs_disabled_flags(x)) { \ + local_irq_disable_hw_notrace(); \ + ipipe_trace_begin(0x80000001); \ + } \ +} while (0) +#define local_irq_restore_hw(x) do { \ + if (!raw_irqs_disabled_flags(x)) \ + ipipe_trace_end(0x80000001); \ + local_irq_restore_hw_notrace(x); \ +} while (0) + +#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define local_irq_save_hw(flags) local_irq_save_hw_notrace(flags) +#define local_irq_enable_hw() local_irq_enable_hw_notrace() +#define local_irq_disable_hw() local_irq_disable_hw_notrace() +#define local_fiq_enable_hw() local_fiq_enable_hw_notrace() +#define local_fiq_disable_hw() local_fiq_disable_hw_notrace() +#define local_irq_restore_hw(flags) local_irq_restore_hw_notrace(flags) + +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#else /* !CONFIG_IPIPE */ + +#define raw_local_irq_save(flags) local_irq_save_hw_notrace(flags) +#define raw_local_irq_enable() local_irq_enable_hw_notrace() +#define raw_local_irq_disable() local_irq_disable_hw_notrace() +#define local_fiq_enable() local_fiq_enable_hw_notrace() +#define local_fiq_disable() local_fiq_disable_hw_notrace() +#define raw_local_save_flags(flags) local_save_flags_hw(flags) +#define raw_local_irq_restore(flags) local_irq_restore_hw_notrace(flags) + +#define local_irq_save_hw(flags) local_irq_save_hw_notrace(flags) +#define local_irq_enable_hw() local_irq_enable_hw_notrace() +#define local_irq_disable_hw() local_irq_disable_hw_notrace() +#define local_fiq_enable_hw() local_fiq_enable_hw_notrace() +#define local_fiq_disable_hw() local_fiq_disable_hw_notrace() +#define local_irq_restore_hw(flags) local_irq_restore_hw_notrace(flags) + +#endif /* CONFIG_IPIPE */ + #endif #endif diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 85763db..0ab361a 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -33,7 +33,12 @@ */ #define PAGE_OFFSET UL(CONFIG_PAGE_OFFSET) #define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(0x01000000)) +#ifndef CONFIG_ARM_FCSE #define TASK_UNMAPPED_BASE (UL(CONFIG_PAGE_OFFSET) / 3) +#else /* CONFIG_ARM_FCSE */ +#define TASK_UNMAPPED_BASE UL(0x01000000) +#define FCSE_TASK_SIZE UL(0x02000000) +#endif /* CONFIG_ARM_FCSE */ /* * The maximum size of a 26-bit user space task. diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index b561584..13c844f 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -7,6 +7,15 @@ typedef struct { #ifdef CONFIG_CPU_HAS_ASID unsigned int id; #endif +#ifdef CONFIG_ARM_FCSE + unsigned long pid; + cpumask_t cpu_tlb_mask; +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + unsigned shared_dirty_pages; + unsigned big; + unsigned high_pages; +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#endif /* CONFIG_ARM_FCSE */ unsigned int kvm_seq; } mm_context_t; diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 263fed0..06197f3 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -19,6 +19,7 @@ #include #include #include +#include void __check_kvm_seq(struct mm_struct *mm); @@ -66,11 +67,50 @@ static inline void check_context(struct mm_struct *mm) __check_kvm_seq(mm); } -#define init_new_context(tsk,mm) 0 +static inline int +init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ +#ifdef CONFIG_ARM_FCSE + int pid; + + cpus_clear(mm->context.cpu_tlb_mask); +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (!mm->context.big) { + pid = fcse_pid_alloc(); + mm->context.pid = pid << FCSE_PID_SHIFT; + } else { + /* We are normally forking a process vith a virtual address + space larger than 32 MB, so its pid should be 0. */ + BUG_ON(mm->context.pid); + fcse_pid_reference(0); + } + /* If we are forking, set_pte_at will restore the correct high pages + count, and shared writable pages are write-protected again. */ + mm->context.shared_dirty_pages = 0; + mm->context.high_pages = 0; +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + pid = fcse_pid_alloc(); + if (pid < 0) + return pid; + mm->context.pid = pid << FCSE_PID_SHIFT; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ +#endif /* CONFIG_ARM_FCSE */ + + return 0; +} #endif -#define destroy_context(mm) do { } while(0) +static inline void destroy_context(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM_FCSE +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + BUG_ON(mm->context.shared_dirty_pages); + BUG_ON(mm->context.high_pages); +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + fcse_pid_free(mm->context.pid >> FCSE_PID_SHIFT); +#endif /* CONFIG_ARM_FCSE */ +} /* * This is called when "tsk" is about to enter lazy TLB mode. @@ -93,26 +133,59 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) * actually changed. */ static inline void -switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) +__switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) { #ifdef CONFIG_MMU - unsigned int cpu = smp_processor_id(); + unsigned int cpu = smp_processor_id_hw(); #ifdef CONFIG_SMP /* check for possible thread migration */ if (!cpus_empty(next->cpu_vm_mask) && !cpu_isset(cpu, next->cpu_vm_mask)) __flush_icache_all(); #endif - if (!cpu_test_and_set(cpu, next->cpu_vm_mask) || prev != next) { + if (cache_is_vivt() && prev && prev != next) + cpu_clear(cpu, fcse_tlb_mask(prev)); + if (!cpu_test_and_set(cpu, fcse_tlb_mask(next)) || prev != next) { + fcse_cpu_set_vm_mask(cpu, next); check_context(next); - cpu_switch_mm(next->pgd, next); - if (cache_is_vivt()) - cpu_clear(cpu, prev->cpu_vm_mask); +#if defined(CONFIG_IPIPE) + if (ipipe_root_domain_p) + do { + /* mark mm state as undefined. */ + per_cpu(ipipe_active_mm, cpu) = NULL; + barrier(); + fcse_pid_set(next->context.pid); + cpu_switch_mm(next->pgd, next, + fcse_needs_flush(prev, next)); + barrier(); + per_cpu(ipipe_active_mm, cpu) = next; + } while (test_and_clear_thread_flag(TIF_MMSWITCH_INT)); + else +#endif /* CONFIG_IPIPE */ + { + fcse_pid_set(next->context.pid); + cpu_switch_mm(next->pgd, next, + fcse_needs_flush(prev, next)); + } } #endif } +static inline void +switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ +#ifndef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + unsigned long flags; + local_irq_save_hw(flags); +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + __switch_mm(prev, next, tsk); +#ifndef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + local_irq_restore_hw(flags); +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ +} + #define deactivate_mm(tsk,mm) do { } while (0) #define activate_mm(prev,next) switch_mm(prev, next, NULL) diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index c433c6c..20c2d90 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -112,6 +112,8 @@ #define LIBRARY_TEXT_START 0x0c000000 #ifndef __ASSEMBLY__ +#include + extern void __pte_error(const char *file, int line, unsigned long val); extern void __pmd_error(const char *file, int line, unsigned long val); extern void __pgd_error(const char *file, int line, unsigned long val); @@ -250,6 +252,40 @@ extern pgprot_t pgprot_kernel; #define __S111 __PAGE_SHARED_EXEC #ifndef __ASSEMBLY__ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +#define fcse_account_page_removal(mm, addr, val) do { \ + struct mm_struct *_mm = (mm); \ + unsigned long _addr = (addr); \ + unsigned long _val = (val); \ + if(pte_present(_val) && ((_val) & L_PTE_SHARED)) \ + --_mm->context.shared_dirty_pages; \ + if (pte_present(_val) && _addr < TASK_SIZE) { \ + if (_addr >= FCSE_TASK_SIZE) \ + --_mm->context.high_pages; \ + } \ +} while (0) + +#define fcse_account_page_addition(mm, addr, val) ({ \ + struct mm_struct *_mm = (mm); \ + unsigned long _addr = (addr); \ + unsigned long _val = (val); \ + if (pte_present(_val) && (_val & L_PTE_SHARED)) { \ + if ((_val & (L_PTE_CACHEABLE | L_PTE_WRITE | L_PTE_DIRTY)) \ + != (L_PTE_CACHEABLE | L_PTE_WRITE | L_PTE_DIRTY)) \ + _val &= ~L_PTE_SHARED; \ + else \ + ++_mm->context.shared_dirty_pages; \ + } \ + if (pte_present(_val) \ + && _addr < TASK_SIZE && _addr >= FCSE_TASK_SIZE) \ + ++_mm->context.high_pages; \ + _val; \ +}) +#else /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */ +#define fcse_account_page_removal(mm, addr, val) do { } while (0) +#define fcse_account_page_addition(mm, addr, val) (val) +#endif /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */ + /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. @@ -261,7 +297,10 @@ extern struct page *empty_zero_page; #define pfn_pte(pfn,prot) (__pte(((pfn) << PAGE_SHIFT) | pgprot_val(prot))) #define pte_none(pte) (!pte_val(pte)) -#define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0) +#define pte_clear(mm,addr,ptep) do { \ + fcse_account_page_removal(mm, addr, pte_val(*ptep)); \ + set_pte_ext(ptep, __pte(0), 0); \ +} while (0) #define pte_page(pte) (pfn_to_page(pte_pfn(pte))) #define pte_offset_kernel(dir,addr) (pmd_page_vaddr(*(dir)) + __pte_index(addr)) #define pte_offset_map(dir,addr) (pmd_page_vaddr(*(dir)) + __pte_index(addr)) @@ -271,9 +310,13 @@ extern struct page *empty_zero_page; #define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext) -#define set_pte_at(mm,addr,ptep,pteval) do { \ - set_pte_ext(ptep, pteval, (addr) >= TASK_SIZE ? 0 : PTE_EXT_NG); \ - } while (0) +#define set_pte_at(mm,addr,ptep,pteval) do { \ + unsigned long _pteval = (pteval); \ + fcse_account_page_removal(mm, addr, pte_val(*ptep)); \ + pte_val(_pteval) = \ + fcse_account_page_addition(mm, addr, pte_val(_pteval)); \ + set_pte_ext(ptep, _pteval, (addr) >= TASK_SIZE ? 0 : PTE_EXT_NG); \ +} while (0) /* * The following only work if pte_present() is true. @@ -355,10 +398,14 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) /* to find an entry in a page-table-directory */ #define pgd_index(addr) ((addr) >> PGDIR_SHIFT) -#define pgd_offset(mm, addr) ((mm)->pgd+pgd_index(addr)) +#define pgd_offset(mm, addr) \ + ({ \ + struct mm_struct *_mm = (mm); \ + (_mm->pgd + pgd_index(fcse_va_to_mva(_mm, (addr)))); \ + }) /* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) +#define pgd_offset_k(addr) (init_mm.pgd + pgd_index(addr)) /* Find an entry in the second-level page table.. */ #define pmd_offset(dir, addr) ((pmd_t *)(dir)) diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index 3976412..dec28b6 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -239,12 +239,13 @@ #ifdef CONFIG_MMU -#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) +#define cpu_switch_mm(pgd,mm,flush) \ + cpu_do_switch_mm(virt_to_phys(pgd), (mm), (flush)) #define cpu_get_pgd() \ ({ \ unsigned long pg; \ - __asm__("mrc p15, 0, %0, c2, c0, 0" \ + __asm__ __volatile__ ("mrc p15, 0, %0, c2, c0, 0" \ : "=r" (pg) : : "cc"); \ pg &= ~0x3fff; \ (pgd_t *)phys_to_virt(pg); \ diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 6a89567..bc33df1 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -23,9 +23,14 @@ #include #ifdef __KERNEL__ +#ifndef CONFIG_ARM_FCSE #define STACK_TOP ((current->personality & ADDR_LIMIT_32BIT) ? \ TASK_SIZE : TASK_SIZE_26) #define STACK_TOP_MAX TASK_SIZE +#else /* CONFIG_ARM_FCSE */ +#define STACK_TOP FCSE_TASK_SIZE +#define STACK_TOP_MAX FCSE_TASK_SIZE +#endif /* CONFIG_ARM_FCSE */ #endif union debug_insn { diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h index d65b2f5..b7f40fc 100644 --- a/arch/arm/include/asm/system.h +++ b/arch/arm/include/asm/system.h @@ -213,10 +213,19 @@ static inline void set_copro_access(unsigned int val) */ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info *, struct thread_info *); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH #define switch_to(prev,next,last) \ do { \ - last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ + local_irq_disable_hw_cond(); \ + last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ + local_irq_enable_hw_cond(); \ } while (0) +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ +#define switch_to(prev,next,last) \ +do { \ + last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ +} while (0) +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ #if defined(CONFIG_CPU_SA1100) || defined(CONFIG_CPU_SA110) /* @@ -277,17 +286,17 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #error SMP is not supported on this platform #endif case 1: - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = *(volatile unsigned char *)ptr; *(volatile unsigned char *)ptr = x; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); break; case 4: - raw_local_irq_save(flags); + local_irq_save_hw(flags); ret = *(volatile unsigned long *)ptr; *(volatile unsigned long *)ptr = x; - raw_local_irq_restore(flags); + local_irq_restore_hw(flags); break; #else case 1: diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index 73394e5..34e55e1 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -140,6 +140,9 @@ extern void vfp_sync_state(struct thread_info *thread); #define TIF_USING_IWMMXT 17 #define TIF_MEMDIE 18 #define TIF_FREEZE 19 +#ifdef CONFIG_IPIPE +#define TIF_MMSWITCH_INT 20 +#endif /* CONFIG_IPIPE */ #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) @@ -147,6 +150,9 @@ extern void vfp_sync_state(struct thread_info *thread); #define _TIF_POLLING_NRFLAG (1 << TIF_POLLING_NRFLAG) #define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT) #define _TIF_FREEZE (1 << TIF_FREEZE) +#ifdef CONFIG_IPIPE +#define _TIF_MMSWITCH_INT (1 << TIF_MMSWITCH_INT) +#endif /* CONFIG_IPIPE */ /* * Change these and you break ASM code in entry-common.S diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index c964f3f..43ae488 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -210,6 +210,7 @@ #ifndef __ASSEMBLY__ #include +#include struct cpu_tlb_fns { void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); @@ -350,7 +351,7 @@ static inline void local_flush_tlb_mm(struct mm_struct *mm) if (tlb_flag(TLB_WB)) dsb(); - if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) { + if (cpu_isset(smp_processor_id(), fcse_tlb_mask(mm))) { if (tlb_flag(TLB_V3_FULL)) asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (zero) : "cc"); if (tlb_flag(TLB_V4_U_FULL)) @@ -383,12 +384,13 @@ local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; - uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); + uaddr = (fcse_va_to_mva(vma->vm_mm, uaddr) & PAGE_MASK) + | ASID(vma->vm_mm); if (tlb_flag(TLB_WB)) dsb(); - if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { + if (cpu_isset(smp_processor_id(), fcse_tlb_mask(vma->vm_mm))) { if (tlb_flag(TLB_V3_PAGE)) asm("mcr p15, 0, %0, c6, c0, 0" : : "r" (uaddr) : "cc"); if (tlb_flag(TLB_V4_U_PAGE)) @@ -504,7 +506,15 @@ static inline void clean_pmd_entry(pmd_t *pmd) /* * Convert calls to our calling convention. */ -#define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) +#define local_flush_tlb_range(vma, start, end) \ + ({ \ + struct mm_struct *_mm = (vma)->vm_mm; \ + unsigned long _start, _end; \ + _start = fcse_va_to_mva(_mm, start); \ + _end = fcse_va_to_mva(_mm, end); \ + __cpu_flush_user_tlb_range(_start, _end, vma); \ + }) + #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) #ifndef CONFIG_SMP diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index ff89d0b..df0f071 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -48,5 +48,7 @@ endif head-y := head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o +obj-$(CONFIG_ARM_FCSE) += fcse.o +obj-$(CONFIG_IPIPE) += ipipe.o extra-y := $(head-y) init_task.o vmlinux.lds diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index fc8af43..418bbd3 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -4,6 +4,7 @@ * Copyright (C) 1996,1997,1998 Russell King. * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) * nommu support by Hyok S. Choi (hyok.choi@samsung.com) + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -28,14 +29,22 @@ * Interrupt handling. Preserves r7, r8, r9 */ .macro irq_handler +#ifdef CONFIG_IPIPE + mov r0, #2 +#endif get_irqnr_preamble r5, lr -1: get_irqnr_and_base r0, r6, r5, lr +1: get_irqnr_and_base r1, r6, r5, lr + movne r0, r1 movne r1, sp @ @ routine called with r0 = irq number, r1 = struct pt_regs * @ adrne lr, 1b +#ifdef CONFIG_IPIPE + bne __ipipe_grab_irq +#else bne asm_do_IRQ +#endif #ifdef CONFIG_SMP /* @@ -44,18 +53,22 @@ * this macro assumes that irqstat (r6) and base (r5) are * preserved from get_irqnr_and_base above */ - test_for_ipi r0, r6, r5, lr + test_for_ipi r1, r6, r5, lr movne r0, sp adrne lr, 1b bne do_IPI #ifdef CONFIG_LOCAL_TIMERS - test_for_ltirq r0, r6, r5, lr + test_for_ltirq r1, r6, r5, lr movne r0, sp adrne lr, 1b bne do_local_timer #endif #endif +#ifdef CONFIG_IPIPE + cmp r0, #2 + bleq __ipipe_check_root_interruptible +#endif .endm @@ -211,20 +224,34 @@ __irq_svc: #endif #ifdef CONFIG_PREEMPT get_thread_info tsk +#ifndef CONFIG_IPIPE ldr r8, [tsk, #TI_PREEMPT] @ get preempt count add r7, r8, #1 @ increment it str r7, [tsk, #TI_PREEMPT] #endif +#endif irq_handler +#ifdef CONFIG_IPIPE + cmp r0, #0 + beq __ipipe_fast_svc_irq_exit +#endif + #ifdef CONFIG_PREEMPT +#ifndef CONFIG_IPIPE str r8, [tsk, #TI_PREEMPT] @ restore preempt count +#else + ldr r8, [tsk, #TI_PREEMPT] +#endif ldr r0, [tsk, #TI_FLAGS] @ get flags teq r8, #0 @ if preempt count != 0 movne r0, #0 @ force flags to 0 tst r0, #_TIF_NEED_RESCHED blne svc_preempt #endif +#ifdef CONFIG_IPIPE +__ipipe_fast_svc_irq_exit: +#endif ldr r0, [sp, #S_PSR] @ irqs are already disabled msr spsr_cxsf, r0 #ifdef CONFIG_TRACE_IRQFLAGS @@ -240,12 +267,25 @@ ENDPROC(__irq_svc) #ifdef CONFIG_PREEMPT svc_preempt: mov r8, lr +#ifndef CONFIG_IPIPE 1: bl preempt_schedule_irq @ irq en/disable is done inside +#else /* CONFIG_IPIPE */ +1: bl __ipipe_preempt_schedule_irq @ irq en/disable is done inside +#endif /* CONFIG_IPIPE */ ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS tst r0, #_TIF_NEED_RESCHED moveq pc, r8 @ go again b 1b -#endif +#ifdef CONFIG_IPIPE +#if __GNUC__ >= 4 +.LCipipe_root_status: + .word __ipipe_root_status +#else /* gcc < 4 */ +.LCipipe_root_status_addr: + .word __ipipe_root_status_addr +#endif /* gcc < 4 */ +#endif /* CONFIG_IPIPE */ +#endif /* CONFIG_PREEMPT */ .align 5 __und_svc: @@ -258,6 +298,17 @@ __und_svc: svc_entry #endif +#ifdef CONFIG_IPIPE + /* Beware, do_undefinstr in arch/arm/traps.c will not signal the trap + when in svc mode because of this hunk. */ + mov r4, r2 + mov r0, #7 @ r0 = IPIPE_TRAP_UNDEFINSTR + mov r1, sp @ r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + bne 1f + mov r2, r4 +#endif /* CONFIG_IPIPE */ @ @ call emulation code, which returns using r9 if it has emulated @ the instruction, or the more conventional lr if we are to treat @@ -442,18 +493,28 @@ __irq_usr: #endif get_thread_info tsk #ifdef CONFIG_PREEMPT +#ifndef CONFIG_IPIPE ldr r8, [tsk, #TI_PREEMPT] @ get preempt count add r7, r8, #1 @ increment it str r7, [tsk, #TI_PREEMPT] #endif +#endif irq_handler +#ifdef CONFIG_IPIPE + cmp r0, #0 + bne __ipipe_usr_irq_continue + slow_restore_user_regs @ Fast exit path over non-root domains +__ipipe_usr_irq_continue: +#endif #ifdef CONFIG_PREEMPT +#ifndef CONFIG_IPIPE ldr r0, [tsk, #TI_PREEMPT] str r8, [tsk, #TI_PREEMPT] teq r0, r7 strne r0, [r0, -r0] #endif +#endif #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_on #endif @@ -565,7 +626,7 @@ call_fpe: mov r7, #1 strb r7, [r10, #TI_USED_CP + 10] @ mark CP#10 as used strb r7, [r10, #TI_USED_CP + 11] @ mark CP#11 as used - b do_vfp @ let VFP handler handle this + b _do_vfp @ let VFP handler handle this 1: #endif tst r0, #0x08000000 @ only CDP/CPRT/LDC/STC have bit 27 @@ -607,8 +668,8 @@ call_fpe: mov pc, lr @ CP#8 mov pc, lr @ CP#9 #ifdef CONFIG_VFP - b do_vfp @ CP#10 (VFP) - b do_vfp @ CP#11 (VFP) + b _do_vfp @ CP#10 (VFP) + b _do_vfp @ CP#11 (VFP) #else mov pc, lr @ CP#10 (VFP) mov pc, lr @ CP#11 (VFP) @@ -643,11 +704,43 @@ call_fpe: #endif do_fpe: +#ifdef CONFIG_IPIPE + mov r4, r0 + mov r5, r2 + mov r6, lr + mov r0, #5 @ r0 = IPIPE_TRAP_FPU + mov r1, sp @ r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + movne pc, r9 + mov lr, r6 + mov r2, r5 + mov r0, r4 +#endif enable_irq ldr r4, .LCfp add r10, r10, #TI_FPSTATE @ r10 = workspace ldr pc, [r4] @ Call FP module USR entry point +#ifdef CONFIG_VFP +_do_vfp: +#ifdef CONFIG_IPIPE + mov r4, r0 + mov r5, r2 + mov r6, lr + mov r0, #6 @ r0 = IPIPE_TRAP_VFP + mov r1, sp @ r1 = ®s + bl __ipipe_dispatch_event @ branch to trap handler + cmp r0, #0 + movne pc, r9 + mov lr, r6 + mov r2, r5 + mov r0, r4 +#endif + b do_vfp +#endif + + /* * The FP module is called with these registers set: * r0 = instruction @@ -694,6 +787,13 @@ __pabt_usr: ENTRY(ret_from_exception) UNWIND(.fnstart ) UNWIND(.cantunwind ) +#ifdef CONFIG_IPIPE + bl __ipipe_check_root + cmp r0, #0 + bne 1f + slow_restore_user_regs @ Fast exit path over non-root domains +1: +#endif /* CONFIG_IPIPE */ get_thread_info tsk mov why, #0 b ret_to_user @@ -735,7 +835,11 @@ ENTRY(__switch_to) add r4, r2, #TI_CPU_SAVE ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH +#ifndef CONFIG_IPIPE bl atomic_notifier_call_chain +#else /* !CONFIG_IPIPE */ + bl __ipipe_switch_to_notifier_call_chain +#endif /* !CONFIG_IPIPE */ mov r0, r5 ldmia r4, {r4 - sl, fp, sp, pc} @ Load all regs saved previously UNWIND(.fnend ) @@ -773,6 +877,18 @@ ENDPROC(__switch_to) * purpose. */ +#ifdef CONFIG_IPIPE +/* We use the same mechanism as Linux user helpers to store variables related to + TSC emulation, so that they can be used in user-space. */ + .align 5 + .globl __ipipe_tsc_area_start + +__ipipe_tsc_area_start: + .rep 12 + .word 0 + .endr +#endif /* CONFIG_IPIPE */ + .macro usr_ret, reg #ifdef CONFIG_ARM_THUMB bx \reg diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 8c3de1a..86382d3 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -2,6 +2,7 @@ * linux/arch/arm/kernel/entry-common.S * * Copyright (C) 2000 Russell King + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -22,6 +23,11 @@ * possible here, and this includes saving r0 back into the SVC * stack. */ +#ifdef CONFIG_IPIPE +__ipipe_ret_fast_syscall: + ldr r0, [sp, #S_R0+S_OFF] @ returned r0 + /* fall through */ +#endif ret_fast_syscall: UNWIND(.fnstart ) UNWIND(.cantunwind ) @@ -30,10 +36,10 @@ ret_fast_syscall: tst r1, #_TIF_WORK_MASK bne fast_work_pending +fast_restore_user_regs: /* perform architecture specific actions before user return */ arch_ret_to_user r1, lr - @ fast_restore_user_regs ldr r1, [sp, #S_OFF + S_PSR] @ get calling cpsr ldr lr, [sp, #S_OFF + S_PC]! @ get pc msr spsr_cxsf, r1 @ save in spsr_svc @@ -41,6 +47,14 @@ ret_fast_syscall: mov r0, r0 add sp, sp, #S_FRAME_SIZE - S_PC movs pc, lr @ return & move spsr_svc into cpsr + +#ifdef CONFIG_IPIPE +__ipipe_fast_exit_syscall: + ldr r0, [sp, #S_R0+S_OFF] @ returned r0 + disable_irq @ disable interrupts + b fast_restore_user_regs +#endif /* CONFIG_IPIPE */ + UNWIND(.fnend ) /* @@ -53,13 +67,16 @@ work_pending: bne work_resched tst r1, #_TIF_SIGPENDING beq no_work_pending + enable_irq_cond mov r0, sp @ 'regs' mov r2, why @ 'syscall' bl do_notify_resume b ret_slow_syscall @ Check work again work_resched: + enable_irq_cond bl schedule + /* * "slow" syscall return path. "why" tells us if this was a real syscall. */ @@ -70,23 +87,14 @@ ret_slow_syscall: tst r1, #_TIF_WORK_MASK bne work_pending no_work_pending: - /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr - - @ slow_restore_user_regs - ldr r1, [sp, #S_PSR] @ get calling cpsr - ldr lr, [sp, #S_PC]! @ get pc - msr spsr_cxsf, r1 @ save in spsr_svc - ldmdb sp, {r0 - lr}^ @ get calling r0 - lr - mov r0, r0 - add sp, sp, #S_FRAME_SIZE - S_PC - movs pc, lr @ return & move spsr_svc into cpsr + slow_restore_user_regs ENDPROC(ret_to_user) /* * This is how we return from a fork. */ ENTRY(ret_from_fork) + enable_irq_cond bl schedule_tail get_thread_info tsk ldr r1, [tsk, #TI_FLAGS] @ check for syscall tracing @@ -268,6 +276,18 @@ ENTRY(vector_swi) #endif stmdb sp!, {r4, r5} @ push fifth and sixth args +#ifdef CONFIG_IPIPE + stmfd sp!, {r0-r3, ip} + sub sp, sp, #4 + add r1, sp, #S_OFF+24 + mov r0, scno + bl __ipipe_syscall_root + cmp r0, #0 + add sp, sp, #4 + ldmfd sp!, {r0-r3, ip} + blt __ipipe_ret_fast_syscall + bgt __ipipe_fast_exit_syscall +#endif /* CONFIG_IPIPE */ tst ip, #_TIF_SYSCALL_TRACE @ are we tracing syscalls? bne __sys_trace @@ -315,6 +335,9 @@ __sys_trace_return: __cr_alignment: .word cr_alignment #endif +#ifdef CONFIG_IPIPE + .word __ipipe_syscall_root +#endif .ltorg /* @@ -485,3 +508,28 @@ ENTRY(sys_oabi_call_table) #endif + +#ifdef CONFIG_FRAME_POINTER + + .text + .align 0 + .type arm_return_addr %function + .global arm_return_addr + +arm_return_addr: + mov ip, r0 + mov r0, fp +3: + cmp r0, #0 + beq 1f @ frame list hit end, bail + cmp ip, #0 + beq 2f @ reached desired frame + ldr r0, [r0, #-12] @ else continue, get next fp + sub ip, ip, #1 + b 3b +2: + ldr r0, [r0, #-4] @ get target return address +1: + mov pc, lr + +#endif diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 87ab4e1..f72280f 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -49,6 +49,17 @@ #endif .endm + .macro slow_restore_user_regs + /* perform architecture specific actions before user return */ + arch_ret_to_user r1, lr + ldr r1, [sp, #S_PSR] @ get calling cpsr + ldr lr, [sp, #S_PC]! @ get pc + msr spsr_cxsf, r1 @ save in spsr_svc + ldmdb sp, {r0 - lr}^ @ get calling r0 - lr + mov r0, r0 + add sp, sp, #S_FRAME_SIZE - S_PC + movs pc, lr @ return & move spsr_svc into cpsr + .endm /* * These are the registers used in the syscall handler, and allow us to diff --git a/arch/arm/kernel/fcse.c b/arch/arm/kernel/fcse.c new file mode 100644 index 0000000..0cfb288 --- /dev/null +++ b/arch/arm/kernel/fcse.c @@ -0,0 +1,176 @@ +#include +#include +#include +#include + +#include +#include +#include + +#define NR_PIDS (TASK_SIZE / FCSE_PID_TASK_SIZE) +#define PIDS_LONGS ((NR_PIDS + BITS_PER_LONG - 1) / BITS_PER_LONG) + +static IPIPE_DEFINE_SPINLOCK(fcse_lock); +static unsigned long fcse_pids_bits[PIDS_LONGS]; +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +static unsigned long fcse_pids_cache_dirty[PIDS_LONGS]; +static unsigned random_pid; +struct { + struct mm_struct *last_mm; + unsigned count; +} per_pid[NR_PIDS]; +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + +static void fcse_pid_reference_inner(unsigned pid) +{ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (++per_pid[pid].count == 1) +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + __set_bit(pid, fcse_pids_bits); +} + +static void fcse_pid_dereference(unsigned pid) +{ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (--per_pid[pid].count == 0) { + __clear_bit(pid, fcse_pids_bits); + per_pid[pid].last_mm = NULL; + } +#else /* CONFIG_ARM_FCSE_BEST_EFFORT */ + __clear_bit(pid, fcse_pids_bits); +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +} + +int fcse_pid_alloc(void) +{ + unsigned long flags; + unsigned pid; + + spin_lock_irqsave(&fcse_lock, flags); + pid = find_next_zero_bit(fcse_pids_bits, NR_PIDS, 1); + if (pid == NR_PIDS) { + /* Allocate zero pid last, since zero pid is also used by + processes with address space larger than 32MB in + best-effort mode. */ + if (!test_bit(0, fcse_pids_bits)) + pid = 0; + else { +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if(++random_pid == NR_PIDS) + random_pid = 0; + pid = random_pid; +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + spin_unlock_irqrestore(&fcse_lock, flags); + return -EAGAIN; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + } + } + fcse_pid_reference_inner(pid); + spin_unlock_irqrestore(&fcse_lock, flags); + + return pid; +} + +void fcse_pid_free(unsigned pid) +{ + unsigned long flags; + + spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_dereference(pid); + spin_unlock_irqrestore(&fcse_lock, flags); +} + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +static void fcse_notify_flush_all_inner(struct mm_struct *next) +{ + unsigned long flags; + + spin_lock_irqsave(&fcse_lock, flags); + switch(ARRAY_SIZE(fcse_pids_cache_dirty)) { + case 4: + fcse_pids_cache_dirty[3] = 0UL; + case 3: + fcse_pids_cache_dirty[2] = 0UL; + case 2: + fcse_pids_cache_dirty[1] = 0UL; + case 1: + fcse_pids_cache_dirty[0] = 0UL; + } + if (next != &init_mm && next) { + unsigned pid = next->context.pid >> FCSE_PID_SHIFT; + __set_bit(pid, fcse_pids_cache_dirty); + } + spin_unlock_irqrestore(&fcse_lock, flags); +} + +int fcse_needs_flush(struct mm_struct *prev, struct mm_struct *next) +{ + unsigned res, reused_pid = 0, pid = next->context.pid >> FCSE_PID_SHIFT; + unsigned long flags; + + spin_lock_irqsave(&fcse_lock, flags); + if (per_pid[pid].last_mm != next) { + if (per_pid[pid].last_mm) + reused_pid = test_bit(pid, fcse_pids_cache_dirty); + per_pid[pid].last_mm = next; + } + __set_bit(pid, fcse_pids_cache_dirty); + spin_unlock_irqrestore(&fcse_lock, flags); + + res = reused_pid + || next->context.high_pages + || !prev + || prev->context.shared_dirty_pages + || prev->context.high_pages; + + if (res) { + cpu_clear(smp_processor_id(), prev->cpu_vm_mask); + fcse_notify_flush_all_inner(next); + } + + return res; +} +EXPORT_SYMBOL_GPL(fcse_needs_flush); + +void fcse_notify_flush_all(void) +{ + fcse_notify_flush_all_inner(current->mm); +} + +void fcse_pid_reference(unsigned pid) +{ + unsigned long flags; + + spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_reference_inner(pid); + spin_unlock_irqrestore(&fcse_lock, flags); +} + +/* Called with mm->mmap_sem write-locked. */ +void fcse_relocate_mm_to_null_pid(struct mm_struct *mm) +{ + pgd_t *to = mm->pgd + pgd_index(0); + pgd_t *from = pgd_offset(mm, 0); + unsigned len = pgd_index(FCSE_TASK_SIZE) * sizeof(*from); + unsigned long flags; + + preempt_disable(); + + memcpy(to, from, len); + spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_dereference(mm->context.pid >> FCSE_PID_SHIFT); + fcse_pid_reference_inner(0); + per_pid[0].last_mm = mm; + spin_unlock_irqrestore(&fcse_lock, flags); + + mm->context.pid = 0; + fcse_pid_set(0); + memset(from, '\0', len); + mb(); + flush_cache_mm(mm); + flush_tlb_mm(mm); + + preempt_enable(); +} + +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ diff --git a/arch/arm/kernel/ipipe.c b/arch/arm/kernel/ipipe.c new file mode 100644 index 0000000..e46a90f --- /dev/null +++ b/arch/arm/kernel/ipipe.c @@ -0,0 +1,532 @@ +/* -*- linux-c -*- + * linux/arch/arm/kernel/ipipe.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2004 Wolfgang Grandegger (Adeos/arm port over 2.4). + * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes). + * Copyright (C) 2005 Stelian Pop. + * Copyright (C) 2006-2008 Gilles Chanteperdrix. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-dependent I-PIPE support for ARM. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Next tick date (timebase value). */ +DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); +DEFINE_PER_CPU(struct mm_struct *,ipipe_active_mm); +EXPORT_PER_CPU_SYMBOL(ipipe_active_mm); +#ifdef __IPIPE_FEATURE_PIC_MUTE +__ipipe_irqbits_t __ipipe_irqbits; +IPIPE_DEFINE_SPINLOCK(__ipipe_irqbits_lock); +#endif /* __IPIPE_FEATURE_PIC_MUTE */ + +extern struct irq_desc irq_desc[]; +asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs); + +#ifdef CONFIG_SMP + +static cpumask_t __ipipe_cpu_sync_map; + +static cpumask_t __ipipe_cpu_lock_map; + +static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier); + +static atomic_t __ipipe_critical_count = ATOMIC_INIT(0); + +static void (*__ipipe_cpu_sync) (void); + +/* Always called with hw interrupts off. */ + + +void __ipipe_do_critical_sync(unsigned irq) +{ + int cpu = ipipe_processor_id(); + + cpu_set(cpu, __ipipe_cpu_sync_map); + + /* + * Now we are in sync with the lock requestor running on another + * CPU. Enter a spinning wait until he releases the global + * lock. + */ + spin_lock(&__ipipe_cpu_barrier); + + /* Got it. Now get out. */ + + if (__ipipe_cpu_sync) + /* Call the sync routine if any. */ + __ipipe_cpu_sync(); + + spin_unlock(&__ipipe_cpu_barrier); + + cpu_clear(cpu, __ipipe_cpu_sync_map); +} + +#endif /* CONFIG_SMP */ + +/* + * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline + * just like if it has been actually received from a hw source. Also + * works for virtual interrupts. + */ +int ipipe_trigger_irq(unsigned irq) +{ + unsigned long flags; + + if (irq >= IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) + return -EINVAL; + + local_irq_save_hw(flags); + + __ipipe_handle_irq(irq, NULL); + + local_irq_restore_hw(flags); + + return 1; +} + +int ipipe_get_sysinfo(struct ipipe_sysinfo *info) +{ + info->ncpus = num_online_cpus(); + info->cpufreq = ipipe_cpu_freq(); + info->archdep.tmirq = __ipipe_mach_timerint; + info->archdep.tmfreq = info->cpufreq; + __ipipe_mach_get_tscinfo(&info->archdep.tsc); + + return 0; +} + +static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc) +{ + desc->ipipe_ack(irq, desc); +} + +static void __ipipe_ack_timerirq(unsigned irq, struct irq_desc *desc) +{ + desc->ipipe_ack(irq, desc); + __ipipe_mach_acktimer(); + desc->ipipe_end(irq, desc); +} + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + unsigned long flags; + irq_desc[irq].status &= ~IRQ_DISABLED; +#ifdef __IPIPE_FEATURE_PIC_MUTE + if (ipd == &ipipe_root) + return; + + spin_lock_irqsave(&__ipipe_irqbits_lock, flags); + __ipipe_irqbits[irq / BITS_PER_LONG] &= ~(1 << (irq % BITS_PER_LONG)); + spin_unlock_irqrestore(&__ipipe_irqbits_lock, flags); +#else + (void) flags; +#endif /* __IPIPE_FEATURE_PIC_MUTE */ +} +EXPORT_SYMBOL(__ipipe_enable_irqdesc); + +#ifdef __IPIPE_FEATURE_PIC_MUTE +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + unsigned long flags; + + if (ipd == &ipipe_root) + return; + + spin_lock_irqsave(&__ipipe_irqbits_lock, flags); + __ipipe_irqbits[irq / BITS_PER_LONG] |= 1 << (irq % BITS_PER_LONG); + spin_unlock_irqrestore(&__ipipe_irqbits_lock, flags); +} +EXPORT_SYMBOL(__ipipe_disable_irqdesc); +EXPORT_SYMBOL(ipipe_mute_pic); +EXPORT_SYMBOL(ipipe_unmute_pic); +#endif /* __IPIPE_FEATURE_PIC_MUTE */ + +/* + * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw + * interrupts are off, and secondary CPUs are still lost in space. + */ +void __ipipe_enable_pipeline(void) +{ + unsigned long flags; + unsigned irq; + +/* We do not want "wfi" to be called in arm926ejs based processor, as + this causes the I-cache to be disabled when idle. */ +#ifdef CONFIG_CPU_ARM926T + disable_hlt(); +#endif + + flags = ipipe_critical_enter(NULL); + + /* First, virtualize all interrupts from the root domain. */ + + for (irq = 0; irq < NR_IRQS; irq++) + ipipe_virtualize_irq(ipipe_root_domain, + irq, + (ipipe_irq_handler_t)&asm_do_IRQ, NULL, + ((irq == __ipipe_mach_timerint) + ? &__ipipe_ack_timerirq + : &__ipipe_ack_irq), + IPIPE_HANDLE_MASK | IPIPE_PASS_MASK); + + ipipe_critical_exit(flags); +} + +/* + * ipipe_critical_enter() -- Grab the superlock excluding all CPUs + * but the current one from a critical section. This lock is used when + * we must enforce a global critical section for a single CPU in a + * possibly SMP system whichever context the CPUs are running. + */ +unsigned long ipipe_critical_enter(void (*syncfn) (void)) +{ + unsigned long flags; + + local_irq_save_hw(flags); + +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */ + int cpu = ipipe_processor_id(); + cpumask_t lock_map; + + if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) { + while (cpu_test_and_set(BITS_PER_LONG - 1, + __ipipe_cpu_lock_map)) { + int n = 0; + do { + cpu_relax(); + } while (++n < cpu); + } + + spin_lock(&__ipipe_cpu_barrier); + + __ipipe_cpu_sync = syncfn; + + /* Send the sync IPI to all processors but the current one. */ + send_IPI_allbutself(IPIPE_CRITICAL_VECTOR); + + cpus_andnot(lock_map, cpu_online_map, + __ipipe_cpu_lock_map); + + while (!cpus_equal(__ipipe_cpu_sync_map, lock_map)) + cpu_relax(); + } + + atomic_inc(&__ipipe_critical_count); + } +#endif /* CONFIG_SMP */ + + return flags; +} + +/* ipipe_critical_exit() -- Release the superlock. */ + +void ipipe_critical_exit(unsigned long flags) +{ +#ifdef CONFIG_SMP + if (num_online_cpus() > 1) { /* We might be running a SMP-kernel on a UP box... */ + if (atomic_dec_and_test(&__ipipe_critical_count)) { + spin_unlock(&__ipipe_cpu_barrier); + + while (!cpus_empty(__ipipe_cpu_sync_map)) + cpu_relax(); + + cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map); + cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map); + } + } +#endif /* CONFIG_SMP */ + + local_irq_restore_hw(flags); +} + +#ifdef CONFIG_PREEMPT + +asmlinkage void __sched __ipipe_preempt_schedule_irq(void) +{ + extern asmlinkage void __sched preempt_schedule_irq(void); + struct ipipe_percpu_domain_data *p; + unsigned long flags; + /* + * preempt_schedule_irq expects the root stage to be stalled. + */ + BUG_ON(!irqs_disabled_hw()); + local_irq_save(flags); + local_irq_enable_hw(); + preempt_schedule_irq(); /* Ok, may reschedule now. */ + local_irq_disable_hw(); + + /* + * Flush any pending interrupt that may have been logged after + * preempt_schedule_irq() stalled the root stage before + * returning to us, and now. + */ + p = ipipe_root_cpudom_ptr(); + if (unlikely(p->irqpend_himask != 0)) { + add_preempt_count(PREEMPT_ACTIVE); + clear_bit(IPIPE_STALL_FLAG, &p->status); + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + sub_preempt_count(PREEMPT_ACTIVE); + } + + __local_irq_restore_nosync(flags); +} + +#endif /* CONFIG_PREEMPT */ + +asmlinkage int __ipipe_check_root(void) +{ + return ipipe_root_domain_p; +} + +asmlinkage int __ipipe_check_root_interruptible(void) +{ + return ipipe_root_domain_p && !__ipipe_test_root(); +} + +__kprobes int +__ipipe_switch_to_notifier_call_chain(struct atomic_notifier_head *nh, + unsigned long val, void *v) +{ + unsigned long flags; + int rc; + + local_irq_save(flags); + rc = atomic_notifier_call_chain(nh, val, v); + __local_irq_restore_nosync(flags); + + return rc; +} + +asmlinkage int __ipipe_syscall_root(unsigned long scno, struct pt_regs *regs) +{ + unsigned long flags, origr7; + + /* We use r7 to pass the syscall number to the other domains */ + origr7 = regs->ARM_r7; + regs->ARM_r7 = __NR_SYSCALL_BASE + scno; + /* + * This routine either returns: + * 0 -- if the syscall is to be passed to Linux; + * >0 -- if the syscall should not be passed to Linux, and no + * tail work should be performed; + * <0 -- if the syscall should not be passed to Linux but the + * tail work has to be performed (for handling signals etc). + */ + + if (__ipipe_syscall_watched_p(current, regs->ARM_r7) && + __ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL) && + __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL,regs) > 0) { + if (ipipe_root_domain_p && !in_atomic()) { + /* + * Sync pending VIRQs before _TIF_NEED_RESCHED + * is tested. + */ + local_irq_save_hw(flags); + if ((ipipe_root_cpudom_var(irqpend_himask) & IPIPE_IRQMASK_VIRT) != 0) + __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); + local_irq_restore_hw(flags); + regs->ARM_r7 = origr7; + return -1; + } + regs->ARM_r7 = origr7; + return 1; + } + + regs->ARM_r7 = origr7; + return 0; +} + +/* + * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic + * interrupt protection log is maintained here for each domain. Hw + * interrupts are off on entry. + */ +int __ipipe_handle_irq(int irq, struct pt_regs *regs) +{ + struct ipipe_domain *this_domain, *next_domain; + struct list_head *head, *pos; + int m_ack; + + m_ack = (regs == NULL); + + if (irq >= IPIPE_NR_IRQS) { + printk(KERN_ERR "I-pipe: spurious interrupt %d\n", irq); + goto finalize_nosync; + } + + this_domain = ipipe_current_domain; + + if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control)) + head = &this_domain->p_link; + else { + head = __ipipe_pipeline.next; + next_domain = list_entry(head, struct ipipe_domain, p_link); + if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) { + if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) + next_domain->irqs[irq].acknowledge(irq, irq_desc + irq); + __ipipe_dispatch_wired(next_domain, irq); + goto finalize_nosync; + } + } + + /* Ack the interrupt. */ + + pos = head; + + while (pos != &__ipipe_pipeline) { + next_domain = list_entry(pos, struct ipipe_domain, p_link); + + /* + * For each domain handling the incoming IRQ, mark it + * as pending in its log. + */ + if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) { + /* + * Domains that handle this IRQ are polled for + * acknowledging it by decreasing priority + * order. The interrupt must be made pending + * _first_ in the domain's status flags before + * the PIC is unlocked. + */ + __ipipe_set_irq_pending(next_domain, irq); + + if (!m_ack && next_domain->irqs[irq].acknowledge) { + next_domain->irqs[irq].acknowledge(irq, irq_desc + irq); + m_ack = 1; + } + } + + /* + * If the domain does not want the IRQ to be passed + * down the interrupt pipe, exit the loop now. + */ + + if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control)) + break; + + pos = next_domain->p_link.next; + } + + /* + * If the interrupt preempted the head domain, then do not + * even try to walk the pipeline, unless an interrupt is + * pending for it. + */ + if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) && + ipipe_head_cpudom_var(irqpend_himask) == 0) + goto finalize_nosync; + + /* + * Now walk the pipeline, yielding control to the highest + * priority domain that has pending interrupt(s) or + * immediately to the current domain if the interrupt has been + * marked as 'sticky'. This search does not go beyond the + * current domain in the pipeline. + */ + + __ipipe_walk_pipeline(head); + +finalize_nosync: + if (!ipipe_root_domain_p || __ipipe_test_root()) + return 0; + +#ifdef CONFIG_SMP + /* + * Prevent a spurious rescheduling from being triggered on + * preemptible kernels along the way out through + * ret_from_intr. + */ + if (!regs) + __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); +#endif /* CONFIG_SMP */ + + return 1; +} + +asmlinkage int __ipipe_grab_irq(int irq, struct pt_regs *regs) +{ + int status; + +#ifdef irq_finish + /* AT91 specific workaround */ + irq_finish(irq); +#endif /* irq_finish */ + + if (irq == __ipipe_mach_timerint) { + /* + * Given our deferred dispatching model for regular IRQs, we + * only record CPU regs for the last timer interrupt, so that + * the timer handler charges CPU times properly. It is assumed + * that other interrupt handlers don't actually care for such + * information. + */ + __raw_get_cpu_var(__ipipe_tick_regs).ARM_cpsr = + (ipipe_root_domain_p + ? regs->ARM_cpsr + : regs->ARM_cpsr | PSR_I_BIT); + __raw_get_cpu_var(__ipipe_tick_regs).ARM_pc = regs->ARM_pc; + } + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_begin(regs->ARM_ORIG_r0); +#endif + + status = __ipipe_handle_irq(irq, regs); + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_end(regs->ARM_ORIG_r0); +#endif + + return status; +} + +EXPORT_SYMBOL_GPL(show_stack); +#ifndef MULTI_CPU +EXPORT_SYMBOL_GPL(cpu_do_switch_mm); +#endif +EXPORT_SYMBOL_GPL(__check_kvm_seq); +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +EXPORT_SYMBOL_GPL(tasklist_lock); +#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */ + +#ifdef CONFIG_CPU_HAS_ASID +EXPORT_SYMBOL_GPL(__new_context); +EXPORT_SYMBOL_GPL(cpu_last_asid); +#endif /* CONFIG_CPU_HAS_ASID */ diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c index b7c3490..ab70e80 100644 --- a/arch/arm/kernel/irq.c +++ b/arch/arm/kernel/irq.c @@ -121,8 +121,10 @@ asmlinkage void __exception asm_do_IRQ(unsigned int irq, struct pt_regs *regs) generic_handle_irq(irq); } +#ifndef CONFIG_IPIPE /* AT91 specific workaround */ irq_finish(irq); +#endif /* !CONFIG_IPIPE */ irq_exit(); set_irq_regs(old_regs); diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index 39196df..e4ce47b 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -127,8 +127,15 @@ EXPORT_SYMBOL_GPL(arm_pm_restart); */ static void default_idle(void) { - if (!need_resched()) + if (!need_resched()) { +#ifdef CONFIG_IPIPE + __ipipe_unstall_root(); +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + ipipe_trace_end(0x8000000E); +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ +#endif /* CONFIG_IPIPE */ arch_idle(); + } local_irq_enable(); } @@ -148,6 +155,7 @@ void cpu_idle(void) /* endless idle loop with no priority at all */ while (1) { tick_nohz_stop_sched_tick(1); + ipipe_suspend_domain(); leds_event(led_idle_start); while (!need_resched()) { #ifdef CONFIG_HOTPLUG_CPU diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index 89882a1..6116290 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -479,6 +479,10 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs) static int break_trap(struct pt_regs *regs, unsigned int instr) { + + if (ipipe_trap_notify(IPIPE_TRAP_BREAK,regs)) + return 0; + ptrace_break(current, regs); return 0; } diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index de885fd..b0b2d81 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -36,6 +36,7 @@ #include #include #include +#include /* * as from 2.5, kernels no longer have an init_tasks structure @@ -643,7 +644,7 @@ void flush_tlb_all(void) void flush_tlb_mm(struct mm_struct *mm) { if (tlb_ops_need_broadcast()) - on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &mm->cpu_vm_mask); + on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, &fcse_tlb_mask(mm)); else local_flush_tlb_mm(mm); } @@ -654,7 +655,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) struct tlb_args ta; ta.ta_vma = vma; ta.ta_start = uaddr; - on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &vma->vm_mm->cpu_vm_mask); + on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, &fcse_tlb_mask(vma->vm_mm)); } else local_flush_tlb_page(vma, uaddr); } @@ -677,7 +678,7 @@ void flush_tlb_range(struct vm_area_struct *vma, ta.ta_vma = vma; ta.ta_start = start; ta.ta_end = end; - on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &vma->vm_mm->cpu_vm_mask); + on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, &fcse_tlb_mask(vma->vm_mm)); } else local_flush_tlb_range(vma, start, end); } diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 57eb0f6..5756efa 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -325,6 +325,11 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) siginfo_t info; void __user *pc; + /* In kernel mode, the trap was already signaled at the beginning of + * __und_svc in arch/arm/kernel/entry-armv.S */ + if (user_mode(regs) && ipipe_trap_notify(IPIPE_TRAP_UNDEFINSTR, regs)) + return; + /* * According to the ARM ARM, PC is 2 or 4 bytes ahead, * depending whether we're in Thumb mode or not. @@ -719,13 +724,22 @@ void __init trap_init(void) return; } +#ifdef CONFIG_IPIPE +void *__ipipe_tsc_area; +#endif /* CONFIG_IPIPE */ + void __init early_trap_init(void) { unsigned long vectors = CONFIG_VECTORS_BASE; extern char __stubs_start[], __stubs_end[]; extern char __vectors_start[], __vectors_end[]; +#ifndef CONFIG_IPIPE extern char __kuser_helper_start[], __kuser_helper_end[]; int kuser_sz = __kuser_helper_end - __kuser_helper_start; +#else /* !CONFIG_IPIPE */ + extern char __ipipe_tsc_area_start[], __kuser_helper_end[]; + int kuser_sz = __kuser_helper_end - __ipipe_tsc_area_start; +#endif /* !CONFIG_IPIPE */ /* * Copy the vectors, stubs and kuser helpers (in entry-armv.S) @@ -734,7 +748,13 @@ void __init early_trap_init(void) */ memcpy((void *)vectors, __vectors_start, __vectors_end - __vectors_start); memcpy((void *)vectors + 0x200, __stubs_start, __stubs_end - __stubs_start); +#ifndef CONFIG_IPIPE memcpy((void *)vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz); +#else /* !CONFIG_IPIPE */ + BUG_ON(0x1000 - kuser_sz < 0x200 + __stubs_end - __stubs_start); + memcpy((void *)vectors + 0x1000 - kuser_sz, __ipipe_tsc_area_start, kuser_sz); + __ipipe_tsc_area = (void *)vectors + 0x1000 - kuser_sz; +#endif /* !CONFIG_IPIPE */ /* * Copy signal return handlers into the vector page, and diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig index 323b47f..c2bc8ea 100644 --- a/arch/arm/mach-at91/Kconfig +++ b/arch/arm/mach-at91/Kconfig @@ -311,6 +311,19 @@ endif # ---------------------------------------------------------- +comment "Adeos I-pipe Options" + +config IPIPE_AT91_TC + depends on IPIPE + int "AT91 TC used as time base by Adeos I-pipe" + default 0 + help + When Adeos interrupt pipeline is enabled, TC0 is used by default + as time base, but you can use TC1 or TC2 by setting this variable to 1 + or 2. This should only be needed to avoid conflicts with other drivers. + +# ---------------------------------------------------------- + comment "AT91 Board Options" config MTD_AT91_DATAFLASH_CARD diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile index c69ff23..228b8e9 100644 --- a/arch/arm/mach-at91/Makefile +++ b/arch/arm/mach-at91/Makefile @@ -71,3 +71,14 @@ obj-$(CONFIG_AT91_SLOW_CLOCK) += pm_slowclock.o ifeq ($(CONFIG_PM_DEBUG),y) CFLAGS_pm.o += -DDEBUG endif + +ifeq ($(CONFIG_IPIPE),y) +obj-y := $(filter-out at91rm9200_time.o at91sam926x_time.o at91x40_time.o, $(obj-y)) +obj-$(CONFIG_ARCH_AT91RM9200) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91SAM9260) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91SAM9261) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91SAM9263) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91SAM9RL) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91SAM9G20) += at91_ipipe_time.o +obj-$(CONFIG_ARCH_AT91X40) += at91_ipipe_time.o +endif diff --git a/arch/arm/mach-at91/at91_ipipe_time.c b/arch/arm/mach-at91/at91_ipipe_time.c new file mode 100644 index 0000000..1621ffc --- /dev/null +++ b/arch/arm/mach-at91/at91_ipipe_time.c @@ -0,0 +1,418 @@ +/* + * linux/arch/arm/mach-at91/at91_ipipe_time.c + * + * Copyright (C) 2007 Gilles Chanteperdrix + * + * Adaptation to AT91SAM926x: + * Copyright (C) 2007 Gregory CLEMENT, Adeneo + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include "clock.h" + +#if defined(CONFIG_ARCH_AT91RM9200) +#define AT91_ID_TC0 AT91RM9200_ID_TC0 +#define AT91_ID_TC1 AT91RM9200_ID_TC1 +#define AT91_ID_TC2 AT91RM9200_ID_TC2 +#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9G20) +#define AT91_ID_TC0 AT91SAM9260_ID_TC0 +#define AT91_ID_TC1 AT91SAM9260_ID_TC1 +#define AT91_ID_TC2 AT91SAM9260_ID_TC2 +#elif defined(CONFIG_ARCH_AT91SAM9261) +#define AT91_ID_TC0 AT91SAM9261_ID_TC0 +#define AT91_ID_TC1 AT91SAM9261_ID_TC1 +#define AT91_ID_TC2 AT91SAM9261_ID_TC2 +#elif defined(CONFIG_ARCH_AT91SAM9263) +#define AT91_ID_TC0 AT91SAM9263_ID_TCB +#define AT91_ID_TC1 AT91SAM9263_ID_TCB +#define AT91_ID_TC2 AT91SAM9263_ID_TCB +#elif defined(CONFIG_ARCH_AT91SAM9RL) +#define AT91_ID_TC0 AT91SAM9RL_ID_TC0 +#define AT91_ID_TC1 AT91SAM9RL_ID_TC1 +#define AT91_ID_TC2 AT91SAM9RL_ID_TC2 +#elif defined(CONFIG_ARCH_AT91X40) +#define AT91_ID_TC0 AT91X40_ID_TC0 +#define AT91_ID_TC1 AT91X40_ID_TC1 +#define AT91_ID_TC2 AT91X40_ID_TC2 +#else +#error "AT91 processor unsupported by Adeos" +#endif + +#if (CONFIG_IPIPE_AT91_TC==0) +# define KERNEL_TIMER_IRQ_NUM AT91_ID_TC0 +#elif (CONFIG_IPIPE_AT91_TC==1) +# define KERNEL_TIMER_IRQ_NUM AT91_ID_TC1 +#elif (CONFIG_IPIPE_AT91_TC==2) +# define KERNEL_TIMER_IRQ_NUM AT91_ID_TC2 +#else +#error IPIPE_AT91_TC must be 0, 1 or 2. +#endif + +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ + +#define TCNXCNS(timer,v) ((v) << ((timer)<<1)) +#define AT91_TC_REG_MASK (0xffff) + +static unsigned long next_match; + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + union { + struct { + unsigned short mid; + unsigned short low; + }; + unsigned long mid_low; + }; + }; +#else /* __LITTLE_ENDIAN */ + struct { + union { + struct { + unsigned short low; + unsigned short mid; + }; + unsigned long mid_low; + }; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; +static unsigned max_delta_ticks, min_delta_ticks; +static struct clock_event_device clkevt; +static int tc_timer_clock; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} + +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + info->u.fr.counter = + (unsigned *) + (AT91_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC + AT91_TC_CV); + info->u.fr.mask = AT91_TC_REG_MASK; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static inline unsigned int at91_tc_read(unsigned int reg_offset) +{ + unsigned long addr = + (AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC); + + return readl((void __iomem *)(addr + reg_offset)); +} + +static inline void at91_tc_write(unsigned int reg_offset, unsigned long value) +{ + unsigned long addr = + (AT91_VA_BASE_TCB0 + 0x40 * CONFIG_IPIPE_AT91_TC); + + writel(value, (void __iomem *)(addr + reg_offset)); +} + +#define read_CV() at91_tc_read(AT91_TC_CV) +#define read_RC() at91_tc_read(AT91_TC_RC) +#define write_RC(value) at91_tc_write(AT91_TC_RC, value) + +int __ipipe_mach_timerint = KERNEL_TIMER_IRQ_NUM; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static void ipipe_mach_update_tsc(unsigned short stamp); + +static int at91_timer_initialized; + +/* + * IRQ handler for the timer. + */ +static irqreturn_t at91_timer_interrupt(int irq, void *dev_id) +{ + clkevt.event_handler(&clkevt); + return IRQ_HANDLED; +} + +static struct irqaction at91_timer_irq = { + .name = "at91_tick", + .flags = IRQF_DISABLED | IRQF_TIMER, + .handler = &at91_timer_interrupt +}; + +static void ipipe_mach_update_tsc(unsigned short stamp) +{ + union tsc_reg *local_tsc; + + local_tsc = &tsc[ipipe_processor_id()]; + if (unlikely(stamp < local_tsc->low)) + local_tsc->full += AT91_TC_REG_MASK + 1; + local_tsc->low = stamp; +} + +void __ipipe_mach_acktimer(void) +{ + at91_tc_read(AT91_TC_SR); + + if (unlikely(!__ipipe_mach_timerstolen)) { + ipipe_mach_update_tsc(read_CV()); + next_match = (next_match + __ipipe_mach_ticks_per_jiffy) + & AT91_TC_REG_MASK; + write_RC(next_match); + } +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(at91_timer_initialized)) { + union tsc_reg *local_tsc, before, after; + unsigned short stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + __asm__ ("ldmia %1, %M0\n": + "=r"(after.full): "r"(local_tsc), "m"(*local_tsc)); + do { + before = after; + stamp = read_CV(); + barrier(); + __asm__ ("ldmia %1, %M0\n": + "=r"(after.full): + "r"(local_tsc), "m"(*local_tsc)); + } while (after.mid_low != before.mid_low); + if (unlikely(stamp < before.low)) + before.full += AT91_TC_REG_MASK + 1; + before.low = stamp; + return before.full; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +cycle_t at91_ipipe_get_clocksource(struct clocksource *cs) +{ + return (cycle_t)__ipipe_mach_get_tsc(); +} + +static struct clocksource clksrc = { + .name = "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC), + .rating = 250, + .read = at91_ipipe_get_clocksource, + .mask = CLOCKSOURCE_MASK(64), + .shift = 20, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +static void +at91_tc_set_mode(enum clock_event_mode mode, struct clock_event_device *dev) +{ + /* Disable the channel */ + at91_tc_write(AT91_TC_CCR, AT91_TC_CLKDIS); + + /* Disable all interrupts. */ + at91_tc_write(AT91_TC_IDR, ~0ul); + + if (mode == CLOCK_EVT_MODE_PERIODIC) { + unsigned long v; + +#ifndef CONFIG_ARCH_AT91SAM9263 + clk_enable(clk_get(NULL, "tc" + __stringify(CONFIG_IPIPE_AT91_TC) "_clk")); +#else /* AT91SAM9263 */ + clk_enable(clk_get(NULL, "tcb_clk")); +#endif /* AT91SAM9263 */ + + /* No Sync. */ + at91_tc_write(AT91_TC_BCR, 0); + + /* program NO signal on XCN */ + v = readl((void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR)); + v &= ~TCNXCNS(CONFIG_IPIPE_AT91_TC, 3); + v |= TCNXCNS(CONFIG_IPIPE_AT91_TC, 1); /* AT91_TC_TCNXCNS_NONE */ + writel(v, (void __iomem *) (AT91_VA_BASE_TCB0 + AT91_TC_BMR)); + + /* Use the clock selected by at91_timer_init as input clock. */ + at91_tc_write(AT91_TC_CMR, tc_timer_clock); + + /* Load the TC register C. */ + next_match = __ipipe_mach_ticks_per_jiffy; + write_RC(next_match); + + /* Enable CPCS interrupt. */ + at91_tc_write(AT91_TC_IER, AT91_TC_CPCS); + + /* Enable the channel. */ + at91_tc_write(AT91_TC_CCR, AT91_TC_CLKEN | AT91_TC_SWTRG); + + at91_timer_initialized = 1; + } +} + +/* + * Reprogram the timer + */ +void __ipipe_mach_set_dec(unsigned long delay) +{ + unsigned short stamp; + unsigned long flags; + + if (delay > max_delta_ticks) + delay = max_delta_ticks; + + if (likely(delay > min_delta_ticks)) { + local_irq_save_hw(flags); + stamp = read_CV(); + ipipe_mach_update_tsc(stamp); + write_RC((stamp + delay) & AT91_TC_REG_MASK); + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(KERNEL_TIMER_IRQ_NUM); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, clkevt.name); +} + +static struct clock_event_device clkevt = { + .name = "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC), + .features = CLOCK_EVT_FEAT_PERIODIC, + .shift = 32, + .rating = 250, + .set_mode = at91_tc_set_mode, +}; + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return (read_RC() - read_CV()) & AT91_TC_REG_MASK; +} + +void __init at91_timer_init(void) +{ + unsigned char tc_divisors[] = { 2, 8, 32, 128, 0, }; + unsigned master_freq, divisor = 0, divided_freq = 0; + unsigned long long wrap_ns; + + /* Disable (boot loader) timer interrupts. */ +#if defined(CONFIG_ARCH_AT91RM9200) + at91_sys_write(AT91_ST_IDR, AT91_ST_PITS | AT91_ST_WDOVF | AT91_ST_RTTINC | AT91_ST_ALMS); + (void) at91_sys_read(AT91_ST_SR); /* Clear any pending interrupts */ +#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \ + || defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL) \ + || defined(CONFIG_ARCH_AT91SAM9G20) + at91_sys_write(AT91_PIT_MR, 0); + + /* Clear any pending interrupts */ + (void) at91_sys_read(AT91_PIT_PIVR); +#endif /* CONFIG_ARCH_AT91SAM926x */ + + master_freq = clk_get_rate(clk_get(NULL, "mck")); + /* Find the first frequency above 1 MHz */ + for (tc_timer_clock = ARRAY_SIZE(tc_divisors) - 1; + tc_timer_clock >= 0; tc_timer_clock--) { + divisor = tc_divisors[tc_timer_clock]; + divided_freq = (divisor + ? master_freq / divisor : AT91_SLOW_CLOCK); + if (divided_freq > 1000000) + break; + } + + wrap_ns = (unsigned long long) (AT91_TC_REG_MASK + 1) * NSEC_PER_SEC; + do_div(wrap_ns, divided_freq); + + if (divided_freq < 1000000) + printk(KERN_INFO "AT91 I-pipe warning: could not find a" + " frequency greater than 1MHz\n"); + + printk(KERN_INFO "AT91 I-pipe timer: div: %u, freq: %u.%06u MHz, wrap: " + "%u.%06u ms\n", divisor, + divided_freq / 1000000, divided_freq % 1000000, + (unsigned) wrap_ns / 1000000, (unsigned) wrap_ns % 1000000); + + /* Add a 1ms margin. It means that when an interrupt occurs, update_tsc + must be called within 1ms. update_tsc is called by acktimer when no + higher domain handles the timer, and called through set_dec when a + higher domain handles the timer. */ + wrap_ns -= 1000000; + /* Set up the interrupt. */ + setup_irq(KERNEL_TIMER_IRQ_NUM, &at91_timer_irq); + +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; +#endif /* CONFIG_SMP */ + + clkevt.mult = div_sc(divided_freq, NSEC_PER_SEC, clkevt.shift); + clkevt.max_delta_ns = wrap_ns; + clkevt.min_delta_ns = 2000; + clkevt.cpumask = cpumask_of(0); + clockevents_register_device(&clkevt); + + clksrc.mult = clocksource_hz2mult(divided_freq, clksrc.shift); + clocksource_register(&clksrc); + + __ipipe_mach_ticks_per_jiffy = (divided_freq + HZ/2) / HZ; + max_delta_ticks = (wrap_ns * clkevt.mult) >> clkevt.shift; + min_delta_ticks = ((unsigned long long) clkevt.min_delta_ns + * clkevt.mult) >> clkevt.shift; +} + +#ifdef CONFIG_ARCH_AT91RM9200 +struct sys_timer at91rm9200_timer = { +#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9261) \ + || defined(CONFIG_ARCH_AT91SAM9263) || defined(CONFIG_ARCH_AT91SAM9RL) \ + || defined(CONFIG_ARCH_AT91SAM9G20) +struct sys_timer at91sam926x_timer = { +#elif defined(CONFIG_ARCH_AT91X40) +struct sys_timer at91x40_timer = { +#else +#error "Unknown machine" +#endif + .init = at91_timer_init, + .suspend = NULL, + .resume = NULL, +}; diff --git a/arch/arm/mach-at91/at91rm9200.c b/arch/arm/mach-at91/at91rm9200.c index 2e9ecad..2dea6d7 100644 --- a/arch/arm/mach-at91/at91rm9200.c +++ b/arch/arm/mach-at91/at91rm9200.c @@ -33,7 +33,14 @@ static struct map_desc at91rm9200_io_desc[] __initdata = { .pfn = __phys_to_pfn(AT91RM9200_BASE_EMAC), .length = SZ_16K, .type = MT_DEVICE, +#ifdef CONFIG_IPIPE }, { + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, +#endif /* CONFIG_IPIPE */ + }, { .virtual = AT91_IO_VIRT_BASE - AT91RM9200_SRAM_SIZE, .pfn = __phys_to_pfn(AT91RM9200_SRAM_BASE), .length = AT91RM9200_SRAM_SIZE, @@ -300,6 +307,7 @@ void __init at91rm9200_initialize(unsigned long main_clock, unsigned short banks * The default interrupt priority levels (0 = lowest, 7 = highest). */ static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = { +#ifndef CONFIG_IPIPE 7, /* Advanced Interrupt Controller (FIQ) */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ @@ -332,6 +340,42 @@ static unsigned int at91rm9200_default_irq_priority[NR_AIC_IRQS] __initdata = { 0, /* Advanced Interrupt Controller (IRQ4) */ 0, /* Advanced Interrupt Controller (IRQ5) */ 0 /* Advanced Interrupt Controller (IRQ6) */ +#else /* CONFIG_IPIPE */ +/* Give the highest priority to TC, since they are used as timer interrupt by + I-pipe. */ + 7, /* Advanced Interrupt Controller */ + 6, /* System Peripheral */ + 0, /* Parallel IO Controller A */ + 0, /* Parallel IO Controller B */ + 0, /* Parallel IO Controller C */ + 0, /* Parallel IO Controller D */ + 5, /* USART 0 */ + 5, /* USART 1 */ + 5, /* USART 2 */ + 5, /* USART 3 */ + 0, /* Multimedia Card Interface */ + 3, /* USB Device Port */ + 0, /* Two-Wire Interface */ + 5, /* Serial Peripheral Interface */ + 4, /* Serial Synchronous Controller */ + 4, /* Serial Synchronous Controller */ + 4, /* Serial Synchronous Controller */ + 7, /* Timer Counter 0 */ + 7, /* Timer Counter 1 */ + 7, /* Timer Counter 2 */ + 0, /* Timer Counter 3 */ + 0, /* Timer Counter 4 */ + 0, /* Timer Counter 5 */ + 2, /* USB Host port */ + 2, /* Ethernet MAC */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0 /* Advanced Interrupt Controller */ +#endif /*CONFIG_IPIPE */ }; void __init at91rm9200_init_interrupts(unsigned int priority[NR_AIC_IRQS]) diff --git a/arch/arm/mach-at91/at91sam9260.c b/arch/arm/mach-at91/at91sam9260.c index 0894f10..63f05ad 100644 --- a/arch/arm/mach-at91/at91sam9260.c +++ b/arch/arm/mach-at91/at91sam9260.c @@ -31,6 +31,13 @@ static struct map_desc at91sam9260_io_desc[] __initdata = { .pfn = __phys_to_pfn(AT91_BASE_SYS), .length = SZ_16K, .type = MT_DEVICE, +#ifdef CONFIG_IPIPE + }, { + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, +#endif /* CONFIG_IPIPE */ } }; @@ -350,6 +357,7 @@ void __init at91sam9260_initialize(unsigned long main_clock) * The default interrupt priority levels (0 = lowest, 7 = highest). */ static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = { +#ifndef CONFIG_IPIPE 7, /* Advanced Interrupt Controller */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ @@ -382,6 +390,42 @@ static unsigned int at91sam9260_default_irq_priority[NR_AIC_IRQS] __initdata = { 0, /* Advanced Interrupt Controller */ 0, /* Advanced Interrupt Controller */ 0, /* Advanced Interrupt Controller */ +#else /* CONFIG_IPIPE */ +/* Give the highest priority to TC, since they are used as timer interrupt by + I-pipe. */ + 7, /* Advanced Interrupt Controller */ + 7, /* System Peripherals */ + 0, /* Parallel IO Controller A */ + 0, /* Parallel IO Controller B */ + 0, /* Parallel IO Controller C */ + 0, /* Analog-to-Digital Converter */ + 6, /* USART 0 */ + 6, /* USART 1 */ + 6, /* USART 2 */ + 0, /* Multimedia Card Interface */ + 4, /* USB Device Port */ + 0, /* Two-Wire Interface */ + 6, /* Serial Peripheral Interface 0 */ + 6, /* Serial Peripheral Interface 1 */ + 5, /* Serial Synchronous Controller */ + 0, + 0, + 7, /* Timer Counter 0 */ + 7, /* Timer Counter 1 */ + 7, /* Timer Counter 2 */ + 3, /* USB Host port */ + 3, /* Ethernet */ + 0, /* Image Sensor Interface */ + 6, /* USART 3 */ + 6, /* USART 4 */ + 6, /* USART 5 */ + 7, /* Timer Counter 3 */ + 7, /* Timer Counter 4 */ + 7, /* Timer Counter 5 */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ +#endif /*CONFIG_IPIPE */ }; void __init at91sam9260_init_interrupts(unsigned int priority[NR_AIC_IRQS]) diff --git a/arch/arm/mach-at91/at91sam9261.c b/arch/arm/mach-at91/at91sam9261.c index 3acd7d7..de0ad62 100644 --- a/arch/arm/mach-at91/at91sam9261.c +++ b/arch/arm/mach-at91/at91sam9261.c @@ -30,6 +30,13 @@ static struct map_desc at91sam9261_io_desc[] __initdata = { .pfn = __phys_to_pfn(AT91_BASE_SYS), .length = SZ_16K, .type = MT_DEVICE, +#ifdef CONFIG_IPIPE + }, { + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, +#endif /* CONFIG_IPIPE */ }, { .virtual = AT91_IO_VIRT_BASE - AT91SAM9261_SRAM_SIZE, .pfn = __phys_to_pfn(AT91SAM9261_SRAM_BASE), @@ -286,6 +293,7 @@ void __init at91sam9261_initialize(unsigned long main_clock) * The default interrupt priority levels (0 = lowest, 7 = highest). */ static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = { +#ifndef CONFIG_IPIPE 7, /* Advanced Interrupt Controller */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ @@ -318,6 +326,42 @@ static unsigned int at91sam9261_default_irq_priority[NR_AIC_IRQS] __initdata = { 0, /* Advanced Interrupt Controller */ 0, /* Advanced Interrupt Controller */ 0, /* Advanced Interrupt Controller */ +#else /* CONFIG_IPIPE */ +/* Give the highest priority to TC, since they are used as timer interrupt by + I-pipe. */ + 7, /* Advanced Interrupt Controller */ + 7, /* System Peripherals */ + 0, /* Parallel IO Controller A */ + 0, /* Parallel IO Controller B */ + 0, /* Parallel IO Controller C */ + 0, + 6, /* USART 0 */ + 6, /* USART 1 */ + 6, /* USART 2 */ + 0, /* Multimedia Card Interface */ + 4, /* USB Device Port */ + 0, /* Two-Wire Interface */ + 6, /* Serial Peripheral Interface 0 */ + 6, /* Serial Peripheral Interface 1 */ + 5, /* Serial Synchronous Controller 0 */ + 5, /* Serial Synchronous Controller 1 */ + 5, /* Serial Synchronous Controller 2 */ + 7, /* Timer Counter 0 */ + 7, /* Timer Counter 1 */ + 7, /* Timer Counter 2 */ + 3, /* USB Host port */ + 3, /* LCD Controller */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ + 0, /* Advanced Interrupt Controller */ +#endif /*CONFIG_IPIPE */ }; void __init at91sam9261_init_interrupts(unsigned int priority[NR_AIC_IRQS]) diff --git a/arch/arm/mach-at91/at91sam9263.c b/arch/arm/mach-at91/at91sam9263.c index 942792d..c043f3a 100644 --- a/arch/arm/mach-at91/at91sam9263.c +++ b/arch/arm/mach-at91/at91sam9263.c @@ -30,6 +30,13 @@ static struct map_desc at91sam9263_io_desc[] __initdata = { .pfn = __phys_to_pfn(AT91_BASE_SYS), .length = SZ_16K, .type = MT_DEVICE, +#ifdef CONFIG_IPIPE + }, { + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, +#endif /* CONFIG_IPIPE */ }, { .virtual = AT91_IO_VIRT_BASE - AT91SAM9263_SRAM0_SIZE, .pfn = __phys_to_pfn(AT91SAM9263_SRAM0_BASE), @@ -311,6 +318,7 @@ void __init at91sam9263_initialize(unsigned long main_clock) * The default interrupt priority levels (0 = lowest, 7 = highest). */ static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = { +#ifndef CONFIG_IPIPE 7, /* Advanced Interrupt Controller (FIQ) */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ @@ -343,6 +351,42 @@ static unsigned int at91sam9263_default_irq_priority[NR_AIC_IRQS] __initdata = { 2, /* USB Host port */ 0, /* Advanced Interrupt Controller (IRQ0) */ 0, /* Advanced Interrupt Controller (IRQ1) */ +#else /* CONFIG_IPIPE */ +/* Give the highest priority to TC, since they are used as timer interrupt by + I-pipe. */ + 7, /* Advanced Interrupt Controller (FIQ) */ + 6, /* System Peripherals */ + 0, /* Parallel IO Controller A */ + 0, /* Parallel IO Controller B */ + 0, /* Parallel IO Controller C, D and E */ + 0, + 0, + 5, /* USART 0 */ + 5, /* USART 1 */ + 5, /* USART 2 */ + 0, /* Multimedia Card Interface 0 */ + 0, /* Multimedia Card Interface 1 */ + 3, /* CAN */ + 0, /* Two-Wire Interface */ + 5, /* Serial Peripheral Interface 0 */ + 5, /* Serial Peripheral Interface 1 */ + 4, /* Serial Synchronous Controller 0 */ + 4, /* Serial Synchronous Controller 1 */ + 5, /* AC97 Controller */ + 7, /* Timer Counter 0, 1 and 2 */ + 0, /* Pulse Width Modulation Controller */ + 2, /* Ethernet */ + 0, + 0, /* 2D Graphic Engine */ + 2, /* USB Device Port */ + 0, /* Image Sensor Interface */ + 2, /* LDC Controller */ + 0, /* DMA Controller */ + 0, + 2, /* USB Host port */ + 0, /* Advanced Interrupt Controller (IRQ0) */ + 0, /* Advanced Interrupt Controller (IRQ1) */ +#endif /*CONFIG_IPIPE */ }; void __init at91sam9263_init_interrupts(unsigned int priority[NR_AIC_IRQS]) diff --git a/arch/arm/mach-at91/at91sam9rl.c b/arch/arm/mach-at91/at91sam9rl.c index 211c5c1..9cdad31 100644 --- a/arch/arm/mach-at91/at91sam9rl.c +++ b/arch/arm/mach-at91/at91sam9rl.c @@ -30,6 +30,13 @@ static struct map_desc at91sam9rl_io_desc[] __initdata = { .pfn = __phys_to_pfn(AT91_BASE_SYS), .length = SZ_16K, .type = MT_DEVICE, +#ifdef CONFIG_IPIPE + }, { + .virtual = AT91_VA_BASE_TCB0, + .pfn = __phys_to_pfn(AT91_BASE_TCB0), + .length = SZ_16K, + .type = MT_DEVICE, +#endif /* CONFIG_IPIPE */ }, }; @@ -303,6 +310,7 @@ void __init at91sam9rl_initialize(unsigned long main_clock) * The default interrupt priority levels (0 = lowest, 7 = highest). */ static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = { +#ifndef CONFIG_IPIPE 7, /* Advanced Interrupt Controller */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ @@ -335,6 +343,42 @@ static unsigned int at91sam9rl_default_irq_priority[NR_AIC_IRQS] __initdata = { 0, 0, 0, /* Advanced Interrupt Controller */ +#else /* CONFIG_IPIPE */ +/* Give the highest priority to TC, since they are used as timer interrupt by + I-pipe. */ + 7, /* Advanced Interrupt Controller */ + 6, /* System Peripherals */ + 1, /* Parallel IO Controller A */ + 1, /* Parallel IO Controller B */ + 1, /* Parallel IO Controller C */ + 1, /* Parallel IO Controller D */ + 4, /* USART 0 */ + 4, /* USART 1 */ + 4, /* USART 2 */ + 4, /* USART 3 */ + 0, /* Multimedia Card Interface */ + 5, /* Two-Wire Interface 0 */ + 5, /* Two-Wire Interface 1 */ + 4, /* Serial Peripheral Interface */ + 3, /* Serial Synchronous Controller 0 */ + 3, /* Serial Synchronous Controller 1 */ + 7, /* Timer Counter 0 */ + 7, /* Timer Counter 1 */ + 7, /* Timer Counter 2 */ + 0, + 0, /* Touch Screen Controller */ + 0, /* DMA Controller */ + 2, /* USB Device High speed port */ + 2, /* LCD Controller */ + 5, /* AC97 Controller */ + 0, + 0, + 0, + 0, + 0, + 0, + 0, /* Advanced Interrupt Controller */ +#endif /*CONFIG_IPIPE */ }; void __init at91sam9rl_init_interrupts(unsigned int priority[NR_AIC_IRQS]) diff --git a/arch/arm/mach-at91/gpio.c b/arch/arm/mach-at91/gpio.c index f2236f0..36f7dd0 100644 --- a/arch/arm/mach-at91/gpio.c +++ b/arch/arm/mach-at91/gpio.c @@ -25,6 +25,13 @@ #include #include +#ifdef CONFIG_IPIPE +#include + +#ifdef __IPIPE_FEATURE_PIC_MUTE +DEFINE_PER_CPU(__ipipe_irqbits_t, __ipipe_muted_irqs); +#endif /* __IPIPE_FEATURE_PIC_MUTE */ +#endif /* CONFIG_IPIPE */ #include "generic.h" @@ -377,6 +384,10 @@ static int gpio_irq_type(unsigned pin, unsigned type) static struct irq_chip gpio_irqchip = { .name = "GPIO", +#ifdef CONFIG_IPIPE + .ack = gpio_irq_mask, + .mask_ack = gpio_irq_mask, +#endif /* CONFIG_IPIPE */ .mask = gpio_irq_mask, .unmask = gpio_irq_unmask, .set_type = gpio_irq_type, @@ -424,7 +435,7 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc) gpio_irq_mask(pin); } else - generic_handle_irq(pin); + ipipe_handle_irq_cond(pin); } pin++; gpio++; @@ -435,6 +446,38 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc) /* now it may re-trigger */ } +#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE) +void ipipe_mute_pic(void) +{ + unsigned long unmasked, muted; + unsigned i; + + for (i = 0; i < gpio_banks; i++) { + unmasked = __raw_readl(gpio_chip[i].regbase + PIO_IMR); + muted = unmasked & __ipipe_irqbits[i + 1]; + __raw_get_cpu_var(__ipipe_muted_irqs)[i + 1] = muted; + __raw_writel(muted, gpio_chip[i].regbase + PIO_IDR); + } + + unmasked = at91_sys_read(AT91_AIC_IMR); + muted = unmasked & __ipipe_irqbits[0]; + __raw_get_cpu_var(__ipipe_muted_irqs)[0] = muted; + at91_sys_write(AT91_AIC_IDCR, muted); +} + +void ipipe_unmute_pic(void) +{ + unsigned i; + + at91_sys_write(AT91_AIC_IECR, __raw_get_cpu_var(__ipipe_muted_irqs)[0]); + for (i = 0; i < gpio_banks; i++) { + unsigned long muted; + muted = __raw_get_cpu_var(__ipipe_muted_irqs)[i + 1]; + __raw_writel(muted, gpio_chip[i].regbase + PIO_IER); + } +} +#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */ + /*--------------------------------------------------------------------------*/ #ifdef CONFIG_DEBUG_FS @@ -511,6 +554,11 @@ void __init at91_gpio_irq_setup(void) unsigned pioc, pin; struct at91_gpio_chip *this, *prev; +#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE) + printk("Setting irqbits to all ones.\n"); + memset(__ipipe_irqbits, ~0, sizeof(__ipipe_irqbits)); +#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */ + for (pioc = 0, pin = PIN_BASE, this = gpio_chip, prev = NULL; pioc++ < gpio_banks; prev = this, this++) { @@ -540,6 +588,9 @@ void __init at91_gpio_irq_setup(void) set_irq_chip_data(id, this); set_irq_chained_handler(id, gpio_irq_handler); +#if defined(CONFIG_IPIPE) && defined(__IPIPE_FEATURE_PIC_MUTE) + __ipipe_irqbits[0] &= ~(1 << id); +#endif /* CONFIG_IPIPE && __IPIPE_FEATURE_PIC_MUTE */ } pr_info("AT91: %d gpio irqs in %d banks\n", pin - PIN_BASE, gpio_banks); } diff --git a/arch/arm/mach-at91/include/mach/hardware.h b/arch/arm/mach-at91/include/mach/hardware.h index da0b681..f737d38 100644 --- a/arch/arm/mach-at91/include/mach/hardware.h +++ b/arch/arm/mach-at91/include/mach/hardware.h @@ -61,6 +61,25 @@ #define AT91_VA_BASE_SYS AT91_IO_P2V(AT91_BASE_SYS) #define AT91_VA_BASE_EMAC AT91_IO_P2V(AT91RM9200_BASE_EMAC) +#ifdef CONFIG_IPIPE +#if defined(CONFIG_ARCH_AT91RM9200) +#define AT91_BASE_TCB0 AT91RM9200_BASE_TCB0 +#elif defined(CONFIG_ARCH_AT91SAM9260) || defined(CONFIG_ARCH_AT91SAM9G20) +#define AT91_BASE_TCB0 AT91SAM9260_BASE_TCB0 +#elif defined(CONFIG_ARCH_AT91SAM9261) +#define AT91_BASE_TCB0 AT91SAM9261_BASE_TCB0 +#elif defined(CONFIG_ARCH_AT91SAM9263) +#define AT91_BASE_TCB0 AT91SAM9263_BASE_TCB0 +#elif defined(CONFIG_ARCH_AT91SAM9RL) +#define AT91_BASE_TCB0 AT91SAM9RL_BASE_TCB0 +#elif defined(CONFIG_ARCH_AT91X40) +#define AT91_BASE_TCB0 (AT91_BASE_SYS + AT91_TC) +#else +#error "AT91 processor unsupported by Adeos" +#endif +#define AT91_VA_BASE_TCB0 AT91_IO_P2V(AT91_BASE_TCB0) +#endif + /* Internal SRAM is mapped below the IO devices */ #define AT91_SRAM_MAX SZ_1M #define AT91_VIRT_BASE (AT91_IO_VIRT_BASE - AT91_SRAM_MAX) diff --git a/arch/arm/mach-at91/include/mach/irqs.h b/arch/arm/mach-at91/include/mach/irqs.h index 36bd55f..cedaab2 100644 --- a/arch/arm/mach-at91/include/mach/irqs.h +++ b/arch/arm/mach-at91/include/mach/irqs.h @@ -45,4 +45,6 @@ /* FIQ is AIC source 0. */ #define FIQ_START AT91_ID_FIQ +#define __IPIPE_FEATURE_PIC_MUTE + #endif diff --git a/arch/arm/mach-at91/include/mach/timex.h b/arch/arm/mach-at91/include/mach/timex.h index d84c994..45a4894 100644 --- a/arch/arm/mach-at91/include/mach/timex.h +++ b/arch/arm/mach-at91/include/mach/timex.h @@ -72,6 +72,6 @@ #define AT91X40_MASTER_CLOCK 40000000 #define CLOCK_TICK_RATE (AT91X40_MASTER_CLOCK) -#endif +#endif /* arch specific */ #endif diff --git a/arch/arm/mach-at91/irq.c b/arch/arm/mach-at91/irq.c index da3494a..d375427 100644 --- a/arch/arm/mach-at91/irq.c +++ b/arch/arm/mach-at91/irq.c @@ -121,6 +121,9 @@ static struct irq_chip at91_aic_chip = { .name = "AIC", .ack = at91_aic_mask_irq, .mask = at91_aic_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = at91_aic_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = at91_aic_unmask_irq, .set_type = at91_aic_set_type, .set_wake = at91_aic_set_wake, diff --git a/arch/arm/mach-integrator/core.c b/arch/arm/mach-integrator/core.c index a0f60e5..84c99a7 100644 --- a/arch/arm/mach-integrator/core.c +++ b/arch/arm/mach-integrator/core.c @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/core.c * * Copyright (C) 2000-2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2, as @@ -230,53 +231,62 @@ EXPORT_SYMBOL(cm_control); /* * How long is the timer interval? */ -#define TIMER_INTERVAL (TICKS_PER_uSEC * mSEC_10) -#if TIMER_INTERVAL >= 0x100000 -#define TICKS2USECS(x) (256 * (x) / TICKS_PER_uSEC) -#elif TIMER_INTERVAL >= 0x10000 -#define TICKS2USECS(x) (16 * (x) / TICKS_PER_uSEC) -#else #define TICKS2USECS(x) ((x) / TICKS_PER_uSEC) -#endif static unsigned long timer_reload; +static unsigned long timer_interval; +static unsigned long timer_lxlost; +static int tscok; + +#ifdef CONFIG_IPIPE +int __ipipe_mach_timerint = IRQ_TIMERINT1; +static unsigned long long __ipipe_mach_tsc; +static IPIPE_DEFINE_SPINLOCK(timer_lock); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} +#endif /* - * Returns number of ms since last clock interrupt. Note that interrupts - * will have been disabled by do_gettimeoffset() + * Called with IRQ disabled from do_gettimeofday(). */ -unsigned long integrator_gettimeoffset(void) +static inline unsigned long integrator_getticksoffset(void) { - unsigned long ticks1, ticks2, status; + unsigned long ticks; - /* - * Get the current number of ticks. Note that there is a race - * condition between us reading the timer and checking for - * an interrupt. We get around this by ensuring that the - * counter has not reloaded between our two reads. - */ - ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - do { - ticks1 = ticks2; - status = __raw_readl(VA_IC_BASE + IRQ_RAW_STATUS); - ticks2 = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - } while (ticks2 > ticks1); + if (!tscok) + return 0; - /* - * Number of ticks since last interrupt. - */ - ticks1 = timer_reload - ticks2; + ticks = readl(TIMER1_VA_BASE + TIMER_VALUE) & 0xffff; - /* - * Interrupt pending? If so, we've reloaded once already. - */ - if (status & (1 << IRQ_TIMERINT1)) - ticks1 += timer_reload; + if (ticks > timer_reload) + ticks = 0xffff + timer_reload - ticks; + else + ticks = timer_reload - ticks; + if (timer_interval < 0x10000) + return ticks; + else if (timer_interval < 0x100000) + return ticks * 16; + else + return ticks * 256; +} + +/* + * Returns number of ms since last clock interrupt. Note that interrupts + * will have been disabled by do_gettimeoffset() + */ +unsigned long integrator_gettimeoffset(void) +{ /* * Convert the ticks to usecs */ - return TICKS2USECS(ticks1); + return TICKS2USECS(timer_lxlost + integrator_getticksoffset()); } /* @@ -285,11 +295,22 @@ unsigned long integrator_gettimeoffset(void) static irqreturn_t integrator_timer_interrupt(int irq, void *dev_id) { + timer_lxlost = 0; + +#ifdef CONFIG_IPIPE /* - * clear the interrupt + * If Linux is the only domain, ack the timer and reprogram it */ - writel(1, TIMER1_VA_BASE + TIMER_INTCLR); + if (!__ipipe_mach_timerstolen) { + __ipipe_mach_tsc += integrator_getticksoffset(); +#else + writel(1, TIMER1_VA_BASE + TIMER_INTCLR); +#endif + writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); +#ifdef CONFIG_IPIPE + } +#endif timer_tick(); return IRQ_HANDLED; @@ -301,24 +322,30 @@ static struct irqaction integrator_timer_irq = { .handler = integrator_timer_interrupt, }; -/* - * Set up timer interrupt, and return the current time in seconds. - */ -void __init integrator_time_init(unsigned long reload, unsigned int ctrl) +static inline void set_dec(unsigned long reload) { - unsigned int timer_ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_PERIODIC; + unsigned int ctrl = TIMER_CTRL_ENABLE | TIMER_CTRL_IE; timer_reload = reload; - timer_ctrl |= ctrl; + timer_interval = reload; - if (timer_reload > 0x100000) { + if (timer_reload >= 0x100000) { timer_reload >>= 8; - timer_ctrl |= TIMER_CTRL_DIV256; - } else if (timer_reload > 0x010000) { + ctrl |= TIMER_CTRL_DIV256; + } else if (timer_reload >= 0x010000) { timer_reload >>= 4; - timer_ctrl |= TIMER_CTRL_DIV16; + ctrl |= TIMER_CTRL_DIV16; } + writel(ctrl, TIMER1_VA_BASE + TIMER_CTRL); + writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); +} + +/* + * Set up timer interrupt, and return the current time in seconds. + */ +void __init integrator_time_init(unsigned long reload, unsigned int ctrl) +{ /* * Initialise to a known state (all timers off) */ @@ -326,12 +353,60 @@ void __init integrator_time_init(unsigned long reload, unsigned int ctrl) writel(0, TIMER1_VA_BASE + TIMER_CTRL); writel(0, TIMER2_VA_BASE + TIMER_CTRL); - writel(timer_reload, TIMER1_VA_BASE + TIMER_LOAD); - writel(timer_reload, TIMER1_VA_BASE + TIMER_VALUE); - writel(timer_ctrl, TIMER1_VA_BASE + TIMER_CTRL); + set_dec(reload); /* * Make irqs happen for the system timer */ setup_irq(IRQ_TIMERINT1, &integrator_timer_irq); + + tscok = 1; +} + +#ifdef CONFIG_IPIPE + +void __ipipe_mach_acktimer(void) +{ + writel(1, TIMER1_VA_BASE + TIMER_INTCLR); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + unsigned long long result; + unsigned long flags; + + local_irq_save_hw_notrace(flags); + spin_lock(&timer_lock); + result = __ipipe_mach_tsc + integrator_getticksoffset(); + spin_unlock(&timer_lock); + local_irq_restore_hw_notrace(flags); + return result; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +void __ipipe_mach_set_dec(unsigned long reload) +{ + unsigned long ticks; + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + ticks = integrator_getticksoffset(); + __ipipe_mach_tsc += ticks; + timer_lxlost += ticks; + + set_dec(reload); + spin_unlock_irqrestore(&timer_lock, flags); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return readl(TIMER1_VA_BASE + TIMER_VALUE); } +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/mach-integrator/include/mach/entry-macro.S b/arch/arm/mach-integrator/include/mach/entry-macro.S index 7649c57..ab3623d 100644 --- a/arch/arm/mach-integrator/include/mach/entry-macro.S +++ b/arch/arm/mach-integrator/include/mach/entry-macro.S @@ -28,7 +28,11 @@ teq \irqstat, #0 ldreq \irqstat, [\base, #(INTEGRATOR_HDR_IC_OFFSET+IRQ_STATUS)] moveq \irqnr, #IRQ_CIC_START - +#ifdef CONFIG_IPIPE + tst \irqstat, #0x00000040 @ check IRQ_TIMERINT1 first + movne \irqnr, #6 + bne 1003f +#endif /* CONFIG_IPIPE */ 1001: tst \irqstat, #15 bne 1002f add \irqnr, \irqnr, #4 diff --git a/arch/arm/mach-integrator/include/mach/irqs.h b/arch/arm/mach-integrator/include/mach/irqs.h index 1fbe6d1..dea6390 100644 --- a/arch/arm/mach-integrator/include/mach/irqs.h +++ b/arch/arm/mach-integrator/include/mach/irqs.h @@ -79,4 +79,3 @@ #define IRQ_SIC_END 46 #define NR_IRQS 47 - diff --git a/arch/arm/mach-integrator/include/mach/platform.h b/arch/arm/mach-integrator/include/mach/platform.h index e00a262..3b39ce3 100644 --- a/arch/arm/mach-integrator/include/mach/platform.h +++ b/arch/arm/mach-integrator/include/mach/platform.h @@ -417,7 +417,7 @@ * Timer definitions * * Only use timer 1 & 2 - * (both run at 24MHz and will need the clock divider set to 16). + * (both run at 1MHZ on /CP and at 24MHz on /AP) * * Timer 0 runs at bus frequency and therefore could vary and currently * uHAL can't handle that. @@ -430,7 +430,12 @@ #define MAX_TIMER 2 #define MAX_PERIOD 699050 + +#ifdef CONFIG_ARCH_INTEGRATOR_CP +#define TICKS_PER_uSEC 1 +#else #define TICKS_PER_uSEC 24 +#endif /* * These are useconds NOT ticks. diff --git a/arch/arm/mach-integrator/include/mach/timex.h b/arch/arm/mach-integrator/include/mach/timex.h index 1dcb420..a04653a 100644 --- a/arch/arm/mach-integrator/include/mach/timex.h +++ b/arch/arm/mach-integrator/include/mach/timex.h @@ -21,6 +21,6 @@ */ /* - * ?? + * Timer rate */ -#define CLOCK_TICK_RATE (50000000 / 16) +#define CLOCK_TICK_RATE (1000000) diff --git a/arch/arm/mach-integrator/integrator_cp.c b/arch/arm/mach-integrator/integrator_cp.c index 4ac0405..a78d046 100644 --- a/arch/arm/mach-integrator/integrator_cp.c +++ b/arch/arm/mach-integrator/integrator_cp.c @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/integrator_cp.c * * Copyright (C) 2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,6 +21,7 @@ #include #include #include +#include #include #include @@ -161,6 +163,9 @@ static struct irq_chip cic_chip = { .name = "CIC", .ack = cic_mask_irq, .mask = cic_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = cic_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = cic_unmask_irq, }; @@ -180,6 +185,9 @@ static struct irq_chip pic_chip = { .name = "PIC", .ack = pic_mask_irq, .mask = pic_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = pic_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = pic_unmask_irq, }; @@ -199,6 +207,9 @@ static struct irq_chip sic_chip = { .name = "SIC", .ack = sic_mask_irq, .mask = sic_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = sic_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = sic_unmask_irq, }; @@ -218,7 +229,7 @@ sic_handle_irq(unsigned int irq, struct irq_desc *desc) irq += IRQ_SIC_START; - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); } while (status); } @@ -569,9 +580,14 @@ static void __init intcp_init(void) #define TIMER_CTRL_IE (1 << 5) /* Interrupt Enable */ +#ifdef CONFIG_IPIPE +unsigned int __ipipe_mach_ticks_per_jiffy = 1000000 * TICKS_PER_uSEC / HZ; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); +#endif + static void __init intcp_timer_init(void) { - integrator_time_init(1000000 / HZ, TIMER_CTRL_IE); + integrator_time_init(1000000 * TICKS_PER_uSEC / HZ, TIMER_CTRL_IE); } static struct sys_timer cp_timer = { diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 1e93dfe..8374fa2 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -45,6 +45,65 @@ static int __init ixp4xx_clocksource_init(void); static int __init ixp4xx_clockevent_init(void); static struct clock_event_device clockevent_ixp4xx; +#ifdef CONFIG_IPIPE +#include + +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif + +int __ipipe_mach_timerint = IRQ_IXP4XX_TIMER1; +int __ipipe_mach_timerstolen = 0; +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_timerint); +EXPORT_SYMBOL(__ipipe_mach_timerstolen); +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int ixp4xx_timer_initialized; + +#define ONE_SHOT_ENABLE (IXP4XX_OST_ENABLE|IXP4XX_OST_ONE_SHOT) + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} + +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + info->u.fr.counter = + (unsigned *) + (IXP4XX_TIMER_BASE_PHYS + IXP4XX_OSTS_OFFSET); + info->u.fr.mask = 0xffffffff; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static void ipipe_mach_update_tsc(void); + +#endif /* CONFIG_IPIPE */ + /************************************************************************* * IXP4xx chipset I/O mapping *************************************************************************/ @@ -269,8 +328,12 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = &clockevent_ixp4xx; +#ifndef CONFIG_IPIPE /* Clear Pending Interrupt by writing '1' to it */ *IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND; +#else /* CONFIG_IPIPE */ + ipipe_mach_update_tsc(); +#endif /* CONFIG_IPIPE */ evt->event_handler(evt); @@ -294,6 +357,14 @@ void __init ixp4xx_timer_init(void) /* Reset time-stamp counter */ *IXP4XX_OSTS = 0; +#ifdef CONFIG_IPIPE +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; + barrier(); +#endif /* CONFIG_SMP */ + + ixp4xx_timer_initialized = 1; +#endif /* Connect the interrupt handler and enable the interrupt */ setup_irq(IRQ_IXP4XX_TIMER1, &ixp4xx_timer_irq); @@ -470,6 +541,13 @@ static void ixp4xx_set_mode(enum clock_event_mode mode, *IXP4XX_OSRT1 = osrt | opts; } +#ifdef CONFIG_IPIPE +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, clockevent_ixp4xx.name); +} +#endif /* CONFIG_IPIPE */ + static struct clock_event_device clockevent_ixp4xx = { .name = "ixp4xx timer1", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, @@ -492,3 +570,98 @@ static int __init ixp4xx_clockevent_init(void) clockevents_register_device(&clockevent_ixp4xx); return 0; } + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + /* Clear Pending Interrupt by writing '1' to it */ + *IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND; +} + +static void ipipe_mach_update_tsc(void) +{ + union tsc_reg *local_tsc; + unsigned long stamp, flags; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = *IXP4XX_OSTS; + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + local_irq_restore_hw(flags); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(ixp4xx_timer_initialized)) { + union tsc_reg *local_tsc, result; + unsigned long stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + __asm__ ("ldmia %1, %M0\n": + "=r"(result.full): "r"(local_tsc), "m"(*local_tsc)); + barrier(); + stamp = *IXP4XX_OSTS; + if (unlikely(stamp < result.low)) + /* 32 bit counter wrapped, increment high word. */ + result.high++; + result.low = stamp; + return result.full; + } + + return 0; +} + +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + * + * The timer is aperiodic (most of the time) when running Xenomai, so + * __ipipe_mach_set_dec is called for each timer tick and programs the + * timer hardware for the next tick. + * + */ +#define MIN_DELAY 333 /* 5 usec with the 66.66 MHz system clock */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + unsigned long flags; + if (delay > MIN_DELAY) { + local_irq_save_hw(flags); + *IXP4XX_OSRT1 = delay | ONE_SHOT_ENABLE; + local_irq_restore_hw(flags); + } else { + ipipe_trigger_irq(IRQ_IXP4XX_TIMER1); + } +} + +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +/* + * This returns the number of clock ticks remaining. + */ +unsigned long __ipipe_mach_get_dec(void) +{ + return(*IXP4XX_OST1); /* remaining */ +} + +EXPORT_SYMBOL(__ipipe_mach_get_dec); + +void __ipipe_mach_release_timer(void) +{ + unsigned long flags; + + local_irq_save_hw(flags); + ixp4xx_set_mode(clockevent_ixp4xx.mode, &clockevent_ixp4xx); + if (clockevent_ixp4xx.mode == CLOCK_EVT_MODE_ONESHOT) + ixp4xx_set_next_event(LATCH, &clockevent_ixp4xx); + local_irq_restore_hw(flags); +} + +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/mach-mx3/mx31ads.c b/arch/arm/mach-mx3/mx31ads.c index 30e2767..2945b2e 100644 --- a/arch/arm/mach-mx3/mx31ads.c +++ b/arch/arm/mach-mx3/mx31ads.c @@ -134,7 +134,7 @@ static void mx31ads_expio_irq_handler(u32 irq, struct irq_desc *desc) if ((int_valid & 1) == 0) continue; - generic_handle_irq(expio_irq); + ipipe_handle_irq_cond(expio_irq); } } diff --git a/arch/arm/mach-mx3/mx31pdk.c b/arch/arm/mach-mx3/mx31pdk.c index c19838d..6fc284b 100644 --- a/arch/arm/mach-mx3/mx31pdk.c +++ b/arch/arm/mach-mx3/mx31pdk.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -108,7 +109,7 @@ static void mx31pdk_expio_irq_handler(uint32_t irq, struct irq_desc *desc) for (; int_valid != 0; int_valid >>= 1, expio_irq++) { if ((int_valid & 1) == 0) continue; - generic_handle_irq(expio_irq); + ipipe_handle_irq_cond(expio_irq); } } diff --git a/arch/arm/mach-omap2/timer-gp.c b/arch/arm/mach-omap2/timer-gp.c index 97eeeeb..6afc49f 100644 --- a/arch/arm/mach-omap2/timer-gp.c +++ b/arch/arm/mach-omap2/timer-gp.c @@ -47,13 +47,74 @@ static struct omap_dm_timer *gptimer; static struct clock_event_device clockevent_gpt; static u8 __initdata gptimer_id = 1; static u8 __initdata inited; +#ifndef CONFIG_OMAP_32K_TIMER +static struct omap_dm_timer *gpt_clocksource; +#endif /* !CONFIG_OMAP_32K_TIMER */ + +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int omap2_timer_initialized; + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + info->u.fr.counter = (unsigned *) + omap_dm_timer_get_phys_counter_addr(gpt_clocksource); + info->u.fr.mask = 0xffffffff; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static void ipipe_mach_update_tsc(void); + +#endif /* CONFIG_IPIPE */ static irqreturn_t omap2_gp_timer_interrupt(int irq, void *dev_id) { - struct omap_dm_timer *gpt = (struct omap_dm_timer *)dev_id; struct clock_event_device *evt = &clockevent_gpt; +#ifndef CONFIG_IPIPE + struct omap_dm_timer *gpt = (struct omap_dm_timer *)dev_id; omap_dm_timer_write_status(gpt, OMAP_TIMER_INT_OVERFLOW); +#else /* CONFIG_IPIPE */ + ipipe_mach_update_tsc(); +#endif /* CONFIG_IPIPE */ evt->event_handler(evt); return IRQ_HANDLED; @@ -125,6 +186,13 @@ int __init omap2_gp_clockevent_set_gptimer(u8 id) return 0; } +#ifdef CONFIG_IPIPE +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, clockevent_gpt.name); +} +#endif /* CONFIG_IPIPE */ + static void __init omap2_gp_clockevent_init(void) { u32 tick_rate; @@ -156,6 +224,12 @@ static void __init omap2_gp_clockevent_init(void) gptimer_id, tick_rate); omap2_gp_timer_irq.dev_id = (void *)gptimer; + +#ifdef CONFIG_IPIPE + __ipipe_mach_timerint = omap_dm_timer_get_irq(gptimer); + __ipipe_mach_ticks_per_jiffy = (tick_rate + HZ / 2) / HZ; +#endif /* CONFIG_IPIPE */ + setup_irq(omap_dm_timer_get_irq(gptimer), &omap2_gp_timer_irq); omap_dm_timer_set_int_enable(gptimer, OMAP_TIMER_INT_OVERFLOW); @@ -166,7 +240,6 @@ static void __init omap2_gp_clockevent_init(void) clockevent_gpt.min_delta_ns = clockevent_delta2ns(3, &clockevent_gpt); /* Timer internal resynch latency. */ - clockevent_gpt.cpumask = cpumask_of(0); clockevents_register_device(&clockevent_gpt); } @@ -185,7 +258,6 @@ static inline void __init omap2_gp_clocksource_init(void) {} /* * clocksource */ -static struct omap_dm_timer *gpt_clocksource; static cycle_t clocksource_read_cycles(struct clocksource *cs) { return (cycle_t)omap_dm_timer_read_counter(gpt_clocksource); @@ -223,6 +295,16 @@ static void __init omap2_gp_clocksource_init(void) clocksource_gpt.mult = clocksource_khz2mult(tick_rate/1000, clocksource_gpt.shift); + +#ifdef CONFIG_IPIPE +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; + barrier(); +#endif /* CONFIG_SMP */ + + omap2_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ + if (clocksource_register(&clocksource_gpt)) printk(err2, clocksource_gpt.name); } @@ -242,3 +324,79 @@ static void __init omap2_gp_timer_init(void) struct sys_timer omap_timer = { .init = omap2_gp_timer_init, }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + omap_dm_timer_write_status(gptimer, OMAP_TIMER_INT_OVERFLOW); + omap_dm_timer_read_status(gptimer); +} + +static void ipipe_mach_update_tsc(void) +{ + union tsc_reg *local_tsc; + unsigned long stamp, flags; + + if (likely(omap2_timer_initialized)) { + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = omap_dm_timer_read_counter(gpt_clocksource); + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + local_irq_restore_hw(flags); + } +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(omap2_timer_initialized)) { + union tsc_reg *local_tsc, result; + unsigned long stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + __asm__ ("ldmia %1, %M0\n" : + "=r"(result.full) : "r"(local_tsc), "m"(*local_tsc)); + barrier(); + stamp = omap_dm_timer_read_counter(gpt_clocksource); + if (unlikely(stamp < result.low)) + /* 32 bit counter wrapped, increment high word. */ + result.high++; + result.low = stamp; + + return result.full; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + if (delay > 3) + omap_dm_timer_set_load_start(gptimer, 0, 0xffffffff - delay); + else + ipipe_trigger_irq(__ipipe_mach_timerint); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + struct clock_event_device *ckdev = &clockevent_gpt; + ckdev->set_mode(ckdev->mode, ckdev); + if (ckdev->mode == CLOCK_EVT_MODE_ONESHOT) + ckdev->set_next_event(__ipipe_mach_ticks_per_jiffy, ckdev); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return 0xffffffff - omap_dm_timer_read_counter(gptimer); +} +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c index f6e0300..a4ffded 100644 --- a/arch/arm/mach-pxa/irq.c +++ b/arch/arm/mach-pxa/irq.c @@ -49,6 +49,9 @@ static struct irq_chip pxa_internal_irq_chip = { .name = "SC", .ack = pxa_mask_irq, .mask = pxa_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = pxa_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = pxa_unmask_irq, }; diff --git a/arch/arm/mach-pxa/leds-idp.c b/arch/arm/mach-pxa/leds-idp.c index 8b9c171..144cdc1 100644 --- a/arch/arm/mach-pxa/leds-idp.c +++ b/arch/arm/mach-pxa/leds-idp.c @@ -13,6 +13,7 @@ #include +#include #include #include diff --git a/arch/arm/mach-pxa/leds-lubbock.c b/arch/arm/mach-pxa/leds-lubbock.c index e26d5ef..706cced 100644 --- a/arch/arm/mach-pxa/leds-lubbock.c +++ b/arch/arm/mach-pxa/leds-lubbock.c @@ -12,6 +12,7 @@ */ #include +#include #include #include diff --git a/arch/arm/mach-pxa/leds-mainstone.c b/arch/arm/mach-pxa/leds-mainstone.c index db4af5e..c204cd4 100644 --- a/arch/arm/mach-pxa/leds-mainstone.c +++ b/arch/arm/mach-pxa/leds-mainstone.c @@ -11,6 +11,7 @@ */ #include +#include #include #include diff --git a/arch/arm/mach-pxa/lpd270.c b/arch/arm/mach-pxa/lpd270.c index d64395f..ac4990a 100644 --- a/arch/arm/mach-pxa/lpd270.c +++ b/arch/arm/mach-pxa/lpd270.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -124,7 +125,7 @@ static void lpd270_irq_handler(unsigned int irq, struct irq_desc *desc) GEDR(0) = GPIO_bit(0); /* clear useless edge notification */ if (likely(pending)) { irq = LPD270_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); pending = __raw_readw(LPD270_INT_STATUS) & lpd270_irq_enabled; diff --git a/arch/arm/mach-pxa/lubbock.c b/arch/arm/mach-pxa/lubbock.c index f04c833..2b78dad 100644 --- a/arch/arm/mach-pxa/lubbock.c +++ b/arch/arm/mach-pxa/lubbock.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -161,7 +162,7 @@ static void lubbock_irq_handler(unsigned int irq, struct irq_desc *desc) GEDR(0) = GPIO_bit(0); /* clear our parent irq */ if (likely(pending)) { irq = LUBBOCK_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); } pending = LUB_IRQ_SET_CLR & lubbock_irq_enabled; } while (pending); diff --git a/arch/arm/mach-pxa/mainstone.c b/arch/arm/mach-pxa/mainstone.c index f4dabf0..b230814 100644 --- a/arch/arm/mach-pxa/mainstone.c +++ b/arch/arm/mach-pxa/mainstone.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -165,7 +166,7 @@ static void mainstone_irq_handler(unsigned int irq, struct irq_desc *desc) GEDR(0) = GPIO_bit(0); /* clear useless edge notification */ if (likely(pending)) { irq = MAINSTONE_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); } pending = MST_INTSETCLR & mainstone_irq_enabled; } while (pending); diff --git a/arch/arm/mach-pxa/pcm990-baseboard.c b/arch/arm/mach-pxa/pcm990-baseboard.c index 01791d7..9eb86ca 100644 --- a/arch/arm/mach-pxa/pcm990-baseboard.c +++ b/arch/arm/mach-pxa/pcm990-baseboard.c @@ -24,6 +24,7 @@ #include #include #include +#include #include @@ -263,7 +264,7 @@ static void pcm990_irq_handler(unsigned int irq, struct irq_desc *desc) GPIO_bit(PCM990_CTRL_INT_IRQ_GPIO); if (likely(pending)) { irq = PCM027_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); } pending = (~PCM990_INTSETCLR) & pcm990_irq_enabled; } while (pending); diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c index 750c448..2b7e7f8 100644 --- a/arch/arm/mach-pxa/time.c +++ b/arch/arm/mach-pxa/time.c @@ -24,6 +24,60 @@ #include #include +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint = IRQ_OST0; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int pxa_timer_initialized; + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} + +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + info->u.fr.counter = (unsigned *) 0x40A00010; + info->u.fr.mask = 0xffffffff; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static void ipipe_mach_update_tsc(void); + +#endif /* CONFIG_IPIPE */ + /* * This is PXA's sched_clock implementation. This has a resolution * of at least 308 ns and a maximum value of 208 days. @@ -66,8 +120,12 @@ pxa_ost0_interrupt(int irq, void *dev_id) struct clock_event_device *c = dev_id; /* Disarm the compare/match, signal the event. */ +#ifndef CONFIG_IPIPE OIER &= ~OIER_E0; OSSR = OSSR_M0; +#else /* CONFIG_IPIPE */ + ipipe_mach_update_tsc(); +#endif /* CONFIG_IPIPE */ c->event_handler(c); return IRQ_HANDLED; @@ -79,8 +137,8 @@ pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev) unsigned long flags, next, oscr; raw_local_irq_save(flags); - OIER |= OIER_E0; next = OSCR + delta; + OIER |= OIER_E0; OSMR0 = next; oscr = OSCR; raw_local_irq_restore(flags); @@ -125,6 +183,13 @@ static struct clock_event_device ckevt_pxa_osmr0 = { .set_mode = pxa_osmr0_set_mode, }; +#ifdef CONFIG_IPIPE +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, ckevt_pxa_osmr0.name); +} +#endif /* CONFIG_IPIPE */ + static cycle_t pxa_read_oscr(struct clocksource *cs) { return OSCR; @@ -168,6 +233,15 @@ static void __init pxa_timer_init(void) setup_irq(IRQ_OST0, &pxa_ost0_irq); +#ifdef CONFIG_IPIPE +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; + barrier(); +#endif /* CONFIG_SMP */ + + pxa_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ + clocksource_register(&cksrc_pxa_oscr0); clockevents_register_device(&ckevt_pxa_osmr0); } @@ -213,3 +287,81 @@ struct sys_timer pxa_timer = { .suspend = pxa_timer_suspend, .resume = pxa_timer_resume, }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + OSSR = OSSR_M0; /* Clear match on timer 0 */ + OIER &= ~OIER_E0; +} + +static void ipipe_mach_update_tsc(void) +{ + union tsc_reg *local_tsc; + unsigned long stamp, flags; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = OSCR; + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + local_irq_restore_hw(flags); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(pxa_timer_initialized)) { + union tsc_reg *local_tsc, result; + unsigned long stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + __asm__ ("ldmia %1, %M0\n": + "=r"(result.full): "r"(local_tsc), "m"(*local_tsc)); + barrier(); + stamp = OSCR; + if (unlikely(stamp < result.low)) + /* 32 bit counter wrapped, increment high word. */ + result.high++; + result.low = stamp; + + return result.full; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + if (delay > MIN_OSCR_DELTA) { + unsigned long flags; + + local_irq_save_hw(flags); + OSMR0 = delay + OSCR; + OIER |= OIER_E0; + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(IRQ_OST0); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + pxa_osmr0_set_mode(ckevt_pxa_osmr0.mode, &ckevt_pxa_osmr0); + if (ckevt_pxa_osmr0.mode == CLOCK_EVT_MODE_ONESHOT) + pxa_osmr0_set_next_event(LATCH, &ckevt_pxa_osmr0); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return OSMR0 - OSCR; +} +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c index d33c232..6cb89dc 100644 --- a/arch/arm/mach-pxa/viper.c +++ b/arch/arm/mach-pxa/viper.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include @@ -270,7 +271,7 @@ static void viper_irq_handler(unsigned int irq, struct irq_desc *desc) if (likely(pending)) { irq = viper_bit_to_irq(__ffs(pending)); - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); } pending = viper_irq_pending(); } while (pending); diff --git a/arch/arm/mach-s3c2410/include/mach/irqs.h b/arch/arm/mach-s3c2410/include/mach/irqs.h index 2a2384f..71e9313 100644 --- a/arch/arm/mach-s3c2410/include/mach/irqs.h +++ b/arch/arm/mach-s3c2410/include/mach/irqs.h @@ -3,6 +3,8 @@ * Copyright (c) 2003-2005 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006, 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as * published by the Free Software Foundation. diff --git a/arch/arm/mach-s3c2440/irq.c b/arch/arm/mach-s3c2440/irq.c index 63c5ab6..089b68f 100644 --- a/arch/arm/mach-s3c2440/irq.c +++ b/arch/arm/mach-s3c2440/irq.c @@ -3,6 +3,8 @@ * Copyright (c) 2003,2004 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006, 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -25,6 +27,7 @@ #include #include #include +#include #include #include @@ -57,10 +60,10 @@ static void s3c_irq_demux_wdtac97(unsigned int irq, if (subsrc != 0) { if (subsrc & 1) { - generic_handle_irq(IRQ_S3C2440_WDT); + ipipe_handle_irq_cond(IRQ_S3C2440_WDT); } if (subsrc & 2) { - generic_handle_irq(IRQ_S3C2440_AC97); + ipipe_handle_irq_cond(IRQ_S3C2440_AC97); } } } diff --git a/arch/arm/mach-sa1100/irq.c b/arch/arm/mach-sa1100/irq.c index 3093d46..04a56d2 100644 --- a/arch/arm/mach-sa1100/irq.c +++ b/arch/arm/mach-sa1100/irq.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -125,7 +126,7 @@ sa1100_high_gpio_handler(unsigned int irq, struct irq_desc *desc) mask >>= 11; do { if (mask & 1) - generic_handle_irq(irq); + ipipe_handle_irq_cond(irq); mask >>= 1; irq++; } while (mask); @@ -217,6 +218,9 @@ static struct irq_chip sa1100_normal_chip = { .name = "SC", .ack = sa1100_mask_irq, .mask = sa1100_mask_irq, +#ifdef CONFIG_IPIPE + .mask_ack = sa1100_mask_irq, +#endif /* CONFIG_IPIPE */ .unmask = sa1100_unmask_irq, .set_wake = sa1100_set_wake, }; diff --git a/arch/arm/mach-sa1100/leds-assabet.c b/arch/arm/mach-sa1100/leds-assabet.c index 64e9b4b..0d4e5a4 100644 --- a/arch/arm/mach-sa1100/leds-assabet.c +++ b/arch/arm/mach-sa1100/leds-assabet.c @@ -10,6 +10,7 @@ * - Red - on if system is not idle */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/leds-badge4.c b/arch/arm/mach-sa1100/leds-badge4.c index cf1e384..e41140f 100644 --- a/arch/arm/mach-sa1100/leds-badge4.c +++ b/arch/arm/mach-sa1100/leds-badge4.c @@ -11,6 +11,7 @@ */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/leds-cerf.c b/arch/arm/mach-sa1100/leds-cerf.c index 259b48e..3ab9eff 100644 --- a/arch/arm/mach-sa1100/leds-cerf.c +++ b/arch/arm/mach-sa1100/leds-cerf.c @@ -4,6 +4,7 @@ * Author: ??? */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/leds-hackkit.c b/arch/arm/mach-sa1100/leds-hackkit.c index 2bce137..0a4fe91 100644 --- a/arch/arm/mach-sa1100/leds-hackkit.c +++ b/arch/arm/mach-sa1100/leds-hackkit.c @@ -10,6 +10,7 @@ * as cpu led, the green one is used as timer led. */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/leds-lart.c b/arch/arm/mach-sa1100/leds-lart.c index 0505a1f..f03128d 100644 --- a/arch/arm/mach-sa1100/leds-lart.c +++ b/arch/arm/mach-sa1100/leds-lart.c @@ -10,6 +10,7 @@ * pace of the LED. */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/leds-simpad.c b/arch/arm/mach-sa1100/leds-simpad.c index d50f4ee..2c154e3 100644 --- a/arch/arm/mach-sa1100/leds-simpad.c +++ b/arch/arm/mach-sa1100/leds-simpad.c @@ -4,6 +4,7 @@ * Author: Juergen Messerer */ #include +#include #include #include diff --git a/arch/arm/mach-sa1100/time.c b/arch/arm/mach-sa1100/time.c index 711c029..58d77fc 100644 --- a/arch/arm/mach-sa1100/time.c +++ b/arch/arm/mach-sa1100/time.c @@ -20,13 +20,71 @@ #define MIN_OSCR_DELTA 2 +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint = IRQ_OST0; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int sa1100_timer_initialized; + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} + +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + info->u.fr.counter = (unsigned *)0x90000010; + info->u.fr.mask = 0xffffffff; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static void ipipe_mach_update_tsc(void); + +#endif /* CONFIG_IPIPE */ + static irqreturn_t sa1100_ost0_interrupt(int irq, void *dev_id) { struct clock_event_device *c = dev_id; /* Disarm the compare/match, signal the event. */ +#ifndef CONFIG_IPIPE OIER &= ~OIER_E0; OSSR = OSSR_M0; +#else /* CONFIG_IPIPE */ + ipipe_mach_update_tsc(); +#endif /* CONFIG_IPIPE */ c->event_handler(c); return IRQ_HANDLED; @@ -77,7 +135,14 @@ static struct clock_event_device ckevt_sa1100_osmr0 = { .set_mode = sa1100_osmr0_set_mode, }; -static cycle_t sa1100_read_oscr(void) +#ifdef CONFIG_IPIPE +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, ckevt_sa1100_osmr0.name); +} +#endif /* CONFIG_IPIPE */ + +static cycle_t sa1100_read_oscr(struct clocksource *cs) { return OSCR; } @@ -116,6 +181,15 @@ static void __init sa1100_timer_init(void) setup_irq(IRQ_OST0, &sa1100_timer_irq); +#ifdef CONFIG_IPIPE +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; + barrier(); +#endif /* CONFIG_SMP */ + + sa1100_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ + clocksource_register(&cksrc_sa1100_oscr); clockevents_register_device(&ckevt_sa1100_osmr0); } @@ -156,3 +230,80 @@ struct sys_timer sa1100_timer = { .suspend = sa1100_timer_suspend, .resume = sa1100_timer_resume, }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + OSSR = OSSR_M0; /* Clear match on timer 0 */ + OIER &= ~OIER_E0; +} + +static void ipipe_mach_update_tsc(void) +{ + union tsc_reg *local_tsc; + unsigned long stamp, flags; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + stamp = OSCR; + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + local_irq_restore_hw(flags); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(sa1100_timer_initialized)) { + union tsc_reg *local_tsc, result; + unsigned long stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + __asm__ ("ldmia %1, %M0\n": + "=r"(result.full): "r"(local_tsc), "m"(*local_tsc)); + barrier(); + stamp = OSCR; + if (unlikely(stamp < result.low)) + /* 32 bit counter wrapped, increment high word. */ + result.high++; + result.low = stamp; + return result.full; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + if (delay > MIN_OSCR_DELTA) { + unsigned long flags; + + local_irq_save_hw(flags); + OSMR0 = delay + OSCR; + OIER |= OIER_E0; + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(IRQ_OST0); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + sa1100_osmr0_set_mode(ckevt_sa1100_osmr0.mode, &ckevt_sa1100_osmr0); + if (ckevt_sa1100_osmr0.mode == CLOCK_EVT_MODE_ONESHOT) + sa1100_osmr0_set_next_event(LATCH, &ckevt_sa1100_osmr0); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return OSMR0 - OSCR; +} +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index 83c025e..0f265dd 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -771,3 +771,48 @@ config CACHE_XSC3L2 select OUTER_CACHE help This option enables the L2 cache on XScale3. + +config ARM_FCSE + bool "Fast Context Switch Extension (EXPERIMENTAL)" + depends on EXPERIMENTAL && !SMP && !CPU_32v6 && !CPU_32v7 + default n + help + The Fast Context Switch Extension (FCSE for short) is an extension of + some ARM processors which allows to switch contexts between processes + without flushing cache and saves a few tens of microseconds in the + worst case. + + Enabling this option makes linux use the FCSE. + + We propose two modes: + - the guaranteed mode: we guarantee that there will never be any cache + flush when switching context, but this means that there can not be + more than 95 running processes in the system, each with a virtual + memory space smaller than 32MB, and that the shared memory + mappings do not use cache; + - the best effort mode: we allow some cache flushes to happen from + time to time, but do not limit the number of processes or the + virtual memory space available for each process, and the shared + memory mappings use cache. + +if ARM_FCSE + +choice + prompt "FCSE mode" + default ARM_FCSE_BEST_EFFORT + help + This option allow setting which FCSE mode will be used. + +config ARM_FCSE_GUARANTEED + bool "guaranteed" + help + Select guaranteed mode. + +config ARM_FCSE_BEST_EFFORT + bool "best effort" + help + Select best-effort mode. + +endchoice + +endif diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 03cd27d..a7d9e3f 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -713,6 +713,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) int isize = 4; int thumb2_32b = 0; + if (ipipe_trap_notify(IPIPE_TRAP_ALIGNMENT,regs)) + return 0; + instrptr = instruction_pointer(regs); fs = get_fs(); diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index fc84fcc..5e3254f 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -14,7 +14,7 @@ #include #include -static DEFINE_SPINLOCK(cpu_asid_lock); +static IPIPE_DEFINE_SPINLOCK(cpu_asid_lock); unsigned int cpu_last_asid = ASID_FIRST_VERSION; /* @@ -31,8 +31,9 @@ void __init_new_context(struct task_struct *tsk, struct mm_struct *mm) void __new_context(struct mm_struct *mm) { unsigned int asid; + unsigned long flags; - spin_lock(&cpu_asid_lock); + spin_lock_irqsave_cond(&cpu_asid_lock, flags); asid = ++cpu_last_asid; if (asid == 0) asid = cpu_last_asid = ASID_FIRST_VERSION; @@ -57,7 +58,7 @@ void __new_context(struct mm_struct *mm) dsb(); } } - spin_unlock(&cpu_asid_lock); + spin_unlock_irqrestore_cond(&cpu_asid_lock, flags); mm->cpu_vm_mask = cpumask_of_cpu(smp_processor_id()); mm->context.id = asid; diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 7370a71..3a7928a 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -44,7 +44,7 @@ static DEFINE_SPINLOCK(minicache_lock); * instruction. If your processor does not supply this, you have to write your * own copy_user_highpage that does the right thing. */ -static void __naked +static void notrace __naked mc_copy_user_page(void *from, void *to) { asm volatile( diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 76824d3..db642ce 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -42,7 +42,7 @@ static DEFINE_SPINLOCK(minicache_lock); * Dcache aliasing issue. The writes will be forwarded to the write buffer, * and merged as appropriate. */ -static void __naked +static void notrace __naked mc_copy_user_page(void *from, void *to) { /* diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index bc0099d..9e7b49f 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -25,6 +25,57 @@ static unsigned long shared_pte_mask = L_PTE_MT_BUFFERABLE; +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +static void fcse_set_pte_shared(struct vm_area_struct *vma, + unsigned long address) +{ + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte, entry; + + if (!(vma->vm_flags & VM_MAYSHARE) || address >= TASK_SIZE) + return; + + pgd = pgd_offset(vma->vm_mm, address); + if (pgd_none(*pgd)) + goto no_pgd; + if (pgd_bad(*pgd)) + goto bad_pgd; + + pmd = pmd_offset(pgd, address); + if (pmd_none(*pmd)) + goto no_pmd; + if (pmd_bad(*pmd)) + goto bad_pmd; + + pte = pte_offset_map(pmd, address); + entry = *pte; + + if((pte_val(entry) + & (L_PTE_PRESENT | L_PTE_CACHEABLE | L_PTE_WRITE | L_PTE_DIRTY)) + == (L_PTE_PRESENT | L_PTE_CACHEABLE | L_PTE_WRITE | L_PTE_DIRTY)) { + pte_val(entry) |= L_PTE_SHARED; + set_pte_at(vma->vm_mm, address, pte, entry); + } + pte_unmap(pte); + return; + +bad_pgd: + pgd_ERROR(*pgd); + pgd_clear(pgd); +no_pgd: + return; + +bad_pmd: + pmd_ERROR(*pmd); + pmd_clear(pmd); +no_pmd: + return; +} +#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */ +#define fcse_set_pte_shared(vma, addr) do { } while (0) +#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */ + /* * We take the easy way out of this problem - we make the * PTE uncacheable. However, we leave the write buffer on. @@ -94,6 +145,7 @@ no_pmd: static void make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, unsigned long pfn) { +#ifndef CONFIG_ARM_FCSE_GUARANTEED struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *mpnt; struct prio_tree_iter iter; @@ -125,8 +177,14 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigne flush_dcache_mmap_unlock(mapping); if (aliases) adjust_pte(vma, addr); - else + else { + fcse_set_pte_shared(vma, addr); flush_cache_page(vma, addr, pfn); + } +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + if (vma->vm_flags & VM_MAYSHARE) + adjust_pte(vma, addr); +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ } /* diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 6fdcbb7..4c92911 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -57,6 +57,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr) if (!mm) mm = &init_mm; +#ifdef CONFIG_ARM_FCSE + printk(KERN_ALERT "fcse pid: %ld, 0x%08lx\n", + mm->context.pid >> FCSE_PID_SHIFT, mm->context.pid); +#endif /* CONFIG_ARM_FCSE */ printk(KERN_ALERT "pgd = %p\n", mm->pgd); pgd = pgd_offset(mm, addr); printk(KERN_ALERT "[%08lx] *pgd=%08lx", addr, pgd_val(*pgd)); @@ -251,6 +255,9 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (notify_page_fault(regs, fsr)) return 0; + if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs)) + return 0; + tsk = current; mm = tsk->mm; @@ -351,6 +358,9 @@ do_translation_fault(unsigned long addr, unsigned int fsr, if (addr < TASK_SIZE) return do_page_fault(addr, fsr, regs); + if (ipipe_trap_notify(IPIPE_TRAP_ACCESS,regs)) + return 0; + index = pgd_index(addr); /* @@ -386,6 +396,10 @@ bad_area: static int do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + + if (ipipe_trap_notify(IPIPE_TRAP_SECTION,regs)) + return 0; + do_bad_area(addr, fsr, regs); return 0; } @@ -396,6 +410,9 @@ do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) static int do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + if (ipipe_trap_notify(IPIPE_TRAP_DABT,regs)) + return 0; + return 1; } @@ -468,9 +485,14 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) const struct fsr_info *inf = fsr_info + (fsr & 15) + ((fsr & (1 << 10)) >> 6); struct siginfo info; + addr = fcse_mva_to_va(addr); + if (!inf->fn(addr, fsr, regs)) return; + if (ipipe_trap_notify(IPIPE_TRAP_UNKNOWN,regs)) + return; + printk(KERN_ALERT "Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); @@ -487,3 +509,43 @@ do_PrefetchAbort(unsigned long addr, struct pt_regs *regs) do_translation_fault(addr, 0, regs); } +#ifdef CONFIG_IPIPE +extern spinlock_t pgd_lock; +extern struct page *pgd_list; + +static void vmalloc_sync_one(pgd_t *pgd, unsigned long addr) +{ + unsigned int index = pgd_index(addr); + pgd_t *pgd_k; + pmd_t *pmd, *pmd_k; + + pgd += index; + pgd_k = init_mm.pgd + index; + + if (!pgd_present(*pgd)) + set_pgd(pgd, *pgd_k); + + pmd_k = pmd_offset(pgd_k, addr); + pmd = pmd_offset(pgd, addr); + + copy_pmd(pmd, pmd_k); +} + +void __ipipe_pin_range_globally(unsigned long start, unsigned long end) +{ + unsigned long next, addr = start; + + do { + unsigned long flags; + struct page *page; + + next = pgd_addr_end(addr, end); + spin_lock_irqsave(&pgd_lock, flags); + for (page = pgd_list; page; page = (struct page *)page->index) + vmalloc_sync_one(page_address(page), addr); + spin_unlock_irqrestore(&pgd_lock, flags); + + } while (addr = next, addr != end); +} +#endif /* CONFIG_IPIPE */ + diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index c07222e..34d106e 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -50,8 +50,10 @@ static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr) void flush_cache_mm(struct mm_struct *mm) { if (cache_is_vivt()) { - if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) + if (cpu_isset(smp_processor_id(), mm->cpu_vm_mask)) { + fcse_notify_flush_all(); __cpuc_flush_user_all(); + } return; } @@ -73,9 +75,11 @@ void flush_cache_mm(struct mm_struct *mm) void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { if (cache_is_vivt()) { - if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) - __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end), - vma->vm_flags); + if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { + start = fcse_va_to_mva(vma->vm_mm, start) & PAGE_MASK; + end = PAGE_ALIGN(fcse_va_to_mva(vma->vm_mm, end)); + __cpuc_flush_user_range(start, end, vma->vm_flags); + } return; } @@ -98,8 +102,11 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig { if (cache_is_vivt()) { if (cpu_isset(smp_processor_id(), vma->vm_mm->cpu_vm_mask)) { - unsigned long addr = user_addr & PAGE_MASK; - __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); + unsigned long addr; + addr = fcse_va_to_mva(vma->vm_mm, user_addr); + addr &= PAGE_MASK; + __cpuc_flush_user_range(addr, addr + PAGE_SIZE, + vma->vm_flags); } return; } diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index 0ab75c6..32fdbe6 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -316,6 +316,7 @@ __arm_ioremap_pfn(unsigned long pfn, unsigned long offset, size_t size, } flush_cache_vmap(addr, addr + size); + __ipipe_pin_range_globally(addr, addr + size); return (void __iomem *) (offset + addr); } EXPORT_SYMBOL(__arm_ioremap_pfn); diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index f7457fe..a388f1e 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -56,7 +56,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (flags & MAP_FIXED) { if (aliasing && flags & MAP_SHARED && addr & (SHMLBA - 1)) return -EINVAL; - return addr; + goto found_addr; } if (len > TASK_SIZE) @@ -71,7 +71,7 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) - return addr; + goto found_addr; } if (len > mm->cached_hole_size) { start_addr = addr = mm->free_area_cache; @@ -105,7 +105,7 @@ full_search: * Remember the place where we stopped the search: */ mm->free_area_cache = addr + len; - return addr; + goto found_addr; } if (addr + mm->cached_hole_size < vma->vm_start) mm->cached_hole_size = vma->vm_start - addr; @@ -113,6 +113,26 @@ full_search: if (do_align) addr = COLOUR_ALIGN(addr, pgoff); } + + found_addr: +#ifdef CONFIG_ARM_FCSE + if (addr + len > FCSE_TASK_SIZE) { + if (start_addr != TASK_UNMAPPED_BASE && !(flags & MAP_FIXED)) { + start_addr = addr = TASK_UNMAPPED_BASE; + mm->cached_hole_size = 0; + goto full_search; + } +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + mm->context.big = 1; + if (mm->context.pid) { + fcse_relocate_mm_to_null_pid(mm); + } +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + return -ENOMEM; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + } +#endif /* CONFIG_ARM_FCSE */ + return addr; } diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index 2690146..16c48a3 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -18,6 +18,41 @@ #define FIRST_KERNEL_PGD_NR (FIRST_USER_PGD_NR + USER_PTRS_PER_PGD) +#ifdef CONFIG_IPIPE +/* Copied from arch/i386/mm/pgdtable.c, maintains the list of pgds for the + implementation of ipipe_pin_range_globally in arch/arm/mm/fault.c. */ +DEFINE_SPINLOCK(pgd_lock); +struct page *pgd_list; + +#define pgd_list_lock(flags) spin_lock_irqsave(&pgd_lock, flags) +#define pgd_list_unlock(flags) spin_unlock_irqrestore(&pgd_lock, flags) + +static inline void pgd_list_add(pgd_t *pgd) +{ + struct page *page = virt_to_page(pgd); + page->index = (unsigned long)pgd_list; + if (pgd_list) + set_page_private(pgd_list, (unsigned long)&page->index); + pgd_list = page; + set_page_private(page, (unsigned long)&pgd_list); +} + +static inline void pgd_list_del(pgd_t *pgd) +{ + struct page *next, **pprev, *page = virt_to_page(pgd); + next = (struct page *)page->index; + pprev = (struct page **)page_private(page); + *pprev = next; + if (next) + set_page_private(next, (unsigned long)pprev); +} +#else /* !CONFIG_IPIPE */ +#define pgd_list_lock(flags) ((void) (flags)) +#define pgd_list_unlock(flags) ((void) (flags)) +#define pgd_list_add(pgd) do { } while (0) +#define pgd_list_del(pgd) do { } while (0) +#endif /* !CONFIG_IPIPE */ + /* * need to get a 16k page for level 1 */ @@ -26,6 +61,7 @@ pgd_t *get_pgd_slow(struct mm_struct *mm) pgd_t *new_pgd, *init_pgd; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; + unsigned long flags; new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2); if (!new_pgd) @@ -37,12 +73,20 @@ pgd_t *get_pgd_slow(struct mm_struct *mm) * Copy over the kernel and IO PGD entries */ init_pgd = pgd_offset_k(0); + pgd_list_lock(flags); memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); + pgd_list_add(new_pgd); + pgd_list_unlock(flags); clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); if (!vectors_high()) { +#ifdef CONFIG_ARM_FCSE + /* FCSE does not work without high vectors. */ + BUG(); +#endif /* CONFIG_ARM_FCSE */ + /* * On ARM, first page must always be allocated since it * contains the machine vectors. @@ -74,6 +118,7 @@ no_pgd: void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd) { + unsigned long flags; pmd_t *pmd; pgtable_t pte; @@ -81,7 +126,7 @@ void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd) return; /* pgd is always present and good */ - pmd = pmd_off(pgd, 0); + pmd = pmd_off(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0); if (pmd_none(*pmd)) goto free; if (pmd_bad(*pmd)) { @@ -95,5 +140,8 @@ void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd) pte_free(mm, pte); pmd_free(mm, pmd); free: + pgd_list_lock(flags); + pgd_list_del(pgd); + pgd_list_unlock(flags); free_pages((unsigned long) pgd, 2); } diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 914d688..3eff19c 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -321,6 +321,11 @@ ENTRY(cpu_arm920_dcache_clean_area) ENTRY(cpu_arm920_switch_mm) #ifdef CONFIG_MMU mov ip, #0 +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 3f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache #else @@ -338,6 +343,10 @@ ENTRY(cpu_arm920_switch_mm) #endif mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c7, c10, 4 @ drain WB +#endif /* !CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +3: +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 5446693..f7a6021 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -337,6 +337,11 @@ ENTRY(cpu_arm926_dcache_clean_area) ENTRY(cpu_arm926_switch_mm) #ifdef CONFIG_MMU mov ip, #0 +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 2f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache #else @@ -346,6 +351,10 @@ ENTRY(cpu_arm926_switch_mm) #endif mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c7, c10, 4 @ drain WB +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +2: +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index 0cce37b..1562e2a 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -415,9 +415,18 @@ ENTRY(cpu_xscale_dcache_clean_area) */ .align 5 ENTRY(cpu_xscale_switch_mm) +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 2f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED clean_d_cache r1, r2 mcr p15, 0, ip, c7, c5, 0 @ Invalidate I cache & BTB mcr p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +2: +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs cpwait_ret lr, ip diff --git a/arch/arm/plat-mxc/gpio.c b/arch/arm/plat-mxc/gpio.c index 7506d96..4f712db 100644 --- a/arch/arm/plat-mxc/gpio.c +++ b/arch/arm/plat-mxc/gpio.c @@ -25,6 +25,13 @@ #include #include #include +#ifdef CONFIG_IPIPE +#include +#ifdef CONFIG_MACH_MX31ADS +#include +#include +#endif /* CONFIG_MACH_MX31ADS */ +#endif /* CONFIG_IPIPE */ static struct mxc_gpio_port *mxc_gpio_ports; static int gpio_table_size; @@ -157,8 +164,7 @@ static void mxc_gpio_irq_handler(struct mxc_gpio_port *port, u32 irq_stat) if (port->both_edges & (1 << (gpio & 31))) mxc_flip_edge(port, gpio); - irq_desc[gpio_irq_no].handle_irq(gpio_irq_no, - &irq_desc[gpio_irq_no]); + ipipe_handle_irq_cond(gpio_irq_no); } } @@ -174,7 +180,7 @@ static void mx3_gpio_irq_handler(u32 irq, struct irq_desc *desc) mxc_gpio_irq_handler(port, irq_stat); } -#endif +#endif /* CONFIG_ARCH_MX3 */ #ifdef CONFIG_ARCH_MX2 /* MX2 has one interrupt *for all* gpio ports */ @@ -195,7 +201,7 @@ static void mx2_gpio_irq_handler(u32 irq, struct irq_desc *desc) mxc_gpio_irq_handler(&port[i], irq_stat); } } -#endif +#endif /* CONFIG_ARCH_MX2 */ static struct irq_chip gpio_irq_chip = { .ack = gpio_ack_irq, @@ -269,7 +275,11 @@ int __init mxc_gpio_init(struct mxc_gpio_port *port, int cnt) for (j = port[i].virtual_irq_start; j < port[i].virtual_irq_start + 32; j++) { set_irq_chip(j, &gpio_irq_chip); +#ifndef CONFIG_IPIPE set_irq_handler(j, handle_edge_irq); +#else /* CONFIG_IPIPE */ + set_irq_handler(j, handle_level_irq); +#endif /* CONFIG_IPIPE */ set_irq_flags(j, IRQF_VALID); } diff --git a/arch/arm/plat-mxc/time.c b/arch/arm/plat-mxc/time.c index 88fb3a5..c34206c 100644 --- a/arch/arm/plat-mxc/time.c +++ b/arch/arm/plat-mxc/time.c @@ -57,6 +57,72 @@ #define MX3_TCN 0x24 #define MX3_TCMP 0x10 +#ifdef CONFIG_IPIPE +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ +int __ipipe_mach_timerint; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +unsigned int __ipipe_mach_ticks_per_jiffy = LATCH; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +static int mxc_timer_initialized; +static unsigned mxc_min_delay; + +union tsc_reg { +#ifdef __BIG_ENDIAN + struct { + unsigned long high; + unsigned long low; + }; +#else /* __LITTLE_ENDIAN */ + struct { + unsigned long low; + unsigned long high; + }; +#endif /* __LITTLE_ENDIAN */ + unsigned long long full; +}; + +#ifdef CONFIG_SMP +static union tsc_reg tsc[NR_CPUS]; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_NONE; +} + +#else /* !CONFIG_SMP */ +static union tsc_reg *tsc; + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_FREERUNNING; + if (cpu_is_mx1()) { +#ifdef CONFIG_ARCH_MX1 + info->u.fr.counter = (unsigned *) (TIM1_BASE_ADDR + MX1_2_TCN); +#endif + } else if (cpu_is_mx2()) { +#ifdef CONFIG_ARCH_MX2 + info->u.fr.counter = (unsigned *) (GPT1_BASE_ADDR + MX1_2_TCN); +#endif + } else if (cpu_is_mx3()) { +#ifdef CONFIG_ARCH_MX3 + info->u.fr.counter = (unsigned *) (GPT1_BASE_ADDR + MX3_TCN); +#endif + } + info->u.fr.mask = 0xffffffff; + info->u.fr.tsc = &tsc->full; +} +#endif /* !CONFIG_SMP */ + +static void ipipe_mach_update_tsc(void); +#endif /* CONFIG_IPIPE */ + static struct clock_event_device clockevent_mxc; static enum clock_event_mode clockevent_mode = CLOCK_EVT_MODE_UNUSED; @@ -120,6 +186,9 @@ static int __init mxc_clocksource_init(struct clk *timer_clk) if (cpu_is_mx3()) clocksource_mxc.read = mx3_get_cycles; +#ifdef CONFIG_IPIPE + __ipipe_mach_ticks_per_jiffy = (c + HZ/2) / HZ; +#endif /* CONFIG_IPIPE */ clocksource_mxc.mult = clocksource_hz2mult(c, clocksource_mxc.shift); clocksource_register(&clocksource_mxc); @@ -238,7 +307,11 @@ static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id) else tstat = __raw_readl(timer_base + MX1_2_TSTAT); +#ifndef CONFIG_IPIPE gpt_irq_acknowledge(); +#else /* !CONFIG_IPIPE */ + ipipe_mach_update_tsc(); +#endif /* !CONFIG_IPIPE */ evt->event_handler(evt); @@ -326,4 +399,116 @@ void __init mxc_timer_init(struct clk *timer_clk) /* Make irqs happen */ setup_irq(irq, &mxc_timer_irq); + +#ifdef CONFIG_IPIPE + __ipipe_mach_timerint = irq; +#ifndef CONFIG_SMP + tsc = (union tsc_reg *) __ipipe_tsc_area; + barrier(); +#endif /* CONFIG_SMP */ + + mxc_min_delay = ((ipipe_cpu_freq() + 500000) / 1000000) ?: 1; + mxc_timer_initialized = 1; +#endif /* CONFIG_IPIPE */ +} + +#ifdef CONFIG_IPIPE +int __ipipe_check_tickdev(const char *devname) +{ + return !strcmp(devname, clockevent_mxc.name); +} + +void __ipipe_mach_acktimer(void) +{ + gpt_irq_acknowledge(); +} + +static void ipipe_mach_update_tsc(void) +{ + union tsc_reg *local_tsc; + unsigned long stamp, flags; + + local_irq_save_hw(flags); + local_tsc = &tsc[ipipe_processor_id()]; + if (!cpu_is_mx3()) + stamp = __raw_readl(timer_base + MX1_2_TCN); + else + stamp = __raw_readl(timer_base + MX3_TCN); + if (unlikely(stamp < local_tsc->low)) + /* 32 bit counter wrapped, increment high word. */ + local_tsc->high++; + local_tsc->low = stamp; + local_irq_restore_hw(flags); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + if (likely(mxc_timer_initialized)) { + union tsc_reg *local_tsc, result; + unsigned long stamp; + + local_tsc = &tsc[ipipe_processor_id()]; + + if (!cpu_is_mx3()) { + __asm__ ("ldmia %1, %M0\n" + : "=r"(result.full), "+&r"(local_tsc) + : "m"(*local_tsc)); + barrier(); + stamp = __raw_readl(timer_base + MX1_2_TCN); + } else { + __asm__ ("ldmia %1, %M0\n" + : "=r"(result.full), "+&r"(local_tsc) + : "m"(*local_tsc)); + barrier(); + stamp = __raw_readl(timer_base + MX3_TCN); + } + if (unlikely(stamp < result.low)) + /* 32 bit counter wrapped, increment high word. */ + result.high++; + result.low = stamp; + + return result.full; + } + + return 0; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +/* + * Reprogram the timer + */ + +void __ipipe_mach_set_dec(unsigned long delay) +{ + if (delay > mxc_min_delay) { + unsigned long flags; + + local_irq_save_hw(flags); + if (!cpu_is_mx3()) + mx1_2_set_next_event(delay, NULL); + else + mx3_set_next_event(delay, NULL); + local_irq_restore_hw(flags); + } else + ipipe_trigger_irq(__ipipe_mach_timerint); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + mxc_set_mode(clockevent_mxc.mode, &clockevent_mxc); + if (clockevent_mxc.mode == CLOCK_EVT_MODE_ONESHOT) + clockevent_mxc.set_next_event(LATCH, &clockevent_mxc); +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + if (!cpu_is_mx3()) + return __raw_readl(timer_base + MX1_2_TCMP) + - __raw_readl(timer_base + MX1_2_TCN); + else + return __raw_readl(timer_base + MX3_TCMP) + - __raw_readl(timer_base + MX3_TCN); } +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/plat-omap/Kconfig b/arch/arm/plat-omap/Kconfig index efe85d0..f1a8e0d 100644 --- a/arch/arm/plat-omap/Kconfig +++ b/arch/arm/plat-omap/Kconfig @@ -131,6 +131,7 @@ config OMAP_MPU_TIMER timer provides more intra-tick resolution than the 32KHz timer, but consumes more power. +if !IPIPE config OMAP_32K_TIMER bool "Use 32KHz timer" depends on ARCH_OMAP16XX || ARCH_OMAP24XX || ARCH_OMAP34XX || ARCH_OMAP4 @@ -140,6 +141,7 @@ config OMAP_32K_TIMER support for no tick during idle. The 32KHz timer provides less intra-tick resolution than OMAP_MPU_TIMER. The 32KHz timer is currently only available for OMAP16XX, 24XX, 34XX and OMAP4. +endif endchoice diff --git a/arch/arm/plat-omap/dmtimer.c b/arch/arm/plat-omap/dmtimer.c index 7f50b61..915409e 100644 --- a/arch/arm/plat-omap/dmtimer.c +++ b/arch/arm/plat-omap/dmtimer.c @@ -475,6 +475,13 @@ int omap_dm_timer_get_irq(struct omap_dm_timer *timer) } EXPORT_SYMBOL_GPL(omap_dm_timer_get_irq); +#ifdef CONFIG_IPIPE +unsigned long omap_dm_timer_get_phys_counter_addr(struct omap_dm_timer *timer) +{ + return timer->phys_base + (OMAP_TIMER_COUNTER_REG & 0xff); +} +#endif /* CONFIG_IPIPE */ + #if defined(CONFIG_ARCH_OMAP1) /** diff --git a/arch/arm/plat-omap/gpio.c b/arch/arm/plat-omap/gpio.c index 9298bc0..6108aa1 100644 --- a/arch/arm/plat-omap/gpio.c +++ b/arch/arm/plat-omap/gpio.c @@ -20,6 +20,7 @@ #include #include #include +#include /* For irq_desc */ #include #include @@ -1200,7 +1201,7 @@ static void gpio_irq_handler(unsigned int irq, struct irq_desc *desc) if (!(isr & 1)) continue; - generic_handle_irq(gpio_irq); + ipipe_handle_irq_cond(gpio_irq); } } /* if bank has any level sensitive GPIO pin interrupt diff --git a/arch/arm/plat-omap/include/mach/dmtimer.h b/arch/arm/plat-omap/include/mach/dmtimer.h index 20f1054..42a0b8d 100644 --- a/arch/arm/plat-omap/include/mach/dmtimer.h +++ b/arch/arm/plat-omap/include/mach/dmtimer.h @@ -56,6 +56,9 @@ void omap_dm_timer_enable(struct omap_dm_timer *timer); void omap_dm_timer_disable(struct omap_dm_timer *timer); int omap_dm_timer_get_irq(struct omap_dm_timer *timer); +#ifdef CONFIG_IPIPE +unsigned long omap_dm_timer_get_phys_counter_addr(struct omap_dm_timer *timer); +#endif /* CONFIG_IPIPE */ u32 omap_dm_timer_modify_idlect_mask(u32 inputmask); struct clk *omap_dm_timer_get_fclk(struct omap_dm_timer *timer); diff --git a/arch/arm/plat-pxa/gpio.c b/arch/arm/plat-pxa/gpio.c index 98548c6..e732e45 100644 --- a/arch/arm/plat-pxa/gpio.c +++ b/arch/arm/plat-pxa/gpio.c @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -220,7 +221,7 @@ static void pxa_gpio_demux_handler(unsigned int irq, struct irq_desc *desc) while (n < BITS_PER_LONG) { loop = 1; - generic_handle_irq(gpio_to_irq(gpio_base + n)); + ipipe_handle_irq_cond(gpio_to_irq(gpio_base + n)); n = find_next_bit(&gedr, BITS_PER_LONG, n + 1); } } @@ -285,7 +286,7 @@ void __init pxa_init_gpio(int mux_irq, int start, int end, set_wake_t fn) for (irq = gpio_to_irq(start); irq <= gpio_to_irq(end); irq++) { set_irq_chip(irq, &pxa_muxed_gpio_chip); - set_irq_handler(irq, handle_edge_irq); + set_irq_handler(irq, handle_level_irq); set_irq_flags(irq, IRQF_VALID | IRQF_PROBE); } diff --git a/arch/arm/plat-s3c/time.c b/arch/arm/plat-s3c/time.c index 3b27b29..8e6e479 100644 --- a/arch/arm/plat-s3c/time.c +++ b/arch/arm/plat-s3c/time.c @@ -3,6 +3,8 @@ * Copyright (C) 2003-2005 Simtec Electronics * Ben Dooks, * + * Copyright (C) 2006, 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -27,6 +29,7 @@ #include #include #include +#include #include #include @@ -42,7 +45,6 @@ #include #include -static unsigned long timer_startval; static unsigned long timer_usec_ticks; #ifndef TICK_MAX @@ -61,6 +63,39 @@ static unsigned long timer_usec_ticks; * Original patch by Dimitry Andric, updated by Ben Dooks */ +static unsigned long last_free_running_tcnt = 0; +static unsigned long free_running_tcon = 0; +static unsigned long timer_lxlost = 0; + +#ifdef CONFIG_IPIPE + +#ifdef CONFIG_NO_IDLE_HZ +#error "dynamic tick timer not yet supported with IPIPE" +#endif /* CONFIG_NO_IDLE_HZ */ + +unsigned int __ipipe_mach_ticks_per_jiffy; +EXPORT_SYMBOL(__ipipe_mach_ticks_per_jiffy); + +int __ipipe_mach_timerint = IRQ_TIMER4; +EXPORT_SYMBOL(__ipipe_mach_timerint); + +static unsigned long long *tsc; +static unsigned *last_cnt; +static unsigned long timer_ackval = 1UL << (IRQ_TIMER4 - IRQ_EINT0); +static IPIPE_DEFINE_SPINLOCK(timer_lock); + +int __ipipe_mach_timerstolen = 0; +EXPORT_SYMBOL(__ipipe_mach_timerstolen); + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + info->type = IPIPE_TSC_TYPE_DECREMENTER; + info->u.dec.counter = (unsigned *)0x51000038; + info->u.dec.mask = 0xffff; + info->u.dec.last_cnt = last_cnt; + info->u.dec.tsc = tsc; +} +#endif /* CONFIG_IPIPE */ /* timer_mask_usec_ticks * @@ -91,38 +126,45 @@ static inline unsigned long timer_ticks_to_usec(unsigned long ticks) return res >> TIMER_USEC_SHIFT; } -/*** - * Returns microsecond since last clock interrupt. Note that interrupts - * will have been disabled by do_gettimeoffset() - * IRQs are disabled before entering here from do_gettimeofday() - */ - -static unsigned long s3c2410_gettimeoffset (void) +static inline unsigned long timer_freerunning_getvalue(void) { - unsigned long tdone; - unsigned long tval; - - /* work out how many ticks have gone since last timer interrupt */ + return __raw_readl(S3C2410_TCNTO(3)); +} - tval = __raw_readl(S3C2410_TCNTO(4)); - tdone = timer_startval - tval; +static inline unsigned long timer_freerunning_getticksoffset(unsigned long tval) +{ + long tdone; - /* check to see if there is an interrupt pending */ + tdone = last_free_running_tcnt - tval; + if (tdone < 0) + tdone += 0x10000; - if (s3c24xx_ostimer_pending()) { - /* re-read the timer, and try and fix up for the missed - * interrupt. Note, the interrupt may go off before the - * timer has re-loaded from wrapping. - */ + return tdone; +} - tval = __raw_readl(S3C2410_TCNTO(4)); - tdone = timer_startval - tval; +static inline unsigned long getticksoffset(void) +{ + return timer_freerunning_getticksoffset(timer_freerunning_getvalue()); +} - if (tval != 0) - tdone += timer_startval; - } +#ifdef CONFIG_IPIPE +static inline unsigned long getticksoffset_tscupdate(void) +{ + unsigned long tval; + unsigned long ticks; + + tval = timer_freerunning_getvalue(); + ticks = timer_freerunning_getticksoffset(tval); + last_free_running_tcnt = tval; + *tsc += ticks; + *last_cnt = last_free_running_tcnt; + return ticks; +} +#endif /* CONFIG_IPIPE */ - return timer_ticks_to_usec(tdone); +static unsigned long s3c2410_gettimeoffset (void) +{ + return timer_ticks_to_usec(timer_lxlost + getticksoffset()); } @@ -132,6 +174,13 @@ static unsigned long s3c2410_gettimeoffset (void) static irqreturn_t s3c2410_timer_interrupt(int irq, void *dev_id) { +#ifdef CONFIG_IPIPE + timer_lxlost = 0; + + if (!__ipipe_mach_timerstolen) + getticksoffset_tscupdate(); +#endif /* CONFIG_IPIPE */ + timer_tick(); return IRQ_HANDLED; } @@ -153,10 +202,10 @@ static struct clk *tdiv; static struct clk *timerclk; /* - * Set up timer interrupt, and return the current time in seconds. + * Set up timer interrupt. * - * Currently we only use timer4, as it is the only timer which has no - * other function that can be exploited externally + * Currently we use timer4 as event timer and timer3 as tick counter which + * permanently counts ticks without interrupt generation. */ static void s3c2410_timer_setup (void) { @@ -164,6 +213,7 @@ static void s3c2410_timer_setup (void) unsigned long tcnt; unsigned long tcfg1; unsigned long tcfg0; + unsigned long intmask; tcnt = TICK_MAX; /* default value for tcnt */ @@ -175,8 +225,8 @@ static void s3c2410_timer_setup (void) tcnt = 12000000 / HZ; tcfg1 = __raw_readl(S3C2410_TCFG1); - tcfg1 &= ~S3C2410_TCFG1_MUX4_MASK; - tcfg1 |= S3C2410_TCFG1_MUX4_TCLK1; + tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK); + tcfg1 |= (S3C2410_TCFG1_MUX4_TCLK1 | S3C2410_TCFG1_MUX3_TCLK1); __raw_writel(tcfg1, S3C2410_TCFG1); } else { unsigned long pclk; @@ -204,12 +254,25 @@ static void s3c2410_timer_setup (void) clk_set_parent(tin, tdiv); tcnt = clk_get_rate(tin) / HZ; + + tcfg1 = __raw_readl(S3C2410_TCFG1); + tcfg1 &= ~(S3C2410_TCFG1_MUX4_MASK | S3C2410_TCFG1_MUX3_MASK); + tcfg1 |= (S3C2410_TCFG1_MUX4_TCLK1 | S3C2410_TCFG1_MUX3_TCLK1); + __raw_writel(tcfg1, S3C2410_TCFG1); } tcon = __raw_readl(S3C2410_TCON); tcfg0 = __raw_readl(S3C2410_TCFG0); tcfg1 = __raw_readl(S3C2410_TCFG1); +#ifdef CONFIG_IPIPE + tsc = (unsigned long long *)__ipipe_tsc_area; + last_cnt = (unsigned *)(tsc + 1); /* means: +8 bytes */ + barrier(); + + __ipipe_mach_ticks_per_jiffy = tcnt; +#endif /* CONFIG_IPIPE */ + /* timers reload after counting zero, so reduce the count by 1 */ tcnt--; @@ -226,23 +289,37 @@ static void s3c2410_timer_setup (void) __raw_writel(tcfg1, S3C2410_TCFG1); __raw_writel(tcfg0, S3C2410_TCFG0); - timer_startval = tcnt; - __raw_writel(tcnt, S3C2410_TCNTB(4)); - - /* ensure timer is stopped... */ + /* ensure timers are stopped... */ + tcon &= ~(0x3f<<17); + __raw_writel(tcon, S3C2410_TCON); - tcon &= ~(7<<20); - tcon |= S3C2410_TCON_T4RELOAD; - tcon |= S3C2410_TCON_T4MANUALUPD; + /* Mask timer3 interrupt. */ + intmask = __raw_readl(S3C2410_INTMSK); + intmask |= 1UL << (IRQ_TIMER3 - IRQ_EINT0); + __raw_writel(intmask, S3C2410_INTMSK); - __raw_writel(tcon, S3C2410_TCON); + /* Set timer values */ __raw_writel(tcnt, S3C2410_TCNTB(4)); __raw_writel(tcnt, S3C2410_TCMPB(4)); + __raw_writel(0xffff, S3C2410_TCNTB(3)); + __raw_writel(0xffff, S3C2410_TCMPB(3)); + + /* Set base tcon value for later programming of timer 4 by Xenomai. */ + free_running_tcon = tcon | S3C2410_TCON_T3RELOAD | S3C2410_TCON_T3START; - /* start the timer running */ - tcon |= S3C2410_TCON_T4START; - tcon &= ~S3C2410_TCON_T4MANUALUPD; + /* Set auto reloads for both timers. */ + tcon |= S3C2410_TCON_T3RELOAD | S3C2410_TCON_T4RELOAD; + + /* Manual update */ + __raw_writel(tcon | S3C2410_TCON_T3MANUALUPD + | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON); + + tcon |= S3C2410_TCON_T3START | S3C2410_TCON_T4START; + /* Start timers.*/ __raw_writel(tcon, S3C2410_TCON); + + /* Save start value of timer 3 as begining of first period. */ + last_free_running_tcnt = 0xffff; } static void __init s3c2410_timer_resources(void) @@ -283,3 +360,58 @@ struct sys_timer s3c24xx_timer = { .offset = s3c2410_gettimeoffset, .resume = s3c2410_timer_setup }; + +#ifdef CONFIG_IPIPE +void __ipipe_mach_acktimer(void) +{ + __raw_writel(timer_ackval, S3C2410_SRCPND); + __raw_writel(timer_ackval, S3C2410_INTPND); +} + +notrace unsigned long long __ipipe_mach_get_tsc(void) +{ + unsigned long long result; + unsigned long flags; + + local_irq_save_hw_notrace(flags); + spin_lock(&timer_lock); + result = *tsc + getticksoffset(); + spin_unlock(&timer_lock); + local_irq_restore_hw_notrace(flags); + return result; +} +EXPORT_SYMBOL(__ipipe_mach_get_tsc); + +static inline void set_dec(unsigned long reload) +{ + __raw_writel(reload, S3C2410_TCNTB(4)); + /* Manual update */ + __raw_writel(free_running_tcon | S3C2410_TCON_T4MANUALUPD, S3C2410_TCON); + /* Start timer */ + __raw_writel(free_running_tcon | S3C2410_TCON_T4START, S3C2410_TCON); +} + +void __ipipe_mach_set_dec(unsigned long reload) +{ + unsigned long flags; + + spin_lock_irqsave(&timer_lock, flags); + timer_lxlost += getticksoffset_tscupdate(); + set_dec(reload); + spin_unlock_irqrestore(&timer_lock, flags); +} +EXPORT_SYMBOL(__ipipe_mach_set_dec); + +void __ipipe_mach_release_timer(void) +{ + free_running_tcon |= S3C2410_TCON_T4RELOAD; + __ipipe_mach_set_dec(__ipipe_mach_ticks_per_jiffy - 1); + free_running_tcon &= ~S3C2410_TCON_T4RELOAD; +} +EXPORT_SYMBOL(__ipipe_mach_release_timer); + +unsigned long __ipipe_mach_get_dec(void) +{ + return __raw_readl(S3C2410_TCNTO(4)); +} +#endif /* CONFIG_IPIPE */ diff --git a/arch/arm/plat-s3c24xx/irq.c b/arch/arm/plat-s3c24xx/irq.c index 9587377..64fa335 100644 --- a/arch/arm/plat-s3c24xx/irq.c +++ b/arch/arm/plat-s3c24xx/irq.c @@ -3,6 +3,8 @@ * Copyright (c) 2003,2004 Simtec Electronics * Ben Dooks * + * Copyright (C) 2006, 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -23,6 +25,7 @@ #include #include #include +#include #include #include @@ -87,6 +90,9 @@ struct irq_chip s3c_irq_level_chip = { .name = "s3c-level", .ack = s3c_irq_maskack, .mask = s3c_irq_mask, +#ifdef CONFIG_IPIPE + .mask_ack = s3c_irq_maskack, +#endif /* CONFIG_IPIPE */ .unmask = s3c_irq_unmask, .set_wake = s3c_irq_wake }; @@ -283,6 +289,9 @@ static struct irq_chip s3c_irq_uart0 = { .mask = s3c_irq_uart0_mask, .unmask = s3c_irq_uart0_unmask, .ack = s3c_irq_uart0_ack, +#ifdef CONFIG_IPIPE + .mask_ack = s3c_irq_uart0_ack, +#endif /* CONFIG_IPIPE */ }; /* UART1 */ @@ -310,6 +319,9 @@ static struct irq_chip s3c_irq_uart1 = { .mask = s3c_irq_uart1_mask, .unmask = s3c_irq_uart1_unmask, .ack = s3c_irq_uart1_ack, +#ifdef CONFIG_IPIPE + .mask_ack = s3c_irq_uart1_ack, +#endif /* CONFIG_IPIPE */ }; /* UART2 */ @@ -337,6 +349,9 @@ static struct irq_chip s3c_irq_uart2 = { .mask = s3c_irq_uart2_mask, .unmask = s3c_irq_uart2_unmask, .ack = s3c_irq_uart2_ack, +#ifdef CONFIG_IPIPE + .mask_ack = s3c_irq_uart2_ack, +#endif /* CONFIG_IPIPE */ }; /* ADC and Touchscreen */ @@ -385,10 +400,10 @@ static void s3c_irq_demux_adc(unsigned int irq, if (subsrc != 0) { if (subsrc & 1) { - generic_handle_irq(IRQ_TC); + ipipe_handle_irq_cond(IRQ_TC); } if (subsrc & 2) { - generic_handle_irq(IRQ_ADC); + ipipe_handle_irq_cond(IRQ_ADC); } } } @@ -413,13 +428,13 @@ static void s3c_irq_demux_uart(unsigned int start) if (subsrc != 0) { if (subsrc & 1) - generic_handle_irq(start); + ipipe_handle_irq_cond(start); if (subsrc & 2) - generic_handle_irq(start+1); + ipipe_handle_irq_cond(start+1); if (subsrc & 4) - generic_handle_irq(start+2); + ipipe_handle_irq_cond(start+2); } } @@ -466,7 +481,7 @@ s3c_irq_demux_extint8(unsigned int irq, eintpnd &= ~(1< * + * Copyright (C) 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -57,10 +59,10 @@ static void s3c_irq_demux_cam(unsigned int irq, if (subsrc != 0) { if (subsrc & 1) { - generic_handle_irq(IRQ_S3C2440_CAM_C); + ipipe_handle_irq_cond(IRQ_S3C2440_CAM_C); } if (subsrc & 2) { - generic_handle_irq(IRQ_S3C2440_CAM_P); + ipipe_handle_irq_cond(IRQ_S3C2440_CAM_P); } } } @@ -89,6 +91,9 @@ static struct irq_chip s3c_irq_cam = { .mask = s3c_irq_cam_mask, .unmask = s3c_irq_cam_unmask, .ack = s3c_irq_cam_ack, +#ifdef CONFIG_IPIPE + .mask_ack = s3c_irq_cam_ack, +#endif /* CONFIG_IPIPE */ }; static int s3c244x_irq_add(struct sys_device *sysdev) diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S index a2bed62..dfe003b 100644 --- a/arch/arm/vfp/entry.S +++ b/arch/arm/vfp/entry.S @@ -25,7 +25,6 @@ ENTRY(do_vfp) add r11, r4, #1 @ increment it str r11, [r10, #TI_PREEMPT] #endif - enable_irq ldr r4, .LCvfp ldr r11, [r10, #TI_CPU] @ CPU number add r10, r10, #TI_VFPSTATE @ r10 = workspace @@ -33,6 +32,7 @@ ENTRY(do_vfp) ENDPROC(do_vfp) ENTRY(vfp_null_entry) + enable_irq #ifdef CONFIG_PREEMPT get_thread_info r10 ldr r4, [r10, #TI_PREEMPT] @ get preempt count @@ -50,6 +50,7 @@ ENDPROC(vfp_null_entry) __INIT ENTRY(vfp_testing_entry) + enable_irq #ifdef CONFIG_PREEMPT get_thread_info r10 ldr r4, [r10, #TI_PREEMPT] @ get preempt count diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index 1aeae38..7045eef 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -19,7 +19,7 @@ #include "../kernel/entry-header.S" .macro DBGSTR, str -#ifdef DEBUG +#if defined(DEBUG) && !defined(CONFIG_IPIPE) stmfd sp!, {r0-r3, ip, lr} add r0, pc, #4 bl printk @@ -31,7 +31,7 @@ .endm .macro DBGSTR1, str, arg -#ifdef DEBUG +#if defined(DEBUG) && !defined(CONFIG_IPIPE) stmfd sp!, {r0-r3, ip, lr} mov r1, \arg add r0, pc, #4 @@ -44,7 +44,7 @@ .endm .macro DBGSTR3, str, arg1, arg2, arg3 -#ifdef DEBUG +#if defined(DEBUG) && !defined(CONFIG_IPIPE) stmfd sp!, {r0-r3, ip, lr} mov r3, \arg3 mov r2, \arg2 @@ -87,6 +87,11 @@ ENTRY(vfp_support_entry) @ still there. In this case, we do @ not want to drop a pending exception. + enable_irq +#ifdef CONFIG_IPIPE + disable_irq + ldr r4, [r3, r11, lsl #2] @ reload last_VFP_context pointer +#endif VFPFMXR FPEXC, r5 @ enable VFP, disable any pending @ exceptions, so we can get at the @ rest of it @@ -139,6 +144,7 @@ check_for_exception: @ out before setting an FPEXC that @ stops us reading stuff VFPFMXR FPEXC, r1 @ restore FPEXC last + enable_irq_cond sub r2, r2, #4 str r2, [sp, #S_PC] @ retry the instruction #ifdef CONFIG_PREEMPT @@ -164,6 +170,7 @@ look_for_VFP_exceptions: @ Fall into hand on to next handler - appropriate coproc instr @ not recognised by VFP + enable_irq_cond DBGSTR "not VFP" #ifdef CONFIG_PREEMPT get_thread_info r10 diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index 2d7423a..f6beeb7 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -42,11 +42,14 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; union vfp_state *vfp; + unsigned long flags; __u32 cpu = thread->cpu; if (likely(cmd == THREAD_NOTIFY_SWITCH)) { - u32 fpexc = fmrx(FPEXC); + u32 fpexc; + local_irq_save_hw_cond(flags); + fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP /* * On SMP, if VFP is enabled, save the old state in @@ -71,6 +74,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); + local_irq_restore_hw_cond(flags); return NOTIFY_DONE; } @@ -87,12 +91,15 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) /* * Disable VFP to ensure we initialise it first. */ + local_irq_save_hw_cond(flags); fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); - } + } else + local_irq_save_hw_cond(flags); /* flush and release case: Per-thread VFP cleanup. */ if (last_VFP_context[cpu] == vfp) last_VFP_context[cpu] = NULL; + local_irq_restore_hw_cond(flags); return NOTIFY_DONE; } @@ -122,7 +129,9 @@ void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs) current->thread.error_code = 0; current->thread.trap_no = 6; + local_irq_enable_hw_cond(); send_sig_info(SIGFPE, &info, current); + local_irq_disable_hw_cond(); } static void vfp_panic(char *reason, u32 inst) @@ -219,7 +228,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) */ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) { - u32 fpscr, orig_fpscr, fpsid, exceptions; + u32 fpscr, orig_fpscr, fpsid, exceptions, next_trigger = 0; pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc); @@ -267,7 +276,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) * unallocated VFP instruction but with FPSCR.IXE set and not * on VFP subarch 1. */ - vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs); + vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs); goto exit; } @@ -285,6 +294,15 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT); } + if (!(fpexc ^ (FPEXC_EX | FPEXC_FP2V))) { + /* + * The barrier() here prevents fpinst2 being read + * before the condition above. + */ + barrier(); + next_trigger = fmrx(FPINST2); + } + /* * Handle the first FP instruction. We used to take note of the * FPEXC bounce reason, but this appears to be unreliable. @@ -301,18 +319,14 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) if (fpexc ^ (FPEXC_EX | FPEXC_FP2V)) goto exit; - /* - * The barrier() here prevents fpinst2 being read - * before the condition above. - */ - barrier(); - trigger = fmrx(FPINST2); + trigger = next_trigger; emulate: exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); if (exceptions) vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); exit: + local_irq_enable_hw_cond(); preempt_enable(); } diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 68ab39d..ede141b 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -32,7 +32,7 @@ config ATMEL_TCLIB config ATMEL_TCB_CLKSRC bool "TC Block Clocksource" - depends on ATMEL_TCLIB && GENERIC_TIME + depends on ATMEL_TCLIB && GENERIC_TIME && !IPIPE default y help Select this to get a high precision clocksource based on a diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c index 737a1c4..15e81de 100644 --- a/drivers/pci/htirq.c +++ b/drivers/pci/htirq.c @@ -21,7 +21,7 @@ * With multiple simultaneous hypertransport irq devices it might pay * to make this more fine grained. But start with simple, stupid, and correct. */ -static DEFINE_SPINLOCK(ht_irq_lock); +static IPIPE_DEFINE_SPINLOCK(ht_irq_lock); struct ht_irq_cfg { struct pci_dev *dev; diff --git a/drivers/serial/8250.c b/drivers/serial/8250.c index fb867a9..8112f59 100644 --- a/drivers/serial/8250.c +++ b/drivers/serial/8250.c @@ -3010,6 +3010,51 @@ static int serial8250_resume(struct platform_device *dev) return 0; } +#if defined(CONFIG_IPIPE_DEBUG) && defined(CONFIG_SERIAL_8250_CONSOLE) + +#include + +void __ipipe_serial_debug(const char *fmt, ...) +{ + struct uart_8250_port *up = &serial8250_ports[0]; + unsigned int ier, count; + unsigned long flags; + char buf[128]; + va_list ap; + + va_start(ap, fmt); + vsprintf(buf, fmt, ap); + va_end(ap); + count = strlen(buf); + + touch_nmi_watchdog(); + + local_irq_save_hw(flags); + + /* + * First save the IER then disable the interrupts + */ + ier = serial_in(up, UART_IER); + + if (up->capabilities & UART_CAP_UUE) + serial_out(up, UART_IER, UART_IER_UUE); + else + serial_out(up, UART_IER, 0); + + uart_console_write(&up->port, buf, count, serial8250_console_putchar); + + /* + * Finally, wait for transmitter to become empty + * and restore the IER + */ + wait_for_xmitr(up, BOTH_EMPTY); + serial_out(up, UART_IER, ier); + + local_irq_restore_hw(flags); +} + +#endif + static struct platform_driver serial8250_isa_driver = { .probe = serial8250_probe, .remove = __devexit_p(serial8250_remove), diff --git a/fs/aio.c b/fs/aio.c index d065b2c..2cb992a 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -608,13 +608,16 @@ static void use_mm(struct mm_struct *mm) { struct mm_struct *active_mm; struct task_struct *tsk = current; + unsigned long flags; task_lock(tsk); active_mm = tsk->active_mm; atomic_inc(&mm->mm_count); tsk->mm = mm; + ipipe_mm_switch_protect(flags); tsk->active_mm = mm; - switch_mm(active_mm, mm, tsk); + __switch_mm(active_mm, mm, tsk); + ipipe_mm_switch_unprotect(flags); task_unlock(tsk); mmdrop(active_mm); diff --git a/fs/exec.c b/fs/exec.c index 172ceb6..496af61 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -699,6 +699,7 @@ static int exec_mmap(struct mm_struct *mm) { struct task_struct *tsk; struct mm_struct * old_mm, *active_mm; + unsigned long flags; /* Notify parent that we're no longer interested in the old VM */ tsk = current; @@ -721,8 +722,10 @@ static int exec_mmap(struct mm_struct *mm) task_lock(tsk); active_mm = tsk->active_mm; tsk->mm = mm; + ipipe_mm_switch_protect(flags); tsk->active_mm = mm; activate_mm(active_mm, mm); + ipipe_mm_switch_unprotect(flags); task_unlock(tsk); arch_pick_mmap_layout(mm); if (old_mm) { diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index c99c64d..5d01b93 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -60,11 +60,11 @@ static inline int atomic_add_return(int i, atomic_t *v) unsigned long flags; int temp; - local_irq_save(flags); + local_irq_save_hw(flags); temp = v->counter; temp += i; v->counter = temp; - local_irq_restore(flags); + local_irq_restore_hw(flags); return temp; } @@ -82,11 +82,11 @@ static inline int atomic_sub_return(int i, atomic_t *v) unsigned long flags; int temp; - local_irq_save(flags); + local_irq_save_hw(flags); temp = v->counter; temp -= i; v->counter = temp; - local_irq_restore(flags); + local_irq_restore_hw(flags); return temp; } @@ -139,9 +139,9 @@ static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr) unsigned long flags; mask = ~mask; - local_irq_save(flags); + local_irq_save_hw(flags); *addr &= mask; - local_irq_restore(flags); + local_irq_restore_hw(flags); } #define atomic_xchg(ptr, v) (xchg(&(ptr)->counter, (v))) diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h index c894646..8d42ffe 100644 --- a/include/asm-generic/bitops/atomic.h +++ b/include/asm-generic/bitops/atomic.h @@ -21,20 +21,20 @@ extern raw_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned; * this is the substitute */ #define _atomic_spin_lock_irqsave(l,f) do { \ raw_spinlock_t *s = ATOMIC_HASH(l); \ - local_irq_save(f); \ + local_irq_save_hw(f); \ __raw_spin_lock(s); \ } while(0) #define _atomic_spin_unlock_irqrestore(l,f) do { \ raw_spinlock_t *s = ATOMIC_HASH(l); \ __raw_spin_unlock(s); \ - local_irq_restore(f); \ + local_irq_restore_hw(f); \ } while(0) #else -# define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0) -# define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0) +# define _atomic_spin_lock_irqsave(l,f) do { local_irq_save_hw(f); } while (0) +# define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore_hw(f); } while (0) #endif /* diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h index b2ba2fc..ed01ab9 100644 --- a/include/asm-generic/cmpxchg-local.h +++ b/include/asm-generic/cmpxchg-local.h @@ -20,7 +20,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, if (size == 8 && sizeof(unsigned long) != 8) wrong_size_cmpxchg(ptr); - local_irq_save(flags); + local_irq_save_hw(flags); switch (size) { case 1: prev = *(u8 *)ptr; if (prev == old) @@ -41,7 +41,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, default: wrong_size_cmpxchg(ptr); } - local_irq_restore(flags); + local_irq_restore_hw(flags); return prev; } @@ -54,11 +54,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr, u64 prev; unsigned long flags; - local_irq_save(flags); + local_irq_save_hw(flags); prev = *(u64 *)ptr; if (prev == old) *(u64 *)ptr = new; - local_irq_restore(flags); + local_irq_restore_hw(flags); return prev; } diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index aa00800..886c45f 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -56,6 +56,20 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define __raw_get_cpu_var(var) \ (*SHIFT_PERCPU_PTR(&per_cpu_var(var), __my_cpu_offset)) +#ifdef CONFIG_IPIPE +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) +extern int __ipipe_check_percpu_access(void); +#define __ipipe_local_cpu_offset \ + ({ \ + WARN_ON_ONCE(__ipipe_check_percpu_access()); \ + __my_cpu_offset; \ + }) +#else +#define __ipipe_local_cpu_offset __my_cpu_offset +#endif +#define __ipipe_get_cpu_var(var) \ + (*SHIFT_PERCPU_PTR(&per_cpu_var(var), __ipipe_local_cpu_offset)) +#endif /* CONFIG_IPIPE */ #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas(void); @@ -66,6 +80,7 @@ extern void setup_per_cpu_areas(void); #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu_var(var))) #define __get_cpu_var(var) per_cpu_var(var) #define __raw_get_cpu_var(var) per_cpu_var(var) +#define __ipipe_get_cpu_var(var) __raw_get_cpu_var(var) #endif /* SMP */ diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 8246c69..77b781b 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -177,24 +177,28 @@ extern void irq_enter(void); */ extern void irq_exit(void); -#define nmi_enter() \ - do { \ - ftrace_nmi_enter(); \ - BUG_ON(in_nmi()); \ - add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ - lockdep_off(); \ - rcu_nmi_enter(); \ - trace_hardirq_enter(); \ +#define nmi_enter() \ + do { \ + if (likely(!ipipe_test_foreign_stack())) { \ + ftrace_nmi_enter(); \ + BUG_ON(in_nmi()); \ + add_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ + lockdep_off(); \ + rcu_nmi_enter(); \ + trace_hardirq_enter(); \ + } \ } while (0) -#define nmi_exit() \ - do { \ - trace_hardirq_exit(); \ - rcu_nmi_exit(); \ - lockdep_on(); \ - BUG_ON(!in_nmi()); \ - sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ - ftrace_nmi_exit(); \ +#define nmi_exit() \ + do { \ + if (likely(!ipipe_test_foreign_stack())) { \ + trace_hardirq_exit(); \ + rcu_nmi_exit(); \ + lockdep_on(); \ + BUG_ON(!in_nmi()); \ + sub_preempt_count(NMI_OFFSET + HARDIRQ_OFFSET); \ + ftrace_nmi_exit(); \ + } \ } while (0) #endif /* LINUX_HARDIRQ_H */ diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h new file mode 100644 index 0000000..8cbcb53 --- /dev/null +++ b/include/linux/ipipe.h @@ -0,0 +1,688 @@ +/* -*- linux-c -*- + * include/linux/ipipe.h + * + * Copyright (C) 2002-2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_H +#define __LINUX_IPIPE_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + +#include +#include + +static inline int ipipe_disable_context_check(int cpu) +{ + return xchg(&per_cpu(ipipe_percpu_context_check, cpu), 0); +} + +static inline void ipipe_restore_context_check(int cpu, int old_state) +{ + per_cpu(ipipe_percpu_context_check, cpu) = old_state; +} + +static inline void ipipe_context_check_off(void) +{ + int cpu; + for_each_online_cpu(cpu) + per_cpu(ipipe_percpu_context_check, cpu) = 0; +} + +#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +static inline int ipipe_disable_context_check(int cpu) +{ + return 0; +} + +static inline void ipipe_restore_context_check(int cpu, int old_state) { } + +static inline void ipipe_context_check_off(void) { } + +#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +#ifdef CONFIG_IPIPE + +/* + * Sanity check: IPIPE_VIRQ_BASE depends on CONFIG_NR_CPUS, and if the + * latter gets too large, we fail to map the virtual interrupts. + */ +#if IPIPE_VIRQ_BASE / BITS_PER_LONG > BITS_PER_LONG +#error "CONFIG_NR_CPUS is too large, please lower it." +#endif + +#define IPIPE_VERSION_STRING IPIPE_ARCH_STRING +#define IPIPE_RELEASE_NUMBER ((IPIPE_MAJOR_NUMBER << 16) | \ + (IPIPE_MINOR_NUMBER << 8) | \ + (IPIPE_PATCH_NUMBER)) + +#ifndef BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0)) +#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1)) +#endif /* !BUILTIN_RETURN_ADDRESS */ + +#define IPIPE_ROOT_PRIO 100 +#define IPIPE_ROOT_ID 0 +#define IPIPE_ROOT_NPTDKEYS 4 /* Must be <= BITS_PER_LONG */ + +#define IPIPE_RESET_TIMER 0x1 +#define IPIPE_GRAB_TIMER 0x2 + +/* Global domain flags */ +#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */ +#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */ + +/* Interrupt control bits */ +#define IPIPE_HANDLE_FLAG 0 +#define IPIPE_PASS_FLAG 1 +#define IPIPE_ENABLE_FLAG 2 +#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG +#define IPIPE_STICKY_FLAG 3 +#define IPIPE_SYSTEM_FLAG 4 +#define IPIPE_LOCK_FLAG 5 +#define IPIPE_WIRED_FLAG 6 +#define IPIPE_EXCLUSIVE_FLAG 7 + +#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG) +#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG) +#define IPIPE_ENABLE_MASK (1 << IPIPE_ENABLE_FLAG) +#define IPIPE_DYNAMIC_MASK IPIPE_HANDLE_MASK +#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG) +#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG) +#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG) +#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG) +#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG) + +#define IPIPE_DEFAULT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK) +#define IPIPE_STDROOT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK) + +#define IPIPE_EVENT_SELF 0x80000000 + +#define IPIPE_NR_CPUS NR_CPUS + +/* This accessor assumes hw IRQs are off on SMP; allows assignment. */ +#define __ipipe_current_domain __ipipe_get_cpu_var(ipipe_percpu_domain) +/* This read-only accessor makes sure that hw IRQs are off on SMP. */ +#define ipipe_current_domain \ + ({ \ + struct ipipe_domain *__ipd__; \ + unsigned long __flags__; \ + local_irq_save_hw_smp(__flags__); \ + __ipd__ = __ipipe_current_domain; \ + local_irq_restore_hw_smp(__flags__); \ + __ipd__; \ + }) + +#define ipipe_virtual_irq_p(irq) ((irq) >= IPIPE_VIRQ_BASE && \ + (irq) < IPIPE_NR_IRQS) + +#define IPIPE_SAME_HANDLER ((ipipe_irq_handler_t)(-1)) + +struct irq_desc; + +typedef void (*ipipe_irq_ackfn_t)(unsigned irq, struct irq_desc *desc); + +typedef int (*ipipe_event_handler_t)(unsigned event, + struct ipipe_domain *from, + void *data); +struct ipipe_domain { + + int slot; /* Slot number in percpu domain data array. */ + struct list_head p_link; /* Link in pipeline */ + ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */ + unsigned long long evself; /* Self-monitored event bits. */ + + struct ipipe_irqdesc { + unsigned long control; + ipipe_irq_ackfn_t acknowledge; + ipipe_irq_handler_t handler; + void *cookie; + } ____cacheline_aligned irqs[IPIPE_NR_IRQS]; + + int priority; + void *pdd; + unsigned long flags; + unsigned domid; + const char *name; + struct mutex mutex; +}; + +#define IPIPE_HEAD_PRIORITY (-1) /* For domains always heading the pipeline */ + +struct ipipe_domain_attr { + + unsigned domid; /* Domain identifier -- Magic value set by caller */ + const char *name; /* Domain name -- Warning: won't be dup'ed! */ + int priority; /* Priority in interrupt pipeline */ + void (*entry) (void); /* Domain entry point */ + void *pdd; /* Per-domain (opaque) data pointer */ +}; + +#define __ipipe_irq_cookie(ipd, irq) (ipd)->irqs[irq].cookie +#define __ipipe_irq_handler(ipd, irq) (ipd)->irqs[irq].handler +#define __ipipe_cpudata_irq_hits(ipd, cpu, irq) ipipe_percpudom(ipd, irqall, cpu)[irq] + +extern unsigned __ipipe_printk_virq; + +extern unsigned long __ipipe_virtual_irq_map; + +extern struct list_head __ipipe_pipeline; + +extern int __ipipe_event_monitors[]; + +/* Private interface */ + +void ipipe_init(void); + +#ifdef CONFIG_PROC_FS +void ipipe_init_proc(void); + +#ifdef CONFIG_IPIPE_TRACE +void __ipipe_init_tracer(void); +#else /* !CONFIG_IPIPE_TRACE */ +#define __ipipe_init_tracer() do { } while(0) +#endif /* CONFIG_IPIPE_TRACE */ + +#else /* !CONFIG_PROC_FS */ +#define ipipe_init_proc() do { } while(0) +#endif /* CONFIG_PROC_FS */ + +void __ipipe_init_stage(struct ipipe_domain *ipd); + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd); + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd); + +void __ipipe_flush_printk(unsigned irq, void *cookie); + +void __ipipe_walk_pipeline(struct list_head *pos); + +void __ipipe_pend_irq(unsigned irq, struct list_head *head); + +int __ipipe_dispatch_event(unsigned event, void *data); + +void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq); + +void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq); + +void __ipipe_sync_stage(unsigned long syncmask); + +void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq); + +void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq); + +void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq); + +void __ipipe_pin_range_globally(unsigned long start, unsigned long end); + +/* Must be called hw IRQs off. */ +static inline void ipipe_irq_lock(unsigned irq) +{ + __ipipe_lock_irq(__ipipe_current_domain, ipipe_processor_id(), irq); +} + +/* Must be called hw IRQs off. */ +static inline void ipipe_irq_unlock(unsigned irq) +{ + __ipipe_unlock_irq(__ipipe_current_domain, irq); +} + +#ifndef __ipipe_sync_pipeline +#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask) +#endif + +#ifndef __ipipe_run_irqtail +#define __ipipe_run_irqtail() do { } while(0) +#endif + +#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next) + +/* + * Keep the following as a macro, so that client code could check for + * the support of the invariant pipeline head optimization. + */ +#define __ipipe_pipeline_head() \ + list_entry(__ipipe_pipeline.next, struct ipipe_domain, p_link) + +#define local_irq_enable_hw_cond() local_irq_enable_hw() +#define local_irq_disable_hw_cond() local_irq_disable_hw() +#define local_irq_save_hw_cond(flags) local_irq_save_hw(flags) +#define local_irq_restore_hw_cond(flags) local_irq_restore_hw(flags) + +#ifdef CONFIG_SMP +cpumask_t __ipipe_set_irq_affinity(unsigned irq, cpumask_t cpumask); +int __ipipe_send_ipi(unsigned ipi, cpumask_t cpumask); +#define local_irq_save_hw_smp(flags) local_irq_save_hw(flags) +#define local_irq_restore_hw_smp(flags) local_irq_restore_hw(flags) +#else /* !CONFIG_SMP */ +#define local_irq_save_hw_smp(flags) do { (void)(flags); } while(0) +#define local_irq_restore_hw_smp(flags) do { } while(0) +#endif /* CONFIG_SMP */ + +#define local_irq_save_full(vflags, rflags) \ + do { \ + local_irq_save(vflags); \ + local_irq_save_hw(rflags); \ + } while(0) + +#define local_irq_restore_full(vflags, rflags) \ + do { \ + local_irq_restore_hw(rflags); \ + local_irq_restore(vflags); \ + } while(0) + +static inline void __local_irq_restore_nosync(unsigned long x) +{ + struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr(); + + if (raw_irqs_disabled_flags(x)) + set_bit(IPIPE_STALL_FLAG, &p->status); + else + clear_bit(IPIPE_STALL_FLAG, &p->status); +} + +static inline void local_irq_restore_nosync(unsigned long x) +{ + unsigned long flags; + local_irq_save_hw_smp(flags); + __local_irq_restore_nosync(x); + local_irq_restore_hw_smp(flags); +} + +#define __ipipe_root_domain_p (__ipipe_current_domain == ipipe_root_domain) +#define ipipe_root_domain_p (ipipe_current_domain == ipipe_root_domain) + +static inline int __ipipe_event_monitored_p(int ev) +{ + if (__ipipe_event_monitors[ev] > 0) + return 1; + + return (ipipe_current_domain->evself & (1LL << ev)) != 0; +} + +#define ipipe_sigwake_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE, p); \ +} while(0) + +#define ipipe_exit_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \ + __ipipe_dispatch_event(IPIPE_EVENT_EXIT, p); \ +} while(0) + +#define ipipe_setsched_notify(p) \ +do { \ + if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SETSCHED, p); \ +} while(0) + +#define ipipe_schedule_notify(prev, next) \ +do { \ + if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) && \ + __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE)) \ + __ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next); \ +} while(0) + +#define ipipe_trap_notify(ex, regs) \ +({ \ + unsigned long __flags__; \ + int __ret__ = 0; \ + local_irq_save_hw_smp(__flags__); \ + if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status)) || \ + ((current)->flags & PF_EVNOTIFY)) && \ + __ipipe_event_monitored_p(ex)) { \ + local_irq_restore_hw_smp(__flags__); \ + __ret__ = __ipipe_dispatch_event(ex, regs); \ + } else \ + local_irq_restore_hw_smp(__flags__); \ + __ret__; \ +}) + +static inline void ipipe_init_notify(struct task_struct *p) +{ + if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT)) + __ipipe_dispatch_event(IPIPE_EVENT_INIT, p); +} + +struct mm_struct; + +static inline void ipipe_cleanup_notify(struct mm_struct *mm) +{ + if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP)) + __ipipe_dispatch_event(IPIPE_EVENT_CLEANUP, mm); +} + +/* Public interface */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr); + +int ipipe_unregister_domain(struct ipipe_domain *ipd); + +void ipipe_suspend_domain(void); + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask); + +int ipipe_control_irq(unsigned irq, + unsigned clrmask, + unsigned setmask); + +unsigned ipipe_alloc_virq(void); + +int ipipe_free_virq(unsigned virq); + +int ipipe_trigger_irq(unsigned irq); + +static inline void __ipipe_propagate_irq(unsigned irq) +{ + struct list_head *next = __ipipe_current_domain->p_link.next; + if (next == &ipipe_root.p_link) { + /* Fast path: root must handle all interrupts. */ + __ipipe_set_irq_pending(&ipipe_root, irq); + return; + } + __ipipe_pend_irq(irq, next); +} + +static inline void __ipipe_schedule_irq(unsigned irq) +{ + __ipipe_pend_irq(irq, &__ipipe_current_domain->p_link); +} + +static inline void __ipipe_schedule_irq_head(unsigned irq) +{ + __ipipe_set_irq_pending(__ipipe_pipeline_head(), irq); +} + +static inline void __ipipe_schedule_irq_root(unsigned irq) +{ + __ipipe_set_irq_pending(&ipipe_root, irq); +} + +static inline void ipipe_propagate_irq(unsigned irq) +{ + unsigned long flags; + + local_irq_save_hw(flags); + __ipipe_propagate_irq(irq); + local_irq_restore_hw(flags); +} + +static inline void ipipe_schedule_irq(unsigned irq) +{ + unsigned long flags; + + local_irq_save_hw(flags); + __ipipe_schedule_irq(irq); + local_irq_restore_hw(flags); +} + +static inline void ipipe_schedule_irq_head(unsigned irq) +{ + unsigned long flags; + + local_irq_save_hw(flags); + __ipipe_schedule_irq_head(irq); + local_irq_restore_hw(flags); +} + +static inline void ipipe_schedule_irq_root(unsigned irq) +{ + unsigned long flags; + + local_irq_save_hw(flags); + __ipipe_schedule_irq_root(irq); + local_irq_restore_hw(flags); +} + +void ipipe_stall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd); + +unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd); + +static inline void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + ipipe_test_and_unstall_pipeline_from(ipd); +} + +void ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x); + +static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd) +{ + return test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)); +} + +static inline void ipipe_stall_pipeline_head(void) +{ + local_irq_disable_hw(); + __set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status)); +} + +static inline unsigned long ipipe_test_and_stall_pipeline_head(void) +{ + local_irq_disable_hw(); + return __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status)); +} + +void ipipe_unstall_pipeline_head(void); + +void __ipipe_restore_pipeline_head(unsigned long x); + +static inline void ipipe_restore_pipeline_head(unsigned long x) +{ + /* On some archs, __test_and_set_bit() might return different + * truth value than test_bit(), so we test the exclusive OR of + * both statuses, assuming that the lowest bit is always set in + * the truth value (if this is wrong, the failed optimization will + * be caught in __ipipe_restore_pipeline_head() if + * CONFIG_DEBUG_KERNEL is set). */ + if ((x ^ test_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status))) & 1) + __ipipe_restore_pipeline_head(x); +} + +#define ipipe_unstall_pipeline() \ + ipipe_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_unstall_pipeline() \ + ipipe_test_and_unstall_pipeline_from(ipipe_current_domain) + +#define ipipe_test_pipeline() \ + ipipe_test_pipeline_from(ipipe_current_domain) + +#define ipipe_test_and_stall_pipeline() \ + ipipe_test_and_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_stall_pipeline() \ + ipipe_stall_pipeline_from(ipipe_current_domain) + +#define ipipe_restore_pipeline(x) \ + ipipe_restore_pipeline_from(ipipe_current_domain, (x)) + +void ipipe_init_attr(struct ipipe_domain_attr *attr); + +int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo); + +unsigned long ipipe_critical_enter(void (*syncfn) (void)); + +void ipipe_critical_exit(unsigned long flags); + +static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd) +{ + set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_printk_async(struct ipipe_domain *ipd) +{ + clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags); +} + +static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + __set_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status)); +} + +static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + __clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status)); +} + +static inline int ipipe_test_foreign_stack(void) +{ + /* Must be called hw interrupts off. */ + return test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status)); +} + +#ifndef ipipe_safe_current +#define ipipe_safe_current() \ +({ \ + struct task_struct *p; \ + unsigned long flags; \ + local_irq_save_hw_smp(flags); \ + p = ipipe_test_foreign_stack() ? &init_task : current; \ + local_irq_restore_hw_smp(flags); \ + p; \ +}) +#endif + +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler); + +cpumask_t ipipe_set_irq_affinity(unsigned irq, + cpumask_t cpumask); + +int ipipe_send_ipi(unsigned ipi, + cpumask_t cpumask); + +int ipipe_setscheduler_root(struct task_struct *p, + int policy, + int prio); + +int ipipe_reenter_root(struct task_struct *prev, + int policy, + int prio); + +int ipipe_alloc_ptdkey(void); + +int ipipe_free_ptdkey(int key); + +int ipipe_set_ptd(int key, + void *value); + +void *ipipe_get_ptd(int key); + +int ipipe_disable_ondemand_mappings(struct task_struct *tsk); + +static inline void ipipe_nmi_enter(void) +{ + int cpu = ipipe_processor_id(); + + per_cpu(ipipe_nmi_saved_root, cpu) = ipipe_root_cpudom_var(status); + __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + per_cpu(ipipe_saved_context_check_state, cpu) = + ipipe_disable_context_check(cpu); +#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */ +} + +static inline void ipipe_nmi_exit(void) +{ + int cpu = ipipe_processor_id(); + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + ipipe_restore_context_check + (cpu, per_cpu(ipipe_saved_context_check_state, cpu)); +#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */ + + if (!test_bit(IPIPE_STALL_FLAG, &per_cpu(ipipe_nmi_saved_root, cpu))) + __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)); +} + +#else /* !CONFIG_IPIPE */ + +#define ipipe_init() do { } while(0) +#define ipipe_suspend_domain() do { } while(0) +#define ipipe_sigwake_notify(p) do { } while(0) +#define ipipe_setsched_notify(p) do { } while(0) +#define ipipe_init_notify(p) do { } while(0) +#define ipipe_exit_notify(p) do { } while(0) +#define ipipe_cleanup_notify(mm) do { } while(0) +#define ipipe_trap_notify(t,r) 0 +#define ipipe_init_proc() do { } while(0) + +static inline void __ipipe_pin_range_globally(unsigned long start, + unsigned long end) +{ +} + +static inline int ipipe_test_foreign_stack(void) +{ + return 0; +} + +#define local_irq_enable_hw_cond() do { } while(0) +#define local_irq_disable_hw_cond() do { } while(0) +#define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0) +#define local_irq_restore_hw_cond(flags) do { } while(0) +#define local_irq_save_hw_smp(flags) do { (void)(flags); } while(0) +#define local_irq_restore_hw_smp(flags) do { } while(0) + +#define ipipe_irq_lock(irq) do { } while(0) +#define ipipe_irq_unlock(irq) do { } while(0) + +#define __ipipe_root_domain_p 1 +#define ipipe_root_domain_p 1 +#define ipipe_safe_current current +#define ipipe_processor_id() smp_processor_id() + +#define ipipe_nmi_enter() do { } while (0) +#define ipipe_nmi_exit() do { } while (0) + +#define local_irq_disable_head() local_irq_disable() + +#define local_irq_save_full(vflags, rflags) do { (void)(vflags); local_irq_save(rflags); } while(0) +#define local_irq_restore_full(vflags, rflags) do { (void)(vflags); local_irq_restore(rflags); } while(0) +#define local_irq_restore_nosync(vflags) local_irq_restore(vflags) + +#endif /* CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_H */ diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h new file mode 100644 index 0000000..ab2c970 --- /dev/null +++ b/include/linux/ipipe_base.h @@ -0,0 +1,103 @@ +/* -*- linux-c -*- + * include/linux/ipipe_base.h + * + * Copyright (C) 2002-2007 Philippe Gerum. + * 2007 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_BASE_H +#define __LINUX_IPIPE_BASE_H + +#ifdef CONFIG_IPIPE + +#include + +/* Number of virtual IRQs */ +#define IPIPE_NR_VIRQS BITS_PER_LONG +/* First virtual IRQ # */ +#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG) +/* Total number of IRQ slots */ +#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS) +/* Number of indirect words needed to map the whole IRQ space. */ +#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) +#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1) +#define IPIPE_IRQMASK_ANY (~0L) +#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG)) + +/* Per-cpu pipeline status */ +#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage -- guaranteed at bit #0 */ +#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is running for the domain */ +#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign stack */ + +#define IPIPE_STALL_MASK (1L << IPIPE_STALL_FLAG) +#define IPIPE_SYNC_MASK (1L << IPIPE_SYNC_FLAG) +#define IPIPE_NOSTACK_MASK (1L << IPIPE_NOSTACK_FLAG) + +typedef void (*ipipe_irq_handler_t)(unsigned irq, + void *cookie); + +extern struct ipipe_domain ipipe_root; + +#define ipipe_root_domain (&ipipe_root) + +void __ipipe_unstall_root(void); + +void __ipipe_restore_root(unsigned long x); + +#define ipipe_preempt_disable(flags) \ + do { \ + local_irq_save_hw(flags); \ + if (__ipipe_root_domain_p) \ + preempt_disable(); \ + } while (0) + +#define ipipe_preempt_enable(flags) \ + do { \ + if (__ipipe_root_domain_p) { \ + preempt_enable_no_resched(); \ + local_irq_restore_hw(flags); \ + preempt_check_resched(); \ + } else \ + local_irq_restore_hw(flags); \ + } while (0) + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT +void ipipe_check_context(struct ipipe_domain *border_ipd); +#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */ +static inline void ipipe_check_context(struct ipipe_domain *border_ipd) { } +#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +/* Generic features */ + +#ifdef CONFIG_GENERIC_CLOCKEVENTS +#define __IPIPE_FEATURE_REQUEST_TICKDEV 1 +#endif +#define __IPIPE_FEATURE_DELAYED_ATOMICSW 1 +#define __IPIPE_FEATURE_FASTPEND_IRQ 1 +#define __IPIPE_FEATURE_TRACE_EVENT 1 + +#else /* !CONFIG_IPIPE */ +#define ipipe_preempt_disable(flags) do { \ + preempt_disable(); \ + (void)(flags); \ + } while (0) +#define ipipe_preempt_enable(flags) preempt_enable() +#define ipipe_check_context(ipd) do { } while(0) +#endif /* CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_BASE_H */ diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h new file mode 100644 index 0000000..50a245c --- /dev/null +++ b/include/linux/ipipe_compat.h @@ -0,0 +1,54 @@ +/* -*- linux-c -*- + * include/linux/ipipe_compat.h + * + * Copyright (C) 2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_COMPAT_H +#define __LINUX_IPIPE_COMPAT_H + +#ifdef CONFIG_IPIPE_COMPAT +/* + * OBSOLETE: defined only for backward compatibility. Will be removed + * in future releases, please update client code accordingly. + */ + +#ifdef CONFIG_SMP +#define ipipe_declare_cpuid int cpuid +#define ipipe_load_cpuid() do { \ + cpuid = ipipe_processor_id(); \ + } while(0) +#define ipipe_lock_cpu(flags) do { \ + local_irq_save_hw(flags); \ + cpuid = ipipe_processor_id(); \ + } while(0) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags) +#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags) +#else /* !CONFIG_SMP */ +#define ipipe_declare_cpuid const int cpuid = 0 +#define ipipe_load_cpuid() do { } while(0) +#define ipipe_lock_cpu(flags) local_irq_save_hw(flags) +#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags) +#define ipipe_get_cpu(flags) do { (void)(flags); } while(0) +#define ipipe_put_cpu(flags) do { } while(0) +#endif /* CONFIG_SMP */ + +#endif /* CONFIG_IPIPE_COMPAT */ + +#endif /* !__LINUX_IPIPE_COMPAT_H */ diff --git a/include/linux/ipipe_percpu.h b/include/linux/ipipe_percpu.h new file mode 100644 index 0000000..4d83119 --- /dev/null +++ b/include/linux/ipipe_percpu.h @@ -0,0 +1,86 @@ +/* -*- linux-c -*- + * include/linux/ipipe_percpu.h + * + * Copyright (C) 2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_PERCPU_H +#define __LINUX_IPIPE_PERCPU_H + +#include +#include + +struct ipipe_domain; + +struct ipipe_percpu_domain_data { + unsigned long status; /* <= Must be first in struct. */ + unsigned long irqpend_himask; + unsigned long irqpend_lomask[IPIPE_IRQ_IWORDS]; + unsigned long irqheld_mask[IPIPE_IRQ_IWORDS]; + unsigned long irqall[IPIPE_NR_IRQS]; + u64 evsync; +}; + +/* + * CAREFUL: all accessors based on __raw_get_cpu_var() you may find in + * this file should be used only while hw interrupts are off, to + * prevent from CPU migration regardless of the running domain. + */ +#ifdef CONFIG_SMP +#define ipipe_percpudom_ptr(ipd, cpu) \ + (&per_cpu(ipipe_percpu_darray, cpu)[(ipd)->slot]) +#define ipipe_cpudom_ptr(ipd) \ + (&__ipipe_get_cpu_var(ipipe_percpu_darray)[(ipd)->slot]) +#else +DECLARE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]); +#define ipipe_percpudom_ptr(ipd, cpu) \ + (per_cpu(ipipe_percpu_daddr, cpu)[(ipd)->slot]) +#define ipipe_cpudom_ptr(ipd) \ + (__ipipe_get_cpu_var(ipipe_percpu_daddr)[(ipd)->slot]) +#endif +#define ipipe_percpudom(ipd, var, cpu) (ipipe_percpudom_ptr(ipd, cpu)->var) +#define ipipe_cpudom_var(ipd, var) (ipipe_cpudom_ptr(ipd)->var) + +#define IPIPE_ROOT_SLOT 0 +#define IPIPE_HEAD_SLOT (CONFIG_IPIPE_DOMAINS - 1) + +DECLARE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]); + +DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain); + +DECLARE_PER_CPU(unsigned long, ipipe_nmi_saved_root); + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT +DECLARE_PER_CPU(int, ipipe_percpu_context_check); +DECLARE_PER_CPU(int, ipipe_saved_context_check_state); +#endif + +#define ipipe_root_cpudom_ptr(var) \ + (&__ipipe_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT]) + +#define ipipe_root_cpudom_var(var) ipipe_root_cpudom_ptr()->var + +#define ipipe_this_cpudom_var(var) \ + ipipe_cpudom_var(__ipipe_current_domain, var) + +#define ipipe_head_cpudom_ptr() \ + (&__ipipe_get_cpu_var(ipipe_percpu_darray)[IPIPE_HEAD_SLOT]) + +#define ipipe_head_cpudom_var(var) ipipe_head_cpudom_ptr()->var + +#endif /* !__LINUX_IPIPE_PERCPU_H */ diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h new file mode 100644 index 0000000..4a1cb1b --- /dev/null +++ b/include/linux/ipipe_tickdev.h @@ -0,0 +1,58 @@ +/* -*- linux-c -*- + * include/linux/ipipe_tickdev.h + * + * Copyright (C) 2007 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_TICKDEV_H +#define __LINUX_IPIPE_TICKDEV_H + +#if defined(CONFIG_IPIPE) && defined(CONFIG_GENERIC_CLOCKEVENTS) + +#include + +struct tick_device; + +struct ipipe_tick_device { + + void (*emul_set_mode)(enum clock_event_mode, + struct clock_event_device *cdev); + int (*emul_set_tick)(unsigned long delta, + struct clock_event_device *cdev); + void (*real_set_mode)(enum clock_event_mode mode, + struct clock_event_device *cdev); + int (*real_set_tick)(unsigned long delta, + struct clock_event_device *cdev); + struct tick_device *slave; + unsigned long real_max_delta_ns; + unsigned long real_mult; + int real_shift; +}; + +int ipipe_request_tickdev(const char *devname, + void (*emumode)(enum clock_event_mode mode, + struct clock_event_device *cdev), + int (*emutick)(unsigned long evt, + struct clock_event_device *cdev), + int cpu, unsigned long *tmfreq); + +void ipipe_release_tickdev(int cpu); + +#endif /* CONFIG_IPIPE && CONFIG_GENERIC_CLOCKEVENTS */ + +#endif /* !__LINUX_IPIPE_TICKDEV_H */ diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h new file mode 100644 index 0000000..627b354 --- /dev/null +++ b/include/linux/ipipe_trace.h @@ -0,0 +1,72 @@ +/* -*- linux-c -*- + * include/linux/ipipe_trace.h + * + * Copyright (C) 2005 Luotao Fu. + * 2005-2007 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef _LINUX_IPIPE_TRACE_H +#define _LINUX_IPIPE_TRACE_H + +#ifdef CONFIG_IPIPE_TRACE + +#include + +void ipipe_trace_begin(unsigned long v); +void ipipe_trace_end(unsigned long v); +void ipipe_trace_freeze(unsigned long v); +void ipipe_trace_special(unsigned char special_id, unsigned long v); +void ipipe_trace_pid(pid_t pid, short prio); +void ipipe_trace_event(unsigned char id, unsigned long delay_tsc); +int ipipe_trace_max_reset(void); +int ipipe_trace_frozen_reset(void); + +#else /* !CONFIG_IPIPE_TRACE */ + +#define ipipe_trace_begin(v) do { (void)(v); } while(0) +#define ipipe_trace_end(v) do { (void)(v); } while(0) +#define ipipe_trace_freeze(v) do { (void)(v); } while(0) +#define ipipe_trace_special(id, v) do { (void)(id); (void)(v); } while(0) +#define ipipe_trace_pid(pid, prio) do { (void)(pid); (void)(prio); } while(0) +#define ipipe_trace_event(id, delay_tsc) do { (void)(id); (void)(delay_tsc); } while(0) +#define ipipe_trace_max_reset() do { } while(0) +#define ipipe_trace_froze_reset() do { } while(0) + +#endif /* !CONFIG_IPIPE_TRACE */ + +#ifdef CONFIG_IPIPE_TRACE_PANIC +void ipipe_trace_panic_freeze(void); +void ipipe_trace_panic_dump(void); +#else +static inline void ipipe_trace_panic_freeze(void) { } +static inline void ipipe_trace_panic_dump(void) { } +#endif + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF +#define ipipe_trace_irq_entry(irq) ipipe_trace_begin(irq) +#define ipipe_trace_irq_exit(irq) ipipe_trace_end(irq) +#define ipipe_trace_irqsoff() ipipe_trace_begin(0x80000000UL) +#define ipipe_trace_irqson() ipipe_trace_end(0x80000000UL) +#else +#define ipipe_trace_irq_entry(irq) do { (void)(irq);} while(0) +#define ipipe_trace_irq_exit(irq) do { (void)(irq);} while(0) +#define ipipe_trace_irqsoff() do { } while(0) +#define ipipe_trace_irqson() do { } while(0) +#endif + +#endif /* !__LINUX_IPIPE_TRACE_H */ diff --git a/include/linux/irq.h b/include/linux/irq.h index cb2e77a..a00a77e 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -119,6 +119,9 @@ struct irq_chip { void (*end)(unsigned int irq); int (*set_affinity)(unsigned int irq, const struct cpumask *dest); +#ifdef CONFIG_IPIPE + void (*move)(unsigned int irq); +#endif /* CONFIG_IPIPE */ int (*retrigger)(unsigned int irq); int (*set_type)(unsigned int irq, unsigned int flow_type); int (*set_wake)(unsigned int irq, unsigned int on); @@ -165,6 +168,12 @@ struct irq_2_iommu; * @name: flow handler name for /proc/interrupts output */ struct irq_desc { +#ifdef CONFIG_IPIPE + void (*ipipe_ack)(unsigned int irq, + struct irq_desc *desc); + void (*ipipe_end)(unsigned int irq, + struct irq_desc *desc); +#endif /* CONFIG_IPIPE */ unsigned int irq; struct timer_rand_state *timer_rand_state; unsigned int *kstat_irqs; @@ -345,6 +354,10 @@ set_irq_chip_and_handler_name(unsigned int irq, struct irq_chip *chip, irq_flow_handler_t handle, const char *name); extern void +___set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, + const char *name); + +extern void __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, const char *name); @@ -359,6 +372,15 @@ static inline void __set_irq_handler_unlocked(int irq, } /* + * Same, but without holding the descriptor lock. + */ +static inline void +_set_irq_handler(unsigned int irq, irq_flow_handler_t handle) +{ + ___set_irq_handler(irq, handle, 0, NULL); +} + +/* * Set a highlevel flow handler for a given IRQ: */ static inline void diff --git a/include/linux/kernel.h b/include/linux/kernel.h index d6320a3..48a27a6 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -119,9 +120,12 @@ struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY extern int _cond_resched(void); -# define might_resched() _cond_resched() +# define might_resched() do { \ + ipipe_check_context(ipipe_root_domain); \ + _cond_resched(); \ + } while (0) #else -# define might_resched() do { } while (0) +# define might_resched() ipipe_check_context(ipipe_root_domain) #endif #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index b25d1b5..f026c11 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -379,7 +379,7 @@ do { \ #endif /* CONFIG_LOCK_STAT */ -#ifdef CONFIG_LOCKDEP +#if defined (CONFIG_LOCKDEP) || defined(CONFIG_IPIPE) /* * On lockdep we dont want the hand-coded irq-enable of diff --git a/include/linux/mm.h b/include/linux/mm.h index 9a72cc7..ba2aded 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -104,6 +104,8 @@ extern unsigned int kobjsize(const void *objp); #define VM_SAO 0x20000000 /* Strong Access Ordering (powerpc) */ #define VM_PFN_AT_MMAP 0x40000000 /* PFNMAP vma that is fully mapped at mmap time */ +#define VM_PINNED 0x80000000 /* Disable faults for the vma */ + #ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS #endif diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 72b1a10..80553be 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -9,13 +9,20 @@ #include #include #include +#include #if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER) extern void add_preempt_count(int val); extern void sub_preempt_count(int val); #else -# define add_preempt_count(val) do { preempt_count() += (val); } while (0) -# define sub_preempt_count(val) do { preempt_count() -= (val); } while (0) +# define add_preempt_count(val) do { \ + ipipe_check_context(ipipe_root_domain); \ + preempt_count() += (val); \ + } while (0) +# define sub_preempt_count(val) do { \ + ipipe_check_context(ipipe_root_domain); \ + preempt_count() -= (val); \ + } while (0) #endif #define inc_preempt_count() add_preempt_count(1) diff --git a/include/linux/sched.h b/include/linux/sched.h index 0f1ea4a..6913503 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -59,6 +59,7 @@ struct sched_param { #include #include #include +#include #include #include @@ -188,6 +189,13 @@ extern unsigned long long time_sync_thresh; /* in tsk->state again */ #define TASK_DEAD 64 #define TASK_WAKEKILL 128 +#ifdef CONFIG_IPIPE +#define TASK_ATOMICSWITCH 512 +#define TASK_NOWAKEUP 1024 +#else /* !CONFIG_IPIPE */ +#define TASK_ATOMICSWITCH 0 +#define TASK_NOWAKEUP 0 +#endif /* CONFIG_IPIPE */ /* Convenience macros for the sake of set_task_state */ #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) @@ -295,6 +303,15 @@ extern void trap_init(void); extern void update_process_times(int user); extern void scheduler_tick(void); +#ifdef CONFIG_IPIPE +void update_root_process_times(struct pt_regs *regs); +#else /* !CONFIG_IPIPE */ +static inline void update_root_process_times(struct pt_regs *regs) +{ + update_process_times(user_mode(regs)); +} +#endif /* CONFIG_IPIPE */ + extern void sched_show_task(struct task_struct *p); #ifdef CONFIG_DETECT_SOFTLOCKUP @@ -342,8 +359,8 @@ extern signed long schedule_timeout(signed long timeout); extern signed long schedule_timeout_interruptible(signed long timeout); extern signed long schedule_timeout_killable(signed long timeout); extern signed long schedule_timeout_uninterruptible(signed long timeout); -asmlinkage void __schedule(void); -asmlinkage void schedule(void); +asmlinkage int __schedule(void); +asmlinkage int schedule(void); extern int mutex_spin_on_owner(struct mutex *lock, struct thread_info *owner); struct nsproxy; @@ -1435,6 +1452,9 @@ struct task_struct { #endif atomic_t fs_excl; /* holding fs exclusive resources */ struct rcu_head rcu; +#ifdef CONFIG_IPIPE + void *ptd[IPIPE_ROOT_NPTDKEYS]; +#endif /* * cache last used pipe for splice @@ -1674,6 +1694,11 @@ extern cputime_t task_gtime(struct task_struct *p); #define PF_EXITING 0x00000004 /* getting shut down */ #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ +#ifdef CONFIG_IPIPE +#define PF_EVNOTIFY 0x00000020 /* Notify other domains about internal events */ +#else +#define PF_EVNOTIFY 0 +#endif /* CONFIG_IPIPE */ #define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */ #define PF_SUPERPRIV 0x00000100 /* used super-user privileges */ #define PF_DUMPCORE 0x00000200 /* dumped core */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 4be57ab..f74139a 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -90,10 +90,14 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock); # include #endif +#undef TYPE_EQUAL +#define TYPE_EQUAL(lock, type) \ + __builtin_types_compatible_p(typeof(lock), type *) + #ifdef CONFIG_DEBUG_SPINLOCK extern void __spin_lock_init(spinlock_t *lock, const char *name, struct lock_class_key *key); -# define spin_lock_init(lock) \ +# define _spin_lock_init(lock) \ do { \ static struct lock_class_key __key; \ \ @@ -101,10 +105,21 @@ do { \ } while (0) #else -# define spin_lock_init(lock) \ +# define _spin_lock_init(lock) \ do { *(lock) = SPIN_LOCK_UNLOCKED; } while (0) #endif +# define spin_lock_init(lock) \ + do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \ + do { \ + IPIPE_DEFINE_SPINLOCK(__lock__); \ + *((ipipe_spinlock_t *)lock) = __lock__; \ + } while(0); \ + else \ + _spin_lock_init((spinlock_t *)lock); \ + } while(0) + #ifdef CONFIG_DEBUG_SPINLOCK extern void __rwlock_init(rwlock_t *lock, const char *name, struct lock_class_key *key); @@ -195,7 +210,94 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } #define read_trylock(lock) __cond_lock(lock, _read_trylock(lock)) #define write_trylock(lock) __cond_lock(lock, _write_trylock(lock)) -#define spin_lock(lock) _spin_lock(lock) +#define PICK_SPINOP(op, lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \ + __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin##op((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINOP_RAW(op, lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \ + __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + else if (TYPE_EQUAL(lock, spinlock_t)) \ + __raw_spin##op(&((spinlock_t *)(lock))->raw_lock); \ +} while (0) + +#define PICK_SPINLOCK_IRQ(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_lock_irq((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINUNLOCK_IRQ(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_unlock_irq((spinlock_t *)(lock)); \ +} while (0) + +#define PICK_SPINLOCK_IRQ_RAW(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + local_irq_disable(); \ + __raw_spin_lock(&((spinlock_t *)(lock))->raw_lock); \ +} while (0) + +#define PICK_SPINUNLOCK_IRQ_RAW(lock) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + __raw_spin_unlock(&((spinlock_t *)(lock))->raw_lock); \ + local_irq_enable(); \ +} while (0) + +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +extern int __bad_spinlock_type(void); + +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + flags = _spin_lock_irqsave((spinlock_t *)(lock)); \ + else __bad_spinlock_type(); \ +} while (0) +#define PICK_SPINLOCK_IRQSAVE_NESTED(lock, flags, subclass) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + flags = _spin_lock_irqsave_nested((spinlock_t *)(lock), subclass); \ + else __bad_spinlock_type(); \ +} while (0) +#else +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ +do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_lock_irqsave((spinlock_t *)(lock), flags); \ +} while (0) +#endif + +#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags) \ + do { \ + if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \ + __ipipe_spin_unlock_irqrestore(&((__ipipe_spinlock_t *)(lock))->__raw_lock, flags); \ + } else if (TYPE_EQUAL(lock, spinlock_t)) \ + _spin_unlock_irqrestore((spinlock_t *)(lock), flags); \ +} while (0) + +#define spin_lock(lock) PICK_SPINOP(_lock, lock) #ifdef CONFIG_DEBUG_LOCK_ALLOC # define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass) @@ -217,7 +319,7 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ - flags = _spin_lock_irqsave(lock); \ + PICK_SPINLOCK_IRQSAVE(lock, flags); \ } while (0) #define read_lock_irqsave(lock, flags) \ do { \ @@ -234,13 +336,13 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } #define spin_lock_irqsave_nested(lock, flags, subclass) \ do { \ typecheck(unsigned long, flags); \ - flags = _spin_lock_irqsave_nested(lock, subclass); \ + PICK_SPINLOCK_IRQSAVE_NESTED(lock, flags, subclass); \ } while (0) #else #define spin_lock_irqsave_nested(lock, flags, subclass) \ do { \ typecheck(unsigned long, flags); \ - flags = _spin_lock_irqsave(lock); \ + PICK_SPINLOCK_IRQSAVE(lock, flags); \ } while (0) #endif @@ -249,7 +351,7 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ - _spin_lock_irqsave(lock, flags); \ + PICK_SPINLOCK_IRQSAVE(lock, flags); \ } while (0) #define read_lock_irqsave(lock, flags) \ do { \ @@ -266,7 +368,7 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } #endif -#define spin_lock_irq(lock) _spin_lock_irq(lock) +#define spin_lock_irq(lock) PICK_SPINLOCK_IRQ(lock) #define spin_lock_bh(lock) _spin_lock_bh(lock) #define read_lock_irq(lock) _read_lock_irq(lock) @@ -280,32 +382,40 @@ static inline void smp_mb__after_lock(void) { smp_mb(); } */ #if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \ !defined(CONFIG_SMP) -# define spin_unlock(lock) _spin_unlock(lock) +#define spin_unlock(lock) PICK_SPINOP(_unlock, lock) # define read_unlock(lock) _read_unlock(lock) # define write_unlock(lock) _write_unlock(lock) -# define spin_unlock_irq(lock) _spin_unlock_irq(lock) -# define read_unlock_irq(lock) _read_unlock_irq(lock) -# define write_unlock_irq(lock) _write_unlock_irq(lock) +# define spin_unlock_irq(lock) PICK_SPINUNLOCK_IRQ(lock) +# define read_unlock_irq(lock) _read_unlock_irq(lock) +# define write_unlock_irq(lock) _write_unlock_irq(lock) #else -# define spin_unlock(lock) \ - do {__raw_spin_unlock(&(lock)->raw_lock); __release(lock); } while (0) -# define read_unlock(lock) \ - do {__raw_read_unlock(&(lock)->raw_lock); __release(lock); } while (0) -# define write_unlock(lock) \ - do {__raw_write_unlock(&(lock)->raw_lock); __release(lock); } while (0) -# define spin_unlock_irq(lock) \ +# define spin_unlock(lock) \ do { \ - __raw_spin_unlock(&(lock)->raw_lock); \ + PICK_SPINOP_RAW(_unlock, lock); \ + __release(lock); \ +} while(0) +# define read_unlock(lock) \ +do { \ + __raw_read_unlock(&(lock)->raw_lock); \ __release(lock); \ - local_irq_enable(); \ } while (0) -# define read_unlock_irq(lock) \ +# define write_unlock(lock) \ +do { \ + __raw_write_unlock(&(lock)->raw_lock); \ + __release(lock); \ +} while (0) +# define spin_unlock_irq(lock) \ +do { \ + PICK_SPINUNLOCK_IRQ_RAW(lock); \ + __release(lock); \ +} while(0) +# define read_unlock_irq(lock) \ do { \ __raw_read_unlock(&(lock)->raw_lock); \ __release(lock); \ local_irq_enable(); \ } while (0) -# define write_unlock_irq(lock) \ +# define write_unlock_irq(lock) \ do { \ __raw_write_unlock(&(lock)->raw_lock); \ __release(lock); \ @@ -316,7 +426,7 @@ do { \ #define spin_unlock_irqrestore(lock, flags) \ do { \ typecheck(unsigned long, flags); \ - _spin_unlock_irqrestore(lock, flags); \ + PICK_SPINUNLOCK_IRQRESTORE(lock, flags); \ } while (0) #define spin_unlock_bh(lock) _spin_unlock_bh(lock) @@ -380,4 +490,29 @@ extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock); */ #define spin_can_lock(lock) (!spin_is_locked(lock)) +#ifdef CONFIG_IPIPE +void __ipipe_spin_lock_irq(raw_spinlock_t *lock); +void __ipipe_spin_unlock_irq(raw_spinlock_t *lock); +unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock); +void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock, + unsigned long x); +void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock); +void __ipipe_spin_unlock_irqcomplete(unsigned long x); +#define spin_lock_irqsave_cond(lock, flags) \ + spin_lock_irqsave(lock, flags) +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock_irqrestore(lock, flags) +#else +#define spin_lock_irqsave_cond(lock, flags) \ + do { (void)(flags); spin_lock(lock); } while(0) +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock(lock) +#define __ipipe_spin_lock_irq(lock) do { } while(0) +#define __ipipe_spin_unlock_irq(lock) do { } while(0) +#define __ipipe_spin_lock_irqsave(lock) 0 +#define __ipipe_spin_unlock_irqrestore(lock, x) do { (void)(x); } while(0) +#define __ipipe_spin_unlock_irqbegin(lock) do { } while(0) +#define __ipipe_spin_unlock_irqcomplete(x) do { (void)(x); } while(0) +#endif + #endif /* __LINUX_SPINLOCK_H */ diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h index 68d88f7..f5ce3c4 100644 --- a/include/linux/spinlock_types.h +++ b/include/linux/spinlock_types.h @@ -31,6 +31,10 @@ typedef struct { #endif } spinlock_t; +typedef struct { + raw_spinlock_t __raw_lock; +} __ipipe_spinlock_t; + #define SPINLOCK_MAGIC 0xdead4ead typedef struct { @@ -92,9 +96,21 @@ typedef struct { * __SPIN_LOCK_UNLOCKED()/__RW_LOCK_UNLOCKED() as appropriate. */ #define SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(old_style_spin_init) +#define IPIPE_SPIN_LOCK_UNLOCKED \ + (__ipipe_spinlock_t) { .__raw_lock = __RAW_SPIN_LOCK_UNLOCKED } #define RW_LOCK_UNLOCKED __RW_LOCK_UNLOCKED(old_style_rw_init) #define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) #define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) +#ifdef CONFIG_IPIPE +# define ipipe_spinlock_t __ipipe_spinlock_t +# define IPIPE_DEFINE_SPINLOCK(x) ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED +# define IPIPE_DECLARE_SPINLOCK(x) extern ipipe_spinlock_t x +#else +# define ipipe_spinlock_t spinlock_t +# define IPIPE_DEFINE_SPINLOCK(x) DEFINE_SPINLOCK(x) +# define IPIPE_DECLARE_SPINLOCK(x) extern spinlock_t x +#endif + #endif /* __LINUX_SPINLOCK_TYPES_H */ diff --git a/init/Kconfig b/init/Kconfig index 3f7e609..987f885 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -78,6 +78,7 @@ config INIT_ENV_ARG_LIMIT config LOCALVERSION string "Local version - append to kernel release" + default "-ipipe" help Append an extra string to the end of your kernel version. This will show up when you type uname, for example. diff --git a/init/main.c b/init/main.c index 11f4f14..30f3a70 100644 --- a/init/main.c +++ b/init/main.c @@ -568,7 +568,7 @@ asmlinkage void __init start_kernel(void) cgroup_init_early(); - local_irq_disable(); + local_irq_disable_hw(); early_boot_irqs_off(); early_init_irq_lock_class(); @@ -632,6 +632,11 @@ asmlinkage void __init start_kernel(void) timekeeping_init(); time_init(); sched_clock_init(); + /* + * We need to wait for the interrupt and time subsystems to be + * initialized before enabling the pipeline. + */ + ipipe_init(); profile_init(); if (!irqs_disabled()) printk(KERN_CRIT "start_kernel(): bug: interrupts were " @@ -811,6 +816,7 @@ static void __init do_basic_setup(void) usermodehelper_init(); driver_init(); init_irq_proc(); + ipipe_init_proc(); do_ctors(); do_initcalls(); } diff --git a/kernel/Makefile b/kernel/Makefile index 2093a69..86416f2 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -86,6 +86,7 @@ obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o obj-$(CONFIG_PREEMPT_RCU_TRACE) += rcupreempt_trace.o obj-$(CONFIG_RELAY) += relay.o +obj-$(CONFIG_IPIPE) += ipipe/ obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o diff --git a/kernel/exit.c b/kernel/exit.c index 869dc22..1c5d62c 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -959,6 +959,7 @@ NORET_TYPE void do_exit(long code) acct_process(); trace_sched_process_exit(tsk); + ipipe_exit_notify(tsk); exit_sem(tsk); exit_files(tsk); exit_fs(tsk); diff --git a/kernel/fork.c b/kernel/fork.c index e6c04d4..1cd6e8e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -487,6 +487,7 @@ void mmput(struct mm_struct *mm) if (atomic_dec_and_test(&mm->mm_users)) { exit_aio(mm); exit_mmap(mm); + ipipe_cleanup_notify(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { spin_lock(&mmlist_lock); @@ -878,7 +879,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p) { unsigned long new_flags = p->flags; - new_flags &= ~PF_SUPERPRIV; + new_flags &= ~(PF_SUPERPRIV | PF_EVNOTIFY); new_flags |= PF_FORKNOEXEC; new_flags |= PF_STARTING; p->flags = new_flags; @@ -1257,6 +1258,9 @@ static struct task_struct *copy_process(unsigned long clone_flags, write_unlock_irq(&tasklist_lock); proc_fork_connector(p); cgroup_post_fork(p); +#ifdef CONFIG_IPIPE + memset(p->ptd, 0, sizeof(p->ptd)); +#endif /* CONFIG_IPIPE */ perf_counter_fork(p); return p; @@ -1658,11 +1662,14 @@ SYSCALL_DEFINE1(unshare, unsigned long, unshare_flags) } if (new_mm) { + unsigned long flags; mm = current->mm; active_mm = current->active_mm; current->mm = new_mm; + ipipe_mm_switch_protect(flags); current->active_mm = new_mm; activate_mm(active_mm, new_mm); + ipipe_mm_switch_unprotect(flags); new_mm = mm; } diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig new file mode 100644 index 0000000..de5e6a3 --- /dev/null +++ b/kernel/ipipe/Kconfig @@ -0,0 +1,35 @@ +config IPIPE + bool "Interrupt pipeline" + default y + ---help--- + Activate this option if you want the interrupt pipeline to be + compiled in. + +config IPIPE_DOMAINS + int "Max domains" + depends on IPIPE + default 4 + ---help--- + The maximum number of I-pipe domains to run concurrently. + +config IPIPE_COMPAT + bool "Maintain code compatibility with older releases" + depends on IPIPE + default y + ---help--- + Activate this option if you want the compatibility code to be + defined, so that older I-pipe clients may use obsolete + constructs. WARNING: obsolete code will be eventually + deprecated in future I-pipe releases, and removed from the + compatibility support as time passes. Please fix I-pipe + clients to get rid of such uses as soon as possible. + +config IPIPE_DELAYED_ATOMICSW + bool + depends on IPIPE + default n + +config IPIPE_UNMASKED_CONTEXT_SWITCH + bool + depends on IPIPE + default n diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug new file mode 100644 index 0000000..629c894 --- /dev/null +++ b/kernel/ipipe/Kconfig.debug @@ -0,0 +1,97 @@ +config IPIPE_DEBUG + bool "I-pipe debugging" + depends on IPIPE + +config IPIPE_DEBUG_CONTEXT + bool "Check for illicit cross-domain calls" + depends on IPIPE_DEBUG + default y + ---help--- + Enable this feature to arm checkpoints in the kernel that + verify the correct invocation context. On entry of critical + Linux services a warning is issued if the caller is not + running over the root domain. + +config IPIPE_DEBUG_INTERNAL + bool "Enable internal debug checks" + depends on IPIPE_DEBUG + default y + ---help--- + When this feature is enabled, I-pipe will perform internal + consistency checks of its subsystems, e.g. on per-cpu variable + access. + +config IPIPE_TRACE + bool "Latency tracing" + depends on IPIPE_DEBUG + select FRAME_POINTER + select KALLSYMS + select PROC_FS + ---help--- + Activate this option if you want to use per-function tracing of + the kernel. The tracer will collect data via instrumentation + features like the one below or with the help of explicite calls + of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the + in-kernel tracing API. The collected data and runtime control + is available via /proc/ipipe/trace/*. + +if IPIPE_TRACE + +config IPIPE_TRACE_ENABLE + bool "Enable tracing on boot" + default y + ---help--- + Disable this option if you want to arm the tracer after booting + manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce + boot time on slow embedded devices due to the tracer overhead. + +config IPIPE_TRACE_MCOUNT + bool "Instrument function entries" + default y + select FUNCTION_TRACER + select TRACING + select CONTEXT_SWITCH_TRACER + select FTRACE_MCOUNT_RECORD + select DYNAMIC_FTRACE + ---help--- + When enabled, records every kernel function entry in the tracer + log. While this slows down the system noticeably, it provides + the highest level of information about the flow of events. + However, it can be switch off in order to record only explicit + I-pipe trace points. + +config IPIPE_TRACE_IRQSOFF + bool "Trace IRQs-off times" + default y + ---help--- + Activate this option if I-pipe shall trace the longest path + with hard-IRQs switched off. + +config IPIPE_TRACE_SHIFT + int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)" + range 10 18 + default 14 + ---help--- + The number of trace points to hold tracing data for each + trace path, as a power of 2. + +config IPIPE_TRACE_VMALLOC + bool "Use vmalloc'ed trace buffer" + default y if EMBEDDED + ---help--- + Instead of reserving static kernel data, the required buffer + is allocated via vmalloc during boot-up when this option is + enabled. This can help to start systems that are low on memory, + but it slightly degrades overall performance. Try this option + when a traced kernel hangs unexpectedly at boot time. + +config IPIPE_TRACE_PANIC + bool "Enable panic back traces" + default y + ---help--- + Provides services to freeze and dump a back trace on panic + situations. This is used on IPIPE_DEBUG_CONTEXT exceptions + as well as ordinary kernel oopses. You can control the number + of printed back trace points via /proc/ipipe/trace. + +endif diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile new file mode 100644 index 0000000..6257dfa --- /dev/null +++ b/kernel/ipipe/Makefile @@ -0,0 +1,3 @@ + +obj-$(CONFIG_IPIPE) += core.o +obj-$(CONFIG_IPIPE_TRACE) += tracer.o diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c new file mode 100644 index 0000000..aa2e5a0 --- /dev/null +++ b/kernel/ipipe/core.c @@ -0,0 +1,1794 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/core.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-independent I-PIPE core support. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_PROC_FS +#include +#include +#endif /* CONFIG_PROC_FS */ +#include +#include +#include + +static int __ipipe_ptd_key_count; + +static unsigned long __ipipe_ptd_key_map; + +static unsigned long __ipipe_domain_slot_map; + +struct ipipe_domain ipipe_root; + +#ifndef CONFIG_SMP +/* + * Create an alias to the unique root status, so that arch-dep code + * may get simple and easy access to this percpu variable. We also + * create an array of pointers to the percpu domain data; this tends + * to produce a better code when reaching non-root domains. We make + * sure that the early boot code would be able to dereference the + * pointer to the root domain data safely by statically initializing + * its value (local_irq*() routines depend on this). + */ +#if __GNUC__ >= 4 +extern unsigned long __ipipe_root_status +__attribute__((alias(__stringify(__raw_get_cpu_var(ipipe_percpu_darray))))); +EXPORT_SYMBOL(__ipipe_root_status); +#else /* __GNUC__ < 4 */ +/* + * Work around a GCC 3.x issue making alias symbols unusable as + * constant initializers. + */ +unsigned long *const __ipipe_root_status_addr = + &__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT].status; +EXPORT_SYMBOL(__ipipe_root_status_addr); +#endif /* __GNUC__ < 4 */ + +DEFINE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]) = +{ [IPIPE_ROOT_SLOT] = (struct ipipe_percpu_domain_data *)&__raw_get_cpu_var(ipipe_percpu_darray) }; +EXPORT_PER_CPU_SYMBOL(ipipe_percpu_daddr); +#endif /* !CONFIG_SMP */ + +DEFINE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]) = +{ [IPIPE_ROOT_SLOT] = { .status = IPIPE_STALL_MASK } }; /* Root domain stalled on each CPU at startup. */ + +DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) = { &ipipe_root }; + +DEFINE_PER_CPU(unsigned long, ipipe_nmi_saved_root); /* Copy of root status during NMI */ + +static IPIPE_DEFINE_SPINLOCK(__ipipe_pipelock); + +LIST_HEAD(__ipipe_pipeline); + +unsigned long __ipipe_virtual_irq_map; + +#ifdef CONFIG_PRINTK +unsigned __ipipe_printk_virq; +#endif /* CONFIG_PRINTK */ + +int __ipipe_event_monitors[IPIPE_NR_EVENTS]; + +#ifdef CONFIG_GENERIC_CLOCKEVENTS + +DECLARE_PER_CPU(struct tick_device, tick_cpu_device); + +static DEFINE_PER_CPU(struct ipipe_tick_device, ipipe_tick_cpu_device); + +int ipipe_request_tickdev(const char *devname, + void (*emumode)(enum clock_event_mode mode, + struct clock_event_device *cdev), + int (*emutick)(unsigned long delta, + struct clock_event_device *cdev), + int cpu, unsigned long *tmfreq) +{ + struct ipipe_tick_device *itd; + struct tick_device *slave; + struct clock_event_device *evtdev; + unsigned long long freq; + unsigned long flags; + int status; + + flags = ipipe_critical_enter(NULL); + + itd = &per_cpu(ipipe_tick_cpu_device, cpu); + + if (itd->slave != NULL) { + status = -EBUSY; + goto out; + } + + slave = &per_cpu(tick_cpu_device, cpu); + + if (strcmp(slave->evtdev->name, devname)) { + /* + * No conflict so far with the current tick device, + * check whether the requested device is sane and has + * been blessed by the kernel. + */ + status = __ipipe_check_tickdev(devname) ? + CLOCK_EVT_MODE_UNUSED : CLOCK_EVT_MODE_SHUTDOWN; + goto out; + } + + /* + * Our caller asks for using the same clock event device for + * ticking than we do, let's create a tick emulation device to + * interpose on the set_next_event() method, so that we may + * both manage the device in oneshot mode. Only the tick + * emulation code will actually program the clockchip hardware + * for the next shot, though. + * + * CAUTION: we still have to grab the tick device even when it + * current runs in periodic mode, since the kernel may switch + * to oneshot dynamically (highres/no_hz tick mode). + */ + + evtdev = slave->evtdev; + status = evtdev->mode; + + if (status == CLOCK_EVT_MODE_SHUTDOWN) + goto out; + + itd->slave = slave; + itd->emul_set_mode = emumode; + itd->emul_set_tick = emutick; + itd->real_set_mode = evtdev->set_mode; + itd->real_set_tick = evtdev->set_next_event; + itd->real_max_delta_ns = evtdev->max_delta_ns; + itd->real_mult = evtdev->mult; + itd->real_shift = evtdev->shift; + freq = (1000000000ULL * evtdev->mult) >> evtdev->shift; + *tmfreq = (unsigned long)freq; + evtdev->set_mode = emumode; + evtdev->set_next_event = emutick; + evtdev->max_delta_ns = ULONG_MAX; + evtdev->mult = 1; + evtdev->shift = 0; +out: + ipipe_critical_exit(flags); + + return status; +} + +void ipipe_release_tickdev(int cpu) +{ + struct ipipe_tick_device *itd; + struct tick_device *slave; + struct clock_event_device *evtdev; + unsigned long flags; + + flags = ipipe_critical_enter(NULL); + + itd = &per_cpu(ipipe_tick_cpu_device, cpu); + + if (itd->slave != NULL) { + slave = &per_cpu(tick_cpu_device, cpu); + evtdev = slave->evtdev; + evtdev->set_mode = itd->real_set_mode; + evtdev->set_next_event = itd->real_set_tick; + evtdev->max_delta_ns = itd->real_max_delta_ns; + evtdev->mult = itd->real_mult; + evtdev->shift = itd->real_shift; + itd->slave = NULL; + } + + ipipe_critical_exit(flags); +} + +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + +/* + * ipipe_init() -- Initialization routine of the IPIPE layer. Called + * by the host kernel early during the boot procedure. + */ +void __init ipipe_init(void) +{ + struct ipipe_domain *ipd = &ipipe_root; + + __ipipe_check_platform(); /* Do platform dependent checks first. */ + + /* + * A lightweight registration code for the root domain. We are + * running on the boot CPU, hw interrupts are off, and + * secondary CPUs are still lost in space. + */ + + /* Reserve percpu data slot #0 for the root domain. */ + ipd->slot = 0; + set_bit(0, &__ipipe_domain_slot_map); + + ipd->name = "Linux"; + ipd->domid = IPIPE_ROOT_ID; + ipd->priority = IPIPE_ROOT_PRIO; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + list_add_tail(&ipd->p_link, &__ipipe_pipeline); + + __ipipe_init_platform(); + +#ifdef CONFIG_PRINTK + __ipipe_printk_virq = ipipe_alloc_virq(); /* Cannot fail here. */ + ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk; + ipd->irqs[__ipipe_printk_virq].cookie = NULL; + ipd->irqs[__ipipe_printk_virq].acknowledge = NULL; + ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK; +#endif /* CONFIG_PRINTK */ + + __ipipe_enable_pipeline(); + + printk(KERN_INFO "I-pipe %s: pipeline enabled.\n", + IPIPE_VERSION_STRING); +} + +void __ipipe_init_stage(struct ipipe_domain *ipd) +{ + int cpu, n; + + for_each_online_cpu(cpu) { + + ipipe_percpudom(ipd, irqpend_himask, cpu) = 0; + + for (n = 0; n < IPIPE_IRQ_IWORDS; n++) { + ipipe_percpudom(ipd, irqpend_lomask, cpu)[n] = 0; + ipipe_percpudom(ipd, irqheld_mask, cpu)[n] = 0; + } + + for (n = 0; n < IPIPE_NR_IRQS; n++) + ipipe_percpudom(ipd, irqall, cpu)[n] = 0; + + ipipe_percpudom(ipd, evsync, cpu) = 0; + } + + for (n = 0; n < IPIPE_NR_IRQS; n++) { + ipd->irqs[n].acknowledge = NULL; + ipd->irqs[n].handler = NULL; + ipd->irqs[n].control = IPIPE_PASS_MASK; /* Pass but don't handle */ + } + + for (n = 0; n < IPIPE_NR_EVENTS; n++) + ipd->evhand[n] = NULL; + + ipd->evself = 0LL; + mutex_init(&ipd->mutex); + + __ipipe_hook_critical_ipi(ipd); +} + +void __ipipe_cleanup_domain(struct ipipe_domain *ipd) +{ + ipipe_unstall_pipeline_from(ipd); + +#ifdef CONFIG_SMP + { + int cpu; + + for_each_online_cpu(cpu) { + while (ipipe_percpudom(ipd, irqpend_himask, cpu) != 0) + cpu_relax(); + } + } +#else + __raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = NULL; +#endif + + clear_bit(ipd->slot, &__ipipe_domain_slot_map); +} + +void __ipipe_unstall_root(void) +{ + struct ipipe_percpu_domain_data *p; + + local_irq_disable_hw(); + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + /* This helps catching bad usage from assembly call sites. */ + BUG_ON(!__ipipe_root_domain_p && !oops_in_progress); +#endif + + p = ipipe_root_cpudom_ptr(); + + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (unlikely(p->irqpend_himask != 0)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + local_irq_enable_hw(); +} + +void __ipipe_restore_root(unsigned long x) +{ +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + BUG_ON(!ipipe_root_domain_p && !oops_in_progress); +#endif + + if (x) + __ipipe_stall_root(); + else + __ipipe_unstall_root(); +} + +void ipipe_stall_pipeline_from(struct ipipe_domain *ipd) +{ + unsigned long flags; + /* + * We have to prevent against race on updating the status + * variable _and_ CPU migration at the same time, so disable + * hw IRQs here. + */ + local_irq_save_hw(flags); + + __set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)); + + if (!__ipipe_pipeline_head_p(ipd)) + local_irq_restore_hw(flags); +} + +unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd) +{ + unsigned long flags, x; + + /* See ipipe_stall_pipeline_from() */ + local_irq_save_hw(flags); + + x = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)); + + if (!__ipipe_pipeline_head_p(ipd)) + local_irq_restore_hw(flags); + + return x; +} + +unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + unsigned long flags, x; + struct list_head *pos; + + local_irq_save_hw(flags); + + x = __test_and_clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)); + + if (ipd == __ipipe_current_domain) + pos = &ipd->p_link; + else + pos = __ipipe_pipeline.next; + + __ipipe_walk_pipeline(pos); + + if (likely(__ipipe_pipeline_head_p(ipd))) + local_irq_enable_hw(); + else + local_irq_restore_hw(flags); + + return x; +} + +void ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x) +{ + if (x) + ipipe_stall_pipeline_from(ipd); + else + ipipe_unstall_pipeline_from(ipd); +} + +void ipipe_unstall_pipeline_head(void) +{ + struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr(); + + local_irq_disable_hw(); + + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (unlikely(p->irqpend_himask != 0)) { + struct ipipe_domain *head_domain = __ipipe_pipeline_head(); + if (likely(head_domain == __ipipe_current_domain)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head_domain->p_link); + } + + local_irq_enable_hw(); +} + +void __ipipe_restore_pipeline_head(unsigned long x) +{ + struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr(); + + local_irq_disable_hw(); + + if (x) { +#ifdef CONFIG_DEBUG_KERNEL + static int warned; + if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) { + /* + * Already stalled albeit ipipe_restore_pipeline_head() + * should have detected it? Send a warning once. + */ + warned = 1; + printk(KERN_WARNING + "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n"); + dump_stack(); + } +#else /* !CONFIG_DEBUG_KERNEL */ + set_bit(IPIPE_STALL_FLAG, &p->status); +#endif /* CONFIG_DEBUG_KERNEL */ + } + else { + __clear_bit(IPIPE_STALL_FLAG, &p->status); + if (unlikely(p->irqpend_himask != 0)) { + struct ipipe_domain *head_domain = __ipipe_pipeline_head(); + if (likely(head_domain == __ipipe_current_domain)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else + __ipipe_walk_pipeline(&head_domain->p_link); + } + local_irq_enable_hw(); + } +} + +void __ipipe_spin_lock_irq(raw_spinlock_t *lock) +{ + local_irq_disable_hw(); + __raw_spin_lock(lock); + __set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)); +} + +void __ipipe_spin_unlock_irq(raw_spinlock_t *lock) +{ + __raw_spin_unlock(lock); + __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)); + local_irq_enable_hw(); +} + +unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock) +{ + unsigned long flags; + int s; + + local_irq_save_hw(flags); + __raw_spin_lock(lock); + s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)); + + return raw_mangle_irq_bits(s, flags); +} + +void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long x) +{ + __raw_spin_unlock(lock); + if (!raw_demangle_irq_bits(&x)) + __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)); + local_irq_restore_hw(x); +} + +void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock) +{ + __raw_spin_unlock(&lock->__raw_lock); +} + +void __ipipe_spin_unlock_irqcomplete(unsigned long x) +{ + if (!raw_demangle_irq_bits(&x)) + __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status)); + local_irq_restore_hw(x); +} + +/* Must be called hw IRQs off. */ +void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq) +{ + int level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK; + struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(ipd); + + prefetchw(p); + + if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) { + __set_bit(rank, &p->irqpend_lomask[level]); + __set_bit(level, &p->irqpend_himask); + } else + __set_bit(rank, &p->irqheld_mask[level]); + + p->irqall[irq]++; +} + +/* Must be called hw IRQs off. */ +void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq) +{ + struct ipipe_percpu_domain_data *p; + int level, rank; + + if (unlikely(test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) + return; + + level = irq >> IPIPE_IRQ_ISHIFT; + rank = irq & IPIPE_IRQ_IMASK; + p = ipipe_percpudom_ptr(ipd, cpu); + + if (__test_and_clear_bit(rank, &p->irqpend_lomask[level])) + __set_bit(rank, &p->irqheld_mask[level]); + if (p->irqpend_lomask[level] == 0) + __clear_bit(level, &p->irqpend_himask); +} + +/* Must be called hw IRQs off. */ +void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq) +{ + struct ipipe_percpu_domain_data *p; + int cpu, level, rank; + + if (unlikely(!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) + return; + + level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK; + for_each_online_cpu(cpu) { + p = ipipe_percpudom_ptr(ipd, cpu); + if (test_and_clear_bit(rank, &p->irqheld_mask[level])) { + /* We need atomic ops here: */ + set_bit(rank, &p->irqpend_lomask[level]); + set_bit(level, &p->irqpend_himask); + } + } +} + +/* + * __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must + * be called with local hw interrupts disabled. + */ +void __ipipe_walk_pipeline(struct list_head *pos) +{ + struct ipipe_domain *this_domain = __ipipe_current_domain, *next_domain; + struct ipipe_percpu_domain_data *p, *np; + + p = ipipe_cpudom_ptr(this_domain); + + while (pos != &__ipipe_pipeline) { + + next_domain = list_entry(pos, struct ipipe_domain, p_link); + np = ipipe_cpudom_ptr(next_domain); + + if (test_bit(IPIPE_STALL_FLAG, &np->status)) + break; /* Stalled stage -- do not go further. */ + + if (np->irqpend_himask) { + if (next_domain == this_domain) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + else { + + p->evsync = 0; + __ipipe_current_domain = next_domain; + ipipe_suspend_domain(); /* Sync stage and propagate interrupts. */ + + if (__ipipe_current_domain == next_domain) + __ipipe_current_domain = this_domain; + /* + * Otherwise, something changed the current domain under our + * feet recycling the register set; do not override the new + * domain. + */ + + if (p->irqpend_himask && + !test_bit(IPIPE_STALL_FLAG, &p->status)) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + } + break; + } else if (next_domain == this_domain) + break; + + pos = next_domain->p_link.next; + } +} + +/* + * ipipe_suspend_domain() -- Suspend the current domain, switching to + * the next one which has pending work down the pipeline. + */ +void ipipe_suspend_domain(void) +{ + struct ipipe_domain *this_domain, *next_domain; + struct ipipe_percpu_domain_data *p; + struct list_head *ln; + unsigned long flags; + + local_irq_save_hw(flags); + + this_domain = next_domain = __ipipe_current_domain; + p = ipipe_cpudom_ptr(this_domain); + p->status &= ~(IPIPE_STALL_MASK|IPIPE_SYNC_MASK); + + if (p->irqpend_himask != 0) + goto sync_stage; + + for (;;) { + ln = next_domain->p_link.next; + + if (ln == &__ipipe_pipeline) + break; + + next_domain = list_entry(ln, struct ipipe_domain, p_link); + p = ipipe_cpudom_ptr(next_domain); + + if (p->status & IPIPE_STALL_MASK) + break; + + if (p->irqpend_himask == 0) + continue; + + __ipipe_current_domain = next_domain; +sync_stage: + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + if (__ipipe_current_domain != next_domain) + /* + * Something has changed the current domain under our + * feet, recycling the register set; take note. + */ + this_domain = __ipipe_current_domain; + } + + __ipipe_current_domain = this_domain; + + local_irq_restore_hw(flags); +} + + +/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt. + * Virtual interrupts are handled in exactly the same way than their + * hw-generated counterparts wrt pipelining. + */ +unsigned ipipe_alloc_virq(void) +{ + unsigned long flags, irq = 0; + int ipos; + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + if (__ipipe_virtual_irq_map != ~0) { + ipos = ffz(__ipipe_virtual_irq_map); + set_bit(ipos, &__ipipe_virtual_irq_map); + irq = ipos + IPIPE_VIRQ_BASE; + } + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return irq; +} + +/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw + acknowledge routine) to an interrupt for a given domain. */ + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t acknowledge, + unsigned modemask) +{ + ipipe_irq_handler_t old_handler; + unsigned long flags; + int err; + + if (irq >= IPIPE_NR_IRQS) + return -EINVAL; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) + return -EPERM; + + if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags)) + /* Silently unwire interrupts for non-heading domains. */ + modemask &= ~IPIPE_WIRED_MASK; + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + old_handler = ipd->irqs[irq].handler; + + if (handler != NULL) { + if (handler == IPIPE_SAME_HANDLER) { + handler = old_handler; + cookie = ipd->irqs[irq].cookie; + + if (handler == NULL) { + err = -EINVAL; + goto unlock_and_exit; + } + } else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 && + old_handler != NULL) { + err = -EBUSY; + goto unlock_and_exit; + } + + /* Wired interrupts can only be delivered to domains + * always heading the pipeline, and using dynamic + * propagation. */ + + if ((modemask & IPIPE_WIRED_MASK) != 0) { + if ((modemask & (IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) { + err = -EINVAL; + goto unlock_and_exit; + } + modemask |= (IPIPE_HANDLE_MASK); + } + + if ((modemask & IPIPE_STICKY_MASK) != 0) + modemask |= IPIPE_HANDLE_MASK; + } else + modemask &= + ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK | + IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK); + + if (acknowledge == NULL && !ipipe_virtual_irq_p(irq)) + /* + * Acknowledge handler unspecified for a hw interrupt: + * use the Linux-defined handler instead. + */ + acknowledge = ipipe_root_domain->irqs[irq].acknowledge; + + ipd->irqs[irq].handler = handler; + ipd->irqs[irq].cookie = cookie; + ipd->irqs[irq].acknowledge = acknowledge; + ipd->irqs[irq].control = modemask; + + if (irq < NR_IRQS && !ipipe_virtual_irq_p(irq)) { + if (handler != NULL) { + __ipipe_enable_irqdesc(ipd, irq); + + if ((modemask & IPIPE_ENABLE_MASK) != 0) { + if (ipd != __ipipe_current_domain) { + /* + * IRQ enable/disable state is domain-sensitive, so we + * may not change it for another domain. What is + * allowed however is forcing some domain to handle an + * interrupt source, by passing the proper 'ipd' + * descriptor which thus may be different from + * __ipipe_current_domain. + */ + err = -EPERM; + goto unlock_and_exit; + } + __ipipe_enable_irq(irq); + } + } else if (old_handler != NULL) + __ipipe_disable_irqdesc(ipd, irq); + } + + err = 0; + + unlock_and_exit: + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return err; +} + +/* ipipe_control_irq() -- Change modes of a pipelined interrupt for + * the current domain. */ + +int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask) +{ + struct ipipe_domain *ipd; + unsigned long flags; + + if (irq >= IPIPE_NR_IRQS) + return -EINVAL; + + spin_lock_irqsave(&__ipipe_pipelock, flags); + + ipd = __ipipe_current_domain; + + if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK) { + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + return -EPERM; + } + + if (ipd->irqs[irq].handler == NULL) + setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + if ((setmask & IPIPE_STICKY_MASK) != 0) + setmask |= IPIPE_HANDLE_MASK; + + if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0) /* If one goes, both go. */ + clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK); + + ipd->irqs[irq].control &= ~clrmask; + ipd->irqs[irq].control |= setmask; + + if ((setmask & IPIPE_ENABLE_MASK) != 0) + __ipipe_enable_irq(irq); + else if ((clrmask & IPIPE_ENABLE_MASK) != 0) + __ipipe_disable_irq(irq); + + spin_unlock_irqrestore(&__ipipe_pipelock, flags); + + return 0; +} + +/* __ipipe_dispatch_event() -- Low-level event dispatcher. */ + +int __ipipe_dispatch_event (unsigned event, void *data) +{ + struct ipipe_domain *start_domain, *this_domain, *next_domain; + ipipe_event_handler_t evhand; + struct list_head *pos, *npos; + unsigned long flags; + int propagate = 1; + + local_irq_save_hw(flags); + + start_domain = this_domain = __ipipe_current_domain; + + list_for_each_safe(pos, npos, &__ipipe_pipeline) { + /* + * Note: Domain migration may occur while running + * event or interrupt handlers, in which case the + * current register set is going to be recycled for a + * different domain than the initiating one. We do + * care for that, always tracking the current domain + * descriptor upon return from those handlers. + */ + next_domain = list_entry(pos, struct ipipe_domain, p_link); + + /* + * Keep a cached copy of the handler's address since + * ipipe_catch_event() may clear it under our feet. + */ + evhand = next_domain->evhand[event]; + + if (evhand != NULL) { + __ipipe_current_domain = next_domain; + ipipe_cpudom_var(next_domain, evsync) |= (1LL << event); + local_irq_restore_hw(flags); + propagate = !evhand(event, start_domain, data); + local_irq_save_hw(flags); + /* + * We may have a migration issue here, if the + * current task is migrated to another CPU on + * behalf of the invoked handler, usually when + * a syscall event is processed. However, + * ipipe_catch_event() will make sure that a + * CPU that clears a handler for any given + * event will not attempt to wait for itself + * to clear the evsync bit for that event, + * which practically plugs the hole, without + * resorting to a much more complex strategy. + */ + ipipe_cpudom_var(next_domain, evsync) &= ~(1LL << event); + if (__ipipe_current_domain != next_domain) + this_domain = __ipipe_current_domain; + } + + if (next_domain != ipipe_root_domain && /* NEVER sync the root stage here. */ + ipipe_cpudom_var(next_domain, irqpend_himask) != 0 && + !test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(next_domain, status))) { + __ipipe_current_domain = next_domain; + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + if (__ipipe_current_domain != next_domain) + this_domain = __ipipe_current_domain; + } + + __ipipe_current_domain = this_domain; + + if (next_domain == this_domain || !propagate) + break; + } + + local_irq_restore_hw(flags); + + return !propagate; +} + +/* + * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired + * interrupts are immediately and unconditionally delivered to the + * domain heading the pipeline upon receipt, and such domain must have + * been registered as an invariant head for the system (priority == + * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is + * to get an extra-fast dispatching path for those IRQs, by relying on + * a straightforward logic based on assumptions that must always be + * true for invariant head domains. The following assumptions are + * made when dealing with such interrupts: + * + * 1- Wired interrupts are purely dynamic, i.e. the decision to + * propagate them down the pipeline must be done from the head domain + * ISR. + * 2- Wired interrupts cannot be shared or sticky. + * 3- The root domain cannot be an invariant pipeline head, in + * consequence of what the root domain cannot handle wired + * interrupts. + * 4- Wired interrupts must have a valid acknowledge handler for the + * head domain (if needed, see __ipipe_handle_irq). + * + * Called with hw interrupts off. + */ + +void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq) +{ + struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head); + + prefetchw(p); + + if (unlikely(test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control))) { + /* + * If we can't process this IRQ right now, we must + * mark it as held, so that it will get played during + * normal log sync when the corresponding interrupt + * source is eventually unlocked. + */ + p->irqall[irq]++; + __set_bit(irq & IPIPE_IRQ_IMASK, &p->irqheld_mask[irq >> IPIPE_IRQ_ISHIFT]); + return; + } + + if (test_bit(IPIPE_STALL_FLAG, &p->status)) { + __ipipe_set_irq_pending(head, irq); + return; + } + + __ipipe_dispatch_wired_nocheck(head, irq); +} + +void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq) /* hw interrupts off */ +{ + struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head); + struct ipipe_domain *old; + + prefetchw(p); + + old = __ipipe_current_domain; + __ipipe_current_domain = head; /* Switch to the head domain. */ + + p->irqall[irq]++; + __set_bit(IPIPE_STALL_FLAG, &p->status); + head->irqs[irq].handler(irq, head->irqs[irq].cookie); /* Call the ISR. */ + __ipipe_run_irqtail(); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (__ipipe_current_domain == head) { + __ipipe_current_domain = old; + if (old == head) { + if (p->irqpend_himask) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + return; + } + } + + __ipipe_walk_pipeline(&head->p_link); +} + +/* + * __ipipe_sync_stage() -- Flush the pending IRQs for the current + * domain (and processor). This routine flushes the interrupt log + * (see "Optimistic interrupt protection" from D. Stodolsky et al. for + * more on the deferred interrupt scheme). Every interrupt that + * occurred while the pipeline was stalled gets played. WARNING: + * callers on SMP boxen should always check for CPU migration on + * return of this routine. One can control the kind of interrupts + * which are going to be sync'ed using the syncmask + * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT + * plays virtual interrupts only. + * + * This routine must be called with hw interrupts off. + */ +void __ipipe_sync_stage(unsigned long syncmask) +{ + struct ipipe_percpu_domain_data *p; + unsigned long mask, submask; + struct ipipe_domain *ipd; + int level, rank, cpu; + unsigned irq; + + ipd = __ipipe_current_domain; + p = ipipe_cpudom_ptr(ipd); + + if (__test_and_set_bit(IPIPE_SYNC_FLAG, &p->status)) + return; + + cpu = ipipe_processor_id(); + + /* + * The policy here is to keep the dispatching code interrupt-free + * by stalling the current stage. If the upper domain handler + * (which we call) wants to re-enable interrupts while in a safe + * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's + * sigaction()), it will have to unstall (then stall again before + * returning to us!) the stage when it sees fit. + */ + while ((mask = (p->irqpend_himask & syncmask)) != 0) { + level = __ipipe_ffnz(mask); + + while ((submask = p->irqpend_lomask[level]) != 0) { + rank = __ipipe_ffnz(submask); + irq = (level << IPIPE_IRQ_ISHIFT) + rank; + + __clear_bit(rank, &p->irqpend_lomask[level]); + + if (p->irqpend_lomask[level] == 0) + __clear_bit(level, &p->irqpend_himask); + /* + * Make sure the compiler will not postpone + * the pending bitmask updates before calling + * the interrupt handling routine. Otherwise, + * those late updates could overwrite any + * change to irqpend_hi/lomask due to a nested + * interrupt, leaving the latter unprocessed + * (seen on mpc836x). + */ + barrier(); + + if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) + continue; + + __set_bit(IPIPE_STALL_FLAG, &p->status); + barrier(); + + if (ipd == ipipe_root_domain) + trace_hardirqs_off(); + + __ipipe_run_isr(ipd, irq); + barrier(); + p = ipipe_cpudom_ptr(__ipipe_current_domain); +#ifdef CONFIG_SMP + { + int newcpu = ipipe_processor_id(); + + if (newcpu != cpu) { /* Handle CPU migration. */ + /* + * We expect any domain to clear the SYNC bit each + * time it switches in a new task, so that preemptions + * and/or CPU migrations (in the SMP case) over the + * ISR do not lock out the log syncer for some + * indefinite amount of time. In the Linux case, + * schedule() handles this (see kernel/sched.c). For + * this reason, we don't bother clearing it here for + * the source CPU in the migration handling case, + * since it must have scheduled another task in by + * now. + */ + __set_bit(IPIPE_SYNC_FLAG, &p->status); + cpu = newcpu; + } + } +#endif /* CONFIG_SMP */ +#ifdef CONFIG_TRACE_IRQFLAGS + if (__ipipe_root_domain_p && + test_bit(IPIPE_STALL_FLAG, &p->status)) + trace_hardirqs_on(); +#endif + __clear_bit(IPIPE_STALL_FLAG, &p->status); + } + } + + __clear_bit(IPIPE_SYNC_FLAG, &p->status); +} + +/* ipipe_register_domain() -- Link a new domain to the pipeline. */ + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr) +{ + struct ipipe_domain *_ipd; + struct list_head *pos = NULL; + unsigned long flags; + + if (!ipipe_root_domain_p) { + printk(KERN_WARNING + "I-pipe: Only the root domain may register a new domain.\n"); + return -EPERM; + } + + flags = ipipe_critical_enter(NULL); + + if (attr->priority == IPIPE_HEAD_PRIORITY) { + if (test_bit(IPIPE_HEAD_SLOT, &__ipipe_domain_slot_map)) { + ipipe_critical_exit(flags); + return -EAGAIN; /* Cannot override current head. */ + } + ipd->slot = IPIPE_HEAD_SLOT; + } else + ipd->slot = ffz(__ipipe_domain_slot_map); + + if (ipd->slot < CONFIG_IPIPE_DOMAINS) { + set_bit(ipd->slot, &__ipipe_domain_slot_map); + list_for_each(pos, &__ipipe_pipeline) { + _ipd = list_entry(pos, struct ipipe_domain, p_link); + if (_ipd->domid == attr->domid) + break; + } + } + + ipipe_critical_exit(flags); + + if (pos != &__ipipe_pipeline) { + if (ipd->slot < CONFIG_IPIPE_DOMAINS) + clear_bit(ipd->slot, &__ipipe_domain_slot_map); + return -EBUSY; + } + +#ifndef CONFIG_SMP + /* + * Set up the perdomain pointers for direct access to the + * percpu domain data. This saves a costly multiply each time + * we need to refer to the contents of the percpu domain data + * array. + */ + __raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = &__raw_get_cpu_var(ipipe_percpu_darray)[ipd->slot]; +#endif + + ipd->name = attr->name; + ipd->domid = attr->domid; + ipd->pdd = attr->pdd; + ipd->flags = 0; + + if (attr->priority == IPIPE_HEAD_PRIORITY) { + ipd->priority = INT_MAX; + __set_bit(IPIPE_AHEAD_FLAG,&ipd->flags); + } + else + ipd->priority = attr->priority; + + __ipipe_init_stage(ipd); + + INIT_LIST_HEAD(&ipd->p_link); + +#ifdef CONFIG_PROC_FS + __ipipe_add_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + flags = ipipe_critical_enter(NULL); + + list_for_each(pos, &__ipipe_pipeline) { + _ipd = list_entry(pos, struct ipipe_domain, p_link); + if (ipd->priority > _ipd->priority) + break; + } + + list_add_tail(&ipd->p_link, pos); + + ipipe_critical_exit(flags); + + printk(KERN_INFO "I-pipe: Domain %s registered.\n", ipd->name); + + /* + * Finally, allow the new domain to perform its initialization + * chores. + */ + + if (attr->entry != NULL) { + local_irq_save_hw_smp(flags); + __ipipe_current_domain = ipd; + local_irq_restore_hw_smp(flags); + attr->entry(); + local_irq_save_hw(flags); + __ipipe_current_domain = ipipe_root_domain; + + if (ipipe_root_cpudom_var(irqpend_himask) != 0 && + !test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status))) + __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); + + local_irq_restore_hw(flags); + } + + return 0; +} + +/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */ + +int ipipe_unregister_domain(struct ipipe_domain *ipd) +{ + unsigned long flags; + + if (!ipipe_root_domain_p) { + printk(KERN_WARNING + "I-pipe: Only the root domain may unregister a domain.\n"); + return -EPERM; + } + + if (ipd == ipipe_root_domain) { + printk(KERN_WARNING + "I-pipe: Cannot unregister the root domain.\n"); + return -EPERM; + } +#ifdef CONFIG_SMP + { + unsigned irq; + int cpu; + + /* + * In the SMP case, wait for the logged events to drain on + * other processors before eventually removing the domain + * from the pipeline. + */ + + ipipe_unstall_pipeline_from(ipd); + + flags = ipipe_critical_enter(NULL); + + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) { + clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control); + clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control); + set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control); + } + + ipipe_critical_exit(flags); + + for_each_online_cpu(cpu) { + while (ipipe_percpudom(ipd, irqpend_himask, cpu) > 0) + cpu_relax(); + } + } +#endif /* CONFIG_SMP */ + + mutex_lock(&ipd->mutex); + +#ifdef CONFIG_PROC_FS + __ipipe_remove_domain_proc(ipd); +#endif /* CONFIG_PROC_FS */ + + /* + * Simply remove the domain from the pipeline and we are almost done. + */ + + flags = ipipe_critical_enter(NULL); + list_del_init(&ipd->p_link); + ipipe_critical_exit(flags); + + __ipipe_cleanup_domain(ipd); + + mutex_unlock(&ipd->mutex); + + printk(KERN_INFO "I-pipe: Domain %s unregistered.\n", ipd->name); + + return 0; +} + +/* + * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of + * a running interrupt handler to the next domain down the pipeline. + * ipipe_schedule_irq() -- Does almost the same as above, but attempts + * to pend the interrupt for the current domain first. + * Must be called hw IRQs off. + */ +void __ipipe_pend_irq(unsigned irq, struct list_head *head) +{ + struct ipipe_domain *ipd; + struct list_head *ln; + +#ifdef CONFIG_IPIPE_DEBUG + BUG_ON(irq >= IPIPE_NR_IRQS || + (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))); +#endif + for (ln = head; ln != &__ipipe_pipeline; ln = ipd->p_link.next) { + ipd = list_entry(ln, struct ipipe_domain, p_link); + if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) { + __ipipe_set_irq_pending(ipd, irq); + return; + } + } +} + +/* ipipe_free_virq() -- Release a virtual/soft interrupt. */ + +int ipipe_free_virq(unsigned virq) +{ + if (!ipipe_virtual_irq_p(virq)) + return -EINVAL; + + clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map); + + return 0; +} + +void ipipe_init_attr(struct ipipe_domain_attr *attr) +{ + attr->name = "anon"; + attr->domid = 1; + attr->entry = NULL; + attr->priority = IPIPE_ROOT_PRIO; + attr->pdd = NULL; +} + +/* + * ipipe_catch_event() -- Interpose or remove an event handler for a + * given domain. + */ +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned event, + ipipe_event_handler_t handler) +{ + ipipe_event_handler_t old_handler; + unsigned long flags; + int self = 0, cpu; + + if (event & IPIPE_EVENT_SELF) { + event &= ~IPIPE_EVENT_SELF; + self = 1; + } + + if (event >= IPIPE_NR_EVENTS) + return NULL; + + flags = ipipe_critical_enter(NULL); + + if (!(old_handler = xchg(&ipd->evhand[event],handler))) { + if (handler) { + if (self) + ipd->evself |= (1LL << event); + else + __ipipe_event_monitors[event]++; + } + } + else if (!handler) { + if (ipd->evself & (1LL << event)) + ipd->evself &= ~(1LL << event); + else + __ipipe_event_monitors[event]--; + } else if ((ipd->evself & (1LL << event)) && !self) { + __ipipe_event_monitors[event]++; + ipd->evself &= ~(1LL << event); + } else if (!(ipd->evself & (1LL << event)) && self) { + __ipipe_event_monitors[event]--; + ipd->evself |= (1LL << event); + } + + ipipe_critical_exit(flags); + + if (!handler && ipipe_root_domain_p) { + /* + * If we cleared a handler on behalf of the root + * domain, we have to wait for any current invocation + * to drain, since our caller might subsequently unmap + * the target domain. To this aim, this code + * synchronizes with __ipipe_dispatch_event(), + * guaranteeing that either the dispatcher sees a null + * handler in which case it discards the invocation + * (which also prevents from entering a livelock), or + * finds a valid handler and calls it. Symmetrically, + * ipipe_catch_event() ensures that the called code + * won't be unmapped under our feet until the event + * synchronization flag is cleared for the given event + * on all CPUs. + */ + preempt_disable(); + cpu = smp_processor_id(); + /* + * Hack: this solves the potential migration issue + * raised in __ipipe_dispatch_event(). This is a + * work-around which makes the assumption that other + * CPUs will subsequently, either process at least one + * interrupt for the target domain, or call + * __ipipe_dispatch_event() without going through a + * migration while running the handler at least once; + * practically, this is safe on any normally running + * system. + */ + ipipe_percpudom(ipd, evsync, cpu) &= ~(1LL << event); + preempt_enable(); + + for_each_online_cpu(cpu) { + while (ipipe_percpudom(ipd, evsync, cpu) & (1LL << event)) + schedule_timeout_interruptible(HZ / 50); + } + } + + return old_handler; +} + +cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask) +{ +#ifdef CONFIG_SMP + if (irq >= IPIPE_NR_XIRQS) + /* Allow changing affinity of external IRQs only. */ + return CPU_MASK_NONE; + + if (num_online_cpus() > 1) + return __ipipe_set_irq_affinity(irq,cpumask); +#endif /* CONFIG_SMP */ + + return CPU_MASK_NONE; +} + +int ipipe_send_ipi (unsigned ipi, cpumask_t cpumask) + +{ +#ifdef CONFIG_SMP + return __ipipe_send_ipi(ipi,cpumask); +#else /* !CONFIG_SMP */ + return -EINVAL; +#endif /* CONFIG_SMP */ +} + +int ipipe_alloc_ptdkey (void) +{ + unsigned long flags; + int key = -1; + + spin_lock_irqsave(&__ipipe_pipelock,flags); + + if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) { + key = ffz(__ipipe_ptd_key_map); + set_bit(key,&__ipipe_ptd_key_map); + __ipipe_ptd_key_count++; + } + + spin_unlock_irqrestore(&__ipipe_pipelock,flags); + + return key; +} + +int ipipe_free_ptdkey (int key) +{ + unsigned long flags; + + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + spin_lock_irqsave(&__ipipe_pipelock,flags); + + if (test_and_clear_bit(key,&__ipipe_ptd_key_map)) + __ipipe_ptd_key_count--; + + spin_unlock_irqrestore(&__ipipe_pipelock,flags); + + return 0; +} + +int ipipe_set_ptd (int key, void *value) + +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + current->ptd[key] = value; + + return 0; +} + +void *ipipe_get_ptd (int key) + +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return NULL; + + return current->ptd[key]; +} + +#ifdef CONFIG_PROC_FS + +struct proc_dir_entry *ipipe_proc_root; + +static int __ipipe_version_info_proc(char *page, + char **start, + off_t off, int count, int *eof, void *data) +{ + int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING); + + len -= off; + + if (len <= off + count) + *eof = 1; + + *start = page + off; + + if(len > count) + len = count; + + if(len < 0) + len = 0; + + return len; +} + +static int __ipipe_common_info_show(struct seq_file *p, void *data) +{ + struct ipipe_domain *ipd = (struct ipipe_domain *)p->private; + char handling, stickiness, lockbit, exclusive, virtuality; + + unsigned long ctlbits; + unsigned irq; + + seq_printf(p, " +----- Handling ([A]ccepted, [G]rabbed, [W]ired, [D]iscarded)\n"); + seq_printf(p, " |+---- Sticky\n"); + seq_printf(p, " ||+--- Locked\n"); + seq_printf(p, " |||+-- Exclusive\n"); + seq_printf(p, " ||||+- Virtual\n"); + seq_printf(p, "[IRQ] |||||\n"); + + mutex_lock(&ipd->mutex); + + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) { + /* Remember to protect against + * ipipe_virtual_irq/ipipe_control_irq if more fields + * get involved. */ + ctlbits = ipd->irqs[irq].control; + + if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) + /* + * There might be a hole between the last external + * IRQ and the first virtual one; skip it. + */ + continue; + + if (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)) + /* Non-allocated virtual IRQ; skip it. */ + continue; + + /* + * Statuses are as follows: + * o "accepted" means handled _and_ passed down the pipeline. + * o "grabbed" means handled, but the interrupt might be + * terminated _or_ passed down the pipeline depending on + * what the domain handler asks for to the I-pipe. + * o "wired" is basically the same as "grabbed", except that + * the interrupt is unconditionally delivered to an invariant + * pipeline head domain. + * o "passed" means unhandled by the domain but passed + * down the pipeline. + * o "discarded" means unhandled and _not_ passed down the + * pipeline. The interrupt merely disappears from the + * current domain down to the end of the pipeline. + */ + if (ctlbits & IPIPE_HANDLE_MASK) { + if (ctlbits & IPIPE_PASS_MASK) + handling = 'A'; + else if (ctlbits & IPIPE_WIRED_MASK) + handling = 'W'; + else + handling = 'G'; + } else if (ctlbits & IPIPE_PASS_MASK) + /* Do not output if no major action is taken. */ + continue; + else + handling = 'D'; + + if (ctlbits & IPIPE_STICKY_MASK) + stickiness = 'S'; + else + stickiness = '.'; + + if (ctlbits & IPIPE_LOCK_MASK) + lockbit = 'L'; + else + lockbit = '.'; + + if (ctlbits & IPIPE_EXCLUSIVE_MASK) + exclusive = 'X'; + else + exclusive = '.'; + + if (ipipe_virtual_irq_p(irq)) + virtuality = 'V'; + else + virtuality = '.'; + + seq_printf(p, " %3u: %c%c%c%c%c\n", + irq, handling, stickiness, lockbit, exclusive, virtuality); + } + + seq_printf(p, "[Domain info]\n"); + + seq_printf(p, "id=0x%.8x\n", ipd->domid); + + if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags)) + seq_printf(p, "priority=topmost\n"); + else + seq_printf(p, "priority=%d\n", ipd->priority); + + mutex_unlock(&ipd->mutex); + + return 0; +} + +static int __ipipe_common_info_open(struct inode *inode, struct file *file) +{ + return single_open(file, __ipipe_common_info_show, PROC_I(inode)->pde->data); +} + +static struct file_operations __ipipe_info_proc_ops = { + .owner = THIS_MODULE, + .open = __ipipe_common_info_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void __ipipe_add_domain_proc(struct ipipe_domain *ipd) +{ + struct proc_dir_entry *e = create_proc_entry(ipd->name, 0444, ipipe_proc_root); + if (e) { + e->proc_fops = &__ipipe_info_proc_ops; + e->data = (void*) ipd; + } +} + +void __ipipe_remove_domain_proc(struct ipipe_domain *ipd) +{ + remove_proc_entry(ipd->name,ipipe_proc_root); +} + +void __init ipipe_init_proc(void) +{ + ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0); + create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL); + __ipipe_add_domain_proc(ipipe_root_domain); + + __ipipe_init_tracer(); +} + +#endif /* CONFIG_PROC_FS */ + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + +DEFINE_PER_CPU(int, ipipe_percpu_context_check) = { 1 }; +DEFINE_PER_CPU(int, ipipe_saved_context_check_state); + +void ipipe_check_context(struct ipipe_domain *border_domain) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_domain *this_domain; + unsigned long flags; + int cpu; + + local_irq_save_hw_smp(flags); + + this_domain = __ipipe_current_domain; + p = ipipe_head_cpudom_ptr(); + if (likely(this_domain->priority <= border_domain->priority && + !test_bit(IPIPE_STALL_FLAG, &p->status))) { + local_irq_restore_hw_smp(flags); + return; + } + + cpu = ipipe_processor_id(); + if (!per_cpu(ipipe_percpu_context_check, cpu)) { + local_irq_restore_hw_smp(flags); + return; + } + + local_irq_restore_hw_smp(flags); + + ipipe_context_check_off(); + ipipe_trace_panic_freeze(); + oops_in_progress = 1; + + if (this_domain->priority > border_domain->priority) + printk(KERN_ERR "I-pipe: Detected illicit call from domain " + "'%s'\n" + KERN_ERR " into a service reserved for domain " + "'%s' and below.\n", + this_domain->name, border_domain->name); + else + printk(KERN_ERR "I-pipe: Detected stalled topmost domain, " + "probably caused by a bug.\n" + " A critical section may have been " + "left unterminated.\n"); + dump_stack(); + ipipe_trace_panic_dump(); +} + +EXPORT_SYMBOL(ipipe_check_context); + +#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */ + +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) + +int notrace __ipipe_check_percpu_access(void) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_domain *this_domain; + unsigned long flags; + int ret = 0; + + local_irq_save_hw_notrace(flags); + + this_domain = __raw_get_cpu_var(ipipe_percpu_domain); + + /* + * Only the root domain may implement preemptive CPU migration + * of tasks, so anything above in the pipeline should be fine. + */ + if (this_domain->priority > IPIPE_ROOT_PRIO) + goto out; + + if (raw_irqs_disabled_flags(flags)) + goto out; + + /* + * Last chance: hw interrupts were enabled on entry while + * running over the root domain, but the root stage might be + * currently stalled, in which case preemption would be + * disabled, and no migration could occur. + */ + if (this_domain == ipipe_root_domain) { + p = ipipe_root_cpudom_ptr(); + if (test_bit(IPIPE_STALL_FLAG, &p->status)) + goto out; + } + /* + * Our caller may end up accessing the wrong per-cpu variable + * instance due to CPU migration; tell it to complain about + * this. + */ + ret = 1; +out: + local_irq_restore_hw_notrace(flags); + + return ret; +} + +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL && CONFIG_SMP */ + +EXPORT_SYMBOL(ipipe_virtualize_irq); +EXPORT_SYMBOL(ipipe_control_irq); +EXPORT_SYMBOL(ipipe_suspend_domain); +EXPORT_SYMBOL(ipipe_alloc_virq); +EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain); +EXPORT_PER_CPU_SYMBOL(ipipe_percpu_darray); +EXPORT_SYMBOL(ipipe_root); +EXPORT_SYMBOL(ipipe_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from); +EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from); +EXPORT_SYMBOL(ipipe_restore_pipeline_from); +EXPORT_SYMBOL(ipipe_unstall_pipeline_head); +EXPORT_SYMBOL(__ipipe_restore_pipeline_head); +EXPORT_SYMBOL(__ipipe_unstall_root); +EXPORT_SYMBOL(__ipipe_restore_root); +EXPORT_SYMBOL(__ipipe_spin_lock_irq); +EXPORT_SYMBOL(__ipipe_spin_unlock_irq); +EXPORT_SYMBOL(__ipipe_spin_lock_irqsave); +EXPORT_SYMBOL(__ipipe_spin_unlock_irqrestore); +EXPORT_SYMBOL(__ipipe_pipeline); +EXPORT_SYMBOL(__ipipe_lock_irq); +EXPORT_SYMBOL(__ipipe_unlock_irq); +EXPORT_SYMBOL(ipipe_register_domain); +EXPORT_SYMBOL(ipipe_unregister_domain); +EXPORT_SYMBOL(ipipe_free_virq); +EXPORT_SYMBOL(ipipe_init_attr); +EXPORT_SYMBOL(ipipe_catch_event); +EXPORT_SYMBOL(ipipe_alloc_ptdkey); +EXPORT_SYMBOL(ipipe_free_ptdkey); +EXPORT_SYMBOL(ipipe_set_ptd); +EXPORT_SYMBOL(ipipe_get_ptd); +EXPORT_SYMBOL(ipipe_set_irq_affinity); +EXPORT_SYMBOL(ipipe_send_ipi); +EXPORT_SYMBOL(__ipipe_pend_irq); +EXPORT_SYMBOL(__ipipe_set_irq_pending); +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) +EXPORT_SYMBOL(__ipipe_check_percpu_access); +#endif +#ifdef CONFIG_GENERIC_CLOCKEVENTS +EXPORT_SYMBOL(ipipe_request_tickdev); +EXPORT_SYMBOL(ipipe_release_tickdev); +#endif + +EXPORT_SYMBOL(ipipe_critical_enter); +EXPORT_SYMBOL(ipipe_critical_exit); +EXPORT_SYMBOL(ipipe_trigger_irq); +EXPORT_SYMBOL(ipipe_get_sysinfo); diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c new file mode 100644 index 0000000..d3c1866 --- /dev/null +++ b/kernel/ipipe/tracer.c @@ -0,0 +1,1441 @@ +/* -*- linux-c -*- + * kernel/ipipe/tracer.c + * + * Copyright (C) 2005 Luotao Fu. + * 2005-2008 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IPIPE_TRACE_PATHS 4 /* Do not lower below 3 */ +#define IPIPE_DEFAULT_ACTIVE 0 +#define IPIPE_DEFAULT_MAX 1 +#define IPIPE_DEFAULT_FROZEN 2 + +#define IPIPE_TRACE_POINTS (1 << CONFIG_IPIPE_TRACE_SHIFT) +#define WRAP_POINT_NO(point) ((point) & (IPIPE_TRACE_POINTS-1)) + +#define IPIPE_DEFAULT_PRE_TRACE 10 +#define IPIPE_DEFAULT_POST_TRACE 10 +#define IPIPE_DEFAULT_BACK_TRACE 100 + +#define IPIPE_DELAY_NOTE 1000 /* in nanoseconds */ +#define IPIPE_DELAY_WARN 10000 /* in nanoseconds */ + +#define IPIPE_TFLG_NMI_LOCK 0x0001 +#define IPIPE_TFLG_NMI_HIT 0x0002 +#define IPIPE_TFLG_NMI_FREEZE_REQ 0x0004 + +#define IPIPE_TFLG_HWIRQ_OFF 0x0100 +#define IPIPE_TFLG_FREEZING 0x0200 +#define IPIPE_TFLG_CURRDOM_SHIFT 10 /* bits 10..11: current domain */ +#define IPIPE_TFLG_CURRDOM_MASK 0x0C00 +#define IPIPE_TFLG_DOMSTATE_SHIFT 12 /* bits 12..15: domain stalled? */ +#define IPIPE_TFLG_DOMSTATE_BITS 3 + +#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \ + (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT))) +#define IPIPE_TFLG_CURRENT_DOMAIN(point) \ + ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT) + +struct ipipe_trace_point { + short type; + short flags; + unsigned long eip; + unsigned long parent_eip; + unsigned long v; + unsigned long long timestamp; +}; + +struct ipipe_trace_path { + volatile int flags; + int dump_lock; /* separated from flags due to cross-cpu access */ + int trace_pos; /* next point to fill */ + int begin, end; /* finalised path begin and end */ + int post_trace; /* non-zero when in post-trace phase */ + unsigned long long length; /* max path length in cycles */ + unsigned long nmi_saved_eip; /* for deferred requests from NMIs */ + unsigned long nmi_saved_parent_eip; + unsigned long nmi_saved_v; + struct ipipe_trace_point point[IPIPE_TRACE_POINTS]; +} ____cacheline_aligned_in_smp; + +enum ipipe_trace_type +{ + IPIPE_TRACE_FUNC = 0, + IPIPE_TRACE_BEGIN, + IPIPE_TRACE_END, + IPIPE_TRACE_FREEZE, + IPIPE_TRACE_SPECIAL, + IPIPE_TRACE_PID, + IPIPE_TRACE_EVENT, +}; + +#define IPIPE_TYPE_MASK 0x0007 +#define IPIPE_TYPE_BITS 3 + +#ifdef CONFIG_IPIPE_TRACE_VMALLOC +static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path); +#else /* !CONFIG_IPIPE_TRACE_VMALLOC */ +static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) = + { [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } }; +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + +int ipipe_trace_enable = 0; + +static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE }; +static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX }; +static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN }; +static IPIPE_DEFINE_SPINLOCK(global_path_lock); +static int pre_trace = IPIPE_DEFAULT_PRE_TRACE; +static int post_trace = IPIPE_DEFAULT_POST_TRACE; +static int back_trace = IPIPE_DEFAULT_BACK_TRACE; +static int verbose_trace = 1; +static unsigned long trace_overhead; + +static unsigned long trigger_begin; +static unsigned long trigger_end; + +static DEFINE_MUTEX(out_mutex); +static struct ipipe_trace_path *print_path; +#ifdef CONFIG_IPIPE_TRACE_PANIC +static struct ipipe_trace_path *panic_path; +#endif /* CONFIG_IPIPE_TRACE_PANIC */ +static int print_pre_trace; +static int print_post_trace; + + +static long __ipipe_signed_tsc2us(long long tsc); +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point); +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip); + + +static notrace void +__ipipe_store_domain_states(struct ipipe_trace_point *point) +{ + struct ipipe_domain *ipd; + struct list_head *pos; + int i = 0; + + list_for_each_prev(pos, &__ipipe_pipeline) { + ipd = list_entry(pos, struct ipipe_domain, p_link); + + if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status))) + point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT); + + if (ipd == __ipipe_current_domain) + point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT; + + if (++i > IPIPE_TFLG_DOMSTATE_BITS) + break; + } +} + +static notrace int __ipipe_get_free_trace_path(int old, int cpu) +{ + int new_active = old; + struct ipipe_trace_path *tp; + + do { + if (++new_active == IPIPE_TRACE_PATHS) + new_active = 0; + tp = &per_cpu(trace_path, cpu)[new_active]; + } while (new_active == per_cpu(max_path, cpu) || + new_active == per_cpu(frozen_path, cpu) || + tp->dump_lock); + + return new_active; +} + +static notrace void +__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp, + struct ipipe_trace_path *old_tp, int old_pos) +{ + int i; + + new_tp->trace_pos = pre_trace+1; + + for (i = new_tp->trace_pos; i > 0; i--) + memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)], + &old_tp->point[WRAP_POINT_NO(old_pos-i)], + sizeof(struct ipipe_trace_point)); + + /* mark the end (i.e. the point before point[0]) invalid */ + new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = per_cpu(active_path, cpu); + unsigned long long length; + + /* do we have a new worst case? */ + length = tp->point[tp->end].timestamp - + tp->point[tp->begin].timestamp; + if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) { + /* we need protection here against other cpus trying + to start a proc dump */ + spin_lock(&global_path_lock); + + /* active path holds new worst case */ + tp->length = length; + per_cpu(max_path, cpu) = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu); + + spin_unlock(&global_path_lock); + + tp = &per_cpu(trace_path, cpu)[active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + } + + return tp; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = per_cpu(active_path, cpu); + int n; + + /* frozen paths have no core (begin=end) */ + tp->begin = tp->end; + + /* we need protection here against other cpus trying + * to set their frozen path or to start a proc dump */ + spin_lock(&global_path_lock); + + per_cpu(frozen_path, cpu) = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu); + + /* check if this is the first frozen path */ + for_each_possible_cpu(n) { + if (n != cpu && + per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0) + tp->end = -1; + } + + spin_unlock(&global_path_lock); + + tp = &per_cpu(trace_path, cpu)[active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + + return tp; +} + +void notrace +__ipipe_trace(enum ipipe_trace_type type, unsigned long eip, + unsigned long parent_eip, unsigned long v) +{ + struct ipipe_trace_path *tp, *old_tp; + int pos, next_pos, begin; + struct ipipe_trace_point *point; + unsigned long flags; + int cpu; + + local_irq_save_hw_notrace(flags); + + cpu = ipipe_processor_id(); + restart: + tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + /* here starts a race window with NMIs - catched below */ + + /* check for NMI recursion */ + if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) { + tp->flags |= IPIPE_TFLG_NMI_HIT; + + /* first freeze request from NMI context? */ + if ((type == IPIPE_TRACE_FREEZE) && + !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) { + /* save arguments and mark deferred freezing */ + tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ; + tp->nmi_saved_eip = eip; + tp->nmi_saved_parent_eip = parent_eip; + tp->nmi_saved_v = v; + } + return; /* no need for restoring flags inside IRQ */ + } + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (unlikely(tp != + &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + /* get the point buffer */ + pos = tp->trace_pos; + point = &tp->point[pos]; + + /* store all trace point data */ + point->type = type; + point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0; + point->eip = eip; + point->parent_eip = parent_eip; + point->v = v; + ipipe_read_tsc(point->timestamp); + + __ipipe_store_domain_states(point); + + /* forward to next point buffer */ + next_pos = WRAP_POINT_NO(pos+1); + tp->trace_pos = next_pos; + + /* only mark beginning if we haven't started yet */ + begin = tp->begin; + if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0)) + tp->begin = pos; + + /* end of critical path, start post-trace if not already started */ + if (unlikely(type == IPIPE_TRACE_END) && + (begin >= 0) && !tp->post_trace) + tp->post_trace = post_trace + 1; + + /* freeze only if the slot is free and we are not already freezing */ + if ((unlikely(type == IPIPE_TRACE_FREEZE) || + (unlikely(eip >= trigger_begin && eip <= trigger_end) && + type == IPIPE_TRACE_FUNC)) && + per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 && + !(tp->flags & IPIPE_TFLG_FREEZING)) { + tp->post_trace = post_trace + 1; + tp->flags |= IPIPE_TFLG_FREEZING; + } + + /* enforce end of trace in case of overflow */ + if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) { + tp->end = pos; + goto enforce_end; + } + + /* stop tracing this path if we are in post-trace and + * a) that phase is over now or + * b) a new TRACE_BEGIN came in but we are not freezing this path */ + if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) || + ((type == IPIPE_TRACE_BEGIN) && + !(tp->flags & IPIPE_TFLG_FREEZING))))) { + /* store the path's end (i.e. excluding post-trace) */ + tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace); + + enforce_end: + if (tp->flags & IPIPE_TFLG_FREEZING) + tp = __ipipe_trace_freeze(cpu, tp, pos); + else + tp = __ipipe_trace_end(cpu, tp, pos); + + /* reset the active path, maybe already start a new one */ + tp->begin = (type == IPIPE_TRACE_BEGIN) ? + WRAP_POINT_NO(tp->trace_pos - 1) : -1; + tp->end = -1; + tp->post_trace = 0; + tp->flags = 0; + + /* update active_path not earlier to avoid races with NMIs */ + per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu); + } + + /* we still have old_tp and point, + * let's reset NMI lock and check for catches */ + old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) { + /* well, this late tagging may not immediately be visible for + * other cpus already dumping this path - a minor issue */ + point->flags |= IPIPE_TFLG_NMI_HIT; + + /* handle deferred freezing from NMI context */ + if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip, + old_tp->nmi_saved_parent_eip, + old_tp->nmi_saved_v); + } + + local_irq_restore_hw_notrace(flags); +} + +static unsigned long __ipipe_global_path_lock(void) +{ + unsigned long flags; + int cpu; + struct ipipe_trace_path *tp; + + spin_lock_irqsave(&global_path_lock, flags); + + cpu = ipipe_processor_id(); + restart: + tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + /* here is small race window with NMIs - catched below */ + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + return flags; +} + +static void __ipipe_global_path_unlock(unsigned long flags) +{ + int cpu; + struct ipipe_trace_path *tp; + + /* release spinlock first - it's not involved in the NMI issue */ + __ipipe_spin_unlock_irqbegin(&global_path_lock); + + cpu = ipipe_processor_id(); + tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* handle deferred freezing from NMI context */ + if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip, + tp->nmi_saved_parent_eip, tp->nmi_saved_v); + + /* See __ipipe_spin_lock_irqsave() and friends. */ + __ipipe_spin_unlock_irqcomplete(flags); +} + +void notrace ipipe_trace_begin(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_begin); + +void notrace ipipe_trace_end(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_end); + +void notrace ipipe_trace_freeze(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_freeze); + +void notrace ipipe_trace_special(unsigned char id, unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL(ipipe_trace_special); + +void notrace ipipe_trace_pid(pid_t pid, short prio) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, pid); +} +EXPORT_SYMBOL(ipipe_trace_pid); + +void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, delay_tsc); +} +EXPORT_SYMBOL(ipipe_trace_event); + +int ipipe_trace_max_reset(void) +{ + int cpu; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_possible_cpu(cpu) { + path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL(ipipe_trace_max_reset); + +int ipipe_trace_frozen_reset(void) +{ + int cpu; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_online_cpu(cpu) { + path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL(ipipe_trace_frozen_reset); + +static void +__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point, + int trylock) +{ + struct task_struct *task = NULL; + char buf[8]; + int i; + int locked = 1; + + if (trylock) { + if (!read_trylock(&tasklist_lock)) + locked = 0; + } else + read_lock(&tasklist_lock); + + if (locked) + task = find_task_by_pid_ns((pid_t)point->v, &init_pid_ns); + + if (task) + strncpy(task_info, task->comm, 11); + else + strcpy(task_info, "--"); + + if (locked) + read_unlock(&tasklist_lock); + + for (i = strlen(task_info); i < 11; i++) + task_info[i] = ' '; + + sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS); + strcpy(task_info + (11 - strlen(buf)), buf); +} + +static void +__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path, + struct ipipe_trace_point *point) +{ + long time; + int type; + + time = __ipipe_signed_tsc2us(point->timestamp - + path->point[path->begin].timestamp + point->v); + type = point->type >> IPIPE_TYPE_BITS; + + if (type == 0) + /* + * Event type #0 is predefined, stands for the next + * timer tick. + */ + sprintf(buf, "tick@%-6ld", time); + else + sprintf(buf, "%3d@%-7ld", type, time); +} + +#ifdef CONFIG_IPIPE_TRACE_PANIC +void ipipe_trace_panic_freeze(void) +{ + unsigned long flags; + int cpu; + + if (!ipipe_trace_enable) + return; + + ipipe_trace_enable = 0; + local_irq_save_hw_notrace(flags); + + cpu = ipipe_processor_id(); + + panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + local_irq_restore_hw(flags); +} +EXPORT_SYMBOL(ipipe_trace_panic_freeze); + +void ipipe_trace_panic_dump(void) +{ + int cnt = back_trace; + int start, pos; + char buf[16]; + + if (!panic_path) + return; + + ipipe_context_check_off(); + + printk("I-pipe tracer log (%d points):\n", cnt); + + start = pos = WRAP_POINT_NO(panic_path->trace_pos-1); + + while (cnt-- > 0) { + struct ipipe_trace_point *point = &panic_path->point[pos]; + long time; + char info[16]; + int i; + + printk(" %c", + (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' '); + + for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--) + printk("%c", + (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '#' : '+') : + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '*' : ' ')); + + if (!point->eip) + printk("--\n"); + else { + __ipipe_trace_point_type(buf, point); + printk("%s", buf); + + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + printk(" "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(info, + point, 1); + printk("%s", info); + break; + + case IPIPE_TRACE_EVENT: + __ipipe_get_event_date(info, + panic_path, point); + printk("%s", info); + break; + + default: + printk("0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + panic_path->point[start].timestamp); + printk(" %5ld ", time); + + __ipipe_print_symname(NULL, point->eip); + printk(" ("); + __ipipe_print_symname(NULL, point->parent_eip); + printk(")\n"); + } + pos = WRAP_POINT_NO(pos - 1); + } + + panic_path = NULL; +} +EXPORT_SYMBOL(ipipe_trace_panic_dump); +#endif /* CONFIG_IPIPE_TRACE_PANIC */ + + +/* --- /proc output --- */ + +static notrace int __ipipe_in_critical_trpath(long point_no) +{ + return ((WRAP_POINT_NO(point_no-print_path->begin) < + WRAP_POINT_NO(print_path->end-print_path->begin)) || + ((print_path->end == print_path->begin) && + (WRAP_POINT_NO(point_no-print_path->end) > + print_post_trace))); +} + +static long __ipipe_signed_tsc2us(long long tsc) +{ + unsigned long long abs_tsc; + long us; + + /* ipipe_tsc2us works on unsigned => handle sign separately */ + abs_tsc = (tsc >= 0) ? tsc : -tsc; + us = ipipe_tsc2us(abs_tsc); + if (tsc < 0) + return -us; + else + return us; +} + +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point) +{ + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + strcpy(buf, "func "); + break; + + case IPIPE_TRACE_BEGIN: + strcpy(buf, "begin "); + break; + + case IPIPE_TRACE_END: + strcpy(buf, "end "); + break; + + case IPIPE_TRACE_FREEZE: + strcpy(buf, "freeze "); + break; + + case IPIPE_TRACE_SPECIAL: + sprintf(buf, "(0x%02x) ", + point->type >> IPIPE_TYPE_BITS); + break; + + case IPIPE_TRACE_PID: + sprintf(buf, "[%5d] ", (pid_t)point->v); + break; + + case IPIPE_TRACE_EVENT: + sprintf(buf, "event "); + break; + } +} + +static void +__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point) +{ + char mark = ' '; + int point_no = point - print_path->point; + int i; + + if (print_path->end == point_no) + mark = '<'; + else if (print_path->begin == point_no) + mark = '>'; + else if (__ipipe_in_critical_trpath(point_no)) + mark = ':'; + seq_printf(m, "%c%c", mark, + (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' '); + + if (!verbose_trace) + return; + + for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--) + seq_printf(m, "%c", + (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '#' : '+') : + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' ')); +} + +static void +__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point) +{ + unsigned long delay = 0; + int next; + char *mark = " "; + + next = WRAP_POINT_NO(point+1 - print_path->point); + + if (next != print_path->trace_pos) + delay = ipipe_tsc2ns(print_path->point[next].timestamp - + point->timestamp); + + if (__ipipe_in_critical_trpath(point - print_path->point)) { + if (delay > IPIPE_DELAY_WARN) + mark = "! "; + else if (delay > IPIPE_DELAY_NOTE) + mark = "+ "; + } + seq_puts(m, mark); + + if (verbose_trace) + seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000, + (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' '); + else + seq_puts(m, " "); +} + +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip) +{ + char namebuf[KSYM_NAME_LEN+1]; + unsigned long size, offset; + const char *sym_name; + char *modname; + + sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf); + +#ifdef CONFIG_IPIPE_TRACE_PANIC + if (!m) { + /* panic dump */ + if (sym_name) { + printk("%s+0x%lx", sym_name, offset); + if (modname) + printk(" [%s]", modname); + } + } else +#endif /* CONFIG_IPIPE_TRACE_PANIC */ + { + if (sym_name) { + if (verbose_trace) { + seq_printf(m, "%s+0x%lx", sym_name, offset); + if (modname) + seq_printf(m, " [%s]", modname); + } else + seq_puts(m, sym_name); + } else + seq_printf(m, "<%08lx>", eip); + } +} + +static void __ipipe_print_headline(struct seq_file *m) +{ + seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu " + "us\n\n", trace_overhead/1000, trace_overhead%1000); + + if (verbose_trace) { + const char *name[4] = { [0 ... 3] = "" }; + struct list_head *pos; + int i = 0; + + list_for_each_prev(pos, &__ipipe_pipeline) { + struct ipipe_domain *ipd = + list_entry(pos, struct ipipe_domain, p_link); + + name[i] = ipd->name; + if (++i > 3) + break; + } + + seq_printf(m, + " +----- Hard IRQs ('|': locked)\n" + " |+---- %s\n" + " ||+--- %s\n" + " |||+-- %s\n" + " ||||+- %s%s\n" + " ||||| +---------- " + "Delay flag ('+': > %d us, '!': > %d us)\n" + " ||||| | +- " + "NMI noise ('N')\n" + " ||||| | |\n" + " Type User Val. Time Delay Function " + "(Parent)\n", + name[3], name[2], name[1], name[0], + name[0] ? " ('*': domain stalled, '+': current, " + "'#': current+stalled)" : "", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); + } else + seq_printf(m, + " +--------------- Hard IRQs ('|': locked)\n" + " | +- Delay flag " + "('+': > %d us, '!': > %d us)\n" + " | |\n" + " Type Time Function (Parent)\n", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); +} + +static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + mutex_lock(&out_mutex); + + if (!n) { + struct ipipe_trace_path *tp; + unsigned long length_usecs; + int points, cpu; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the longest of all per-cpu paths */ + print_path = NULL; + for_each_online_cpu(cpu) { + tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)]; + if ((print_path == NULL) || + (tp->length > print_path->length)) { + print_path = tp; + break; + } + } + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + /* does this path actually contain data? */ + if (print_path->end == print_path->begin) + return NULL; + + /* number of points inside the critical path */ + points = WRAP_POINT_NO(print_path->end-print_path->begin+1); + + /* pre- and post-tracing length, post-trace length was frozen + in __ipipe_trace, pre-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = pre_trace; + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 1 - points - + print_post_trace; + + length_usecs = ipipe_tsc2us(print_path->length); + seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n" + "------------------------------------------------------------\n", + UTS_RELEASE, IPIPE_ARCH_STRING); + seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: " + "%d (-%d/+%d), Length: %lu us\n", + cpu, print_path->point[print_path->begin].timestamp, + points, print_pre_trace, print_post_trace, length_usecs); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + n)]; +} + +static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos) +{ + loff_t n = ++*pos; + + /* check if we are inside the trace range with the next entry */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + *pos)]; +} + +static void __ipipe_prtrace_stop(struct seq_file *m, void *p) +{ + if (print_path) + print_path->dump_lock = 0; + mutex_unlock(&out_mutex); +} + +static int __ipipe_prtrace_show(struct seq_file *m, void *p) +{ + long time; + struct ipipe_trace_point *point = p; + char buf[16]; + + if (!point->eip) { + seq_puts(m, "--\n"); + return 0; + } + + __ipipe_print_pathmark(m, point); + __ipipe_trace_point_type(buf, point); + seq_puts(m, buf); + if (verbose_trace) + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + seq_puts(m, " "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(buf, point, 0); + seq_puts(m, buf); + break; + + case IPIPE_TRACE_EVENT: + __ipipe_get_event_date(buf, print_path, point); + seq_puts(m, buf); + break; + + default: + seq_printf(m, "0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + print_path->point[print_path->begin].timestamp); + seq_printf(m, "%5ld", time); + + __ipipe_print_delay(m, point); + __ipipe_print_symname(m, point->eip); + seq_puts(m, " ("); + __ipipe_print_symname(m, point->parent_eip); + seq_puts(m, ")\n"); + + return 0; +} + +static struct seq_operations __ipipe_max_ptrace_ops = { + .start = __ipipe_max_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_max_ptrace_ops); +} + +static ssize_t +__ipipe_max_reset(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + mutex_lock(&out_mutex); + ipipe_trace_max_reset(); + mutex_unlock(&out_mutex); + + return count; +} + +struct file_operations __ipipe_max_prtrace_fops = { + .open = __ipipe_max_prtrace_open, + .read = seq_read, + .write = __ipipe_max_reset, + .llseek = seq_lseek, + .release = seq_release, +}; + +static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + mutex_lock(&out_mutex); + + if (!n) { + struct ipipe_trace_path *tp; + int cpu; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the first of all per-cpu frozen paths */ + print_path = NULL; + for_each_online_cpu(cpu) { + tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)]; + if (tp->end >= 0) { + print_path = tp; + break; + } + } + if (print_path) + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + if (!print_path) + return NULL; + + /* back- and post-tracing length, post-trace length was frozen + in __ipipe_trace, back-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = back_trace-1; /* substract freeze point */ + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 2 - + print_post_trace; + + seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n" + "------------------------------------------------------" + "------\n", + UTS_RELEASE, IPIPE_ARCH_STRING); + seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n", + cpu, print_path->point[print_path->begin].timestamp, + print_pre_trace+1, print_post_trace); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= print_pre_trace + 1 + print_post_trace) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin- + print_pre_trace+n)]; +} + +static struct seq_operations __ipipe_frozen_ptrace_ops = { + .start = __ipipe_frozen_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_frozen_ptrace_ops); +} + +static ssize_t +__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, pbuffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + ipipe_trace_frozen_reset(); + if (val > 0) + ipipe_trace_freeze(-1); + mutex_unlock(&out_mutex); + + return count; +} + +struct file_operations __ipipe_frozen_prtrace_fops = { + .open = __ipipe_frozen_prtrace_open, + .read = seq_read, + .write = __ipipe_frozen_ctrl, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __ipipe_rd_proc_val(char *page, char **start, off_t off, + int count, int *eof, void *data) +{ + int len; + + len = sprintf(page, "%u\n", *(int *)data); + len -= off; + if (len <= off + count) + *eof = 1; + *start = page + off; + if (len > count) + len = count; + if (len < 0) + len = 0; + + return len; +} + +static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, buffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + *(int *)data = val; + mutex_unlock(&out_mutex); + + return count; +} + +static int __ipipe_rd_trigger(char *page, char **start, off_t off, int count, + int *eof, void *data) +{ + int len; + + if (!trigger_begin) + return 0; + + len = sprint_symbol(page, trigger_begin); + page[len++] = '\n'; + + len -= off; + if (len <= off + count) + *eof = 1; + *start = page + off; + if (len > count) + len = count; + if (len < 0) + len = 0; + + return len; +} + +static int __ipipe_wr_trigger(struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + char buf[KSYM_SYMBOL_LEN]; + unsigned long begin, end; + + if (count > sizeof(buf) - 1) + count = sizeof(buf) - 1; + if (copy_from_user(buf, buffer, count)) + return -EFAULT; + buf[count] = 0; + if (buf[count-1] == '\n') + buf[count-1] = 0; + + begin = kallsyms_lookup_name(buf); + if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL)) + return -ENOENT; + end += begin - 1; + + mutex_lock(&out_mutex); + /* invalidate the current range before setting a new one */ + trigger_end = 0; + wmb(); + ipipe_trace_frozen_reset(); + + /* set new range */ + trigger_begin = begin; + wmb(); + trigger_end = end; + mutex_unlock(&out_mutex); + + return count; +} + +#ifdef CONFIG_IPIPE_TRACE_MCOUNT +static void notrace +ipipe_trace_function(unsigned long ip, unsigned long parent_ip) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_FUNC, ip, parent_ip, 0); +} + +static struct ftrace_ops ipipe_trace_ops = { + .func = ipipe_trace_function +}; + +static int __ipipe_wr_enable(struct file *file, const char __user *buffer, + unsigned long count, void *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, buffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + + if (ipipe_trace_enable) { + if (!val) + unregister_ftrace_function(&ipipe_trace_ops); + } else if (val) + register_ftrace_function(&ipipe_trace_ops); + + ipipe_trace_enable = val; + + mutex_unlock(&out_mutex); + + return count; +} +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ + +extern struct proc_dir_entry *ipipe_proc_root; + +static struct proc_dir_entry * __init +__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir, + const char *name, int *value_ptr) +{ + struct proc_dir_entry *entry; + + entry = create_proc_entry(name, 0644, trace_dir); + if (entry) { + entry->data = value_ptr; + entry->read_proc = __ipipe_rd_proc_val; + entry->write_proc = __ipipe_wr_proc_val; + } + return entry; +} + +void __init __ipipe_init_tracer(void) +{ + struct proc_dir_entry *trace_dir; + struct proc_dir_entry *entry; + unsigned long long start, end, min = ULLONG_MAX; + int i; +#ifdef CONFIG_IPIPE_TRACE_VMALLOC + int cpu, path; + + for_each_possible_cpu(cpu) { + struct ipipe_trace_path *tp_buf; + + tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) * + IPIPE_TRACE_PATHS, cpu_to_node(cpu)); + if (!tp_buf) { + printk(KERN_ERR "I-pipe: " + "insufficient memory for trace buffer.\n"); + return; + } + memset(tp_buf, 0, + sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS); + for (path = 0; path < IPIPE_TRACE_PATHS; path++) { + tp_buf[path].begin = -1; + tp_buf[path].end = -1; + } + per_cpu(trace_path, cpu) = tp_buf; + } +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + + /* Calculate minimum overhead of __ipipe_trace() */ + local_irq_disable_hw(); + for (i = 0; i < 100; i++) { + ipipe_read_tsc(start); + __ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, 0); + ipipe_read_tsc(end); + + end -= start; + if (end < min) + min = end; + } + local_irq_enable_hw(); + trace_overhead = ipipe_tsc2ns(min); + +#ifdef CONFIG_IPIPE_TRACE_ENABLE + ipipe_trace_enable = 1; +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + register_ftrace_function(&ipipe_trace_ops); +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ +#endif /* CONFIG_IPIPE_TRACE_ENABLE */ + + trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root); + + entry = create_proc_entry("max", 0644, trace_dir); + if (entry) + entry->proc_fops = &__ipipe_max_prtrace_fops; + + entry = create_proc_entry("frozen", 0644, trace_dir); + if (entry) + entry->proc_fops = &__ipipe_frozen_prtrace_fops; + + entry = create_proc_entry("trigger", 0644, trace_dir); + if (entry) { + entry->read_proc = __ipipe_rd_trigger; + entry->write_proc = __ipipe_wr_trigger; + } + + __ipipe_create_trace_proc_val(trace_dir, "pre_trace_points", + &pre_trace); + __ipipe_create_trace_proc_val(trace_dir, "post_trace_points", + &post_trace); + __ipipe_create_trace_proc_val(trace_dir, "back_trace_points", + &back_trace); + __ipipe_create_trace_proc_val(trace_dir, "verbose", + &verbose_trace); + entry = __ipipe_create_trace_proc_val(trace_dir, "enable", + &ipipe_trace_enable); +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + if (entry) + entry->write_proc = __ipipe_wr_enable; +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ +} diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 13c68e7..f4fbfd4 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -358,7 +358,9 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) irqreturn_t action_ret; spin_lock(&desc->lock); +#ifndef CONFIG_IPIPE mask_ack_irq(desc, irq); +#endif if (unlikely(desc->status & IRQ_INPROGRESS)) goto out_unlock; @@ -435,8 +437,13 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) spin_lock(&desc->lock); desc->status &= ~IRQ_INPROGRESS; +#ifdef CONFIG_IPIPE + desc->chip->unmask(irq); +out: +#else out: desc->chip->eoi(irq); +#endif spin_unlock(&desc->lock); } @@ -478,8 +485,10 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc) kstat_incr_irqs_this_cpu(irq, desc); /* Start handling the irq */ +#ifndef CONFIG_IPIPE if (desc->chip->ack) desc->chip->ack(irq); +#endif /* Mark the IRQ currently in progress.*/ desc->status |= IRQ_INPROGRESS; @@ -519,6 +528,85 @@ out_unlock: spin_unlock(&desc->lock); } +#ifdef CONFIG_IPIPE + +void __ipipe_ack_simple_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void __ipipe_end_simple_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc) +{ + mask_ack_irq(desc, irq); +} + +void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->unmask) + desc->chip->unmask(irq); +} + +void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ + desc->chip->eoi(irq); +} + +void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ + /* + * Non-requestable IRQs should not be masked in EOI handler. + */ + if (!(desc->status & IRQ_NOREQUEST)) + desc->chip->unmask(irq); +} + +void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc) +{ + desc->chip->ack(irq); +} + +void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->ack) + desc->chip->ack(irq); +} + +void __ipipe_end_percpu_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->chip->eoi) + desc->chip->eoi(irq); +} + +void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc) +{ + static int done; + + handle_bad_irq(irq, desc); + + if (!done) { + printk(KERN_WARNING "%s: unknown flow handler for IRQ %d\n", + __FUNCTION__, irq); + done = 1; + } +} + +void __ipipe_noack_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void __ipipe_noend_irq(unsigned irq, struct irq_desc *desc) +{ +} + +#endif /* CONFIG_IPIPE */ + /** * handle_percpu_IRQ - Per CPU local irq handler * @irq: the interrupt number @@ -533,8 +621,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) kstat_incr_irqs_this_cpu(irq, desc); +#ifndef CONFIG_IPIPE if (desc->chip->ack) desc->chip->ack(irq); +#endif /* CONFIG_IPIPE */ action_ret = handle_IRQ_event(irq, desc->action); if (!noirqdebug) @@ -545,20 +635,35 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) } void -__set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, - const char *name) +___set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, + const char *name) { struct irq_desc *desc = irq_to_desc(irq); - unsigned long flags; - - if (!desc) { - printk(KERN_ERR - "Trying to install type control for IRQ%d\n", irq); - return; - } if (!handle) handle = handle_bad_irq; +#ifdef CONFIG_IPIPE + else if (handle == &handle_simple_irq) { + desc->ipipe_ack = &__ipipe_ack_simple_irq; + desc->ipipe_end = &__ipipe_end_simple_irq; + } + else if (handle == &handle_level_irq) { + desc->ipipe_ack = &__ipipe_ack_level_irq; + desc->ipipe_end = &__ipipe_end_level_irq; + } + else if (handle == &handle_edge_irq) { + desc->ipipe_ack = &__ipipe_ack_edge_irq; + desc->ipipe_end = &__ipipe_end_edge_irq; + } + else if (handle == &handle_fasteoi_irq) { + desc->ipipe_ack = &__ipipe_ack_fasteoi_irq; + desc->ipipe_end = &__ipipe_end_fasteoi_irq; + } + else if (handle == &handle_percpu_irq) { + desc->ipipe_ack = &__ipipe_ack_percpu_irq; + desc->ipipe_end = &__ipipe_end_percpu_irq; + } +#endif /* CONFIG_IPIPE */ else if (desc->chip == &no_irq_chip) { printk(KERN_WARNING "Trying to install %sinterrupt handler " "for IRQ%d\n", is_chained ? "chained " : "", irq); @@ -570,9 +675,21 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, * dummy_irq_chip for easy transition. */ desc->chip = &dummy_irq_chip; +#ifdef CONFIG_IPIPE + desc->ipipe_ack = &__ipipe_noack_irq; + desc->ipipe_end = &__ipipe_noend_irq; +#endif /* CONFIG_IPIPE */ } - - spin_lock_irqsave(&desc->lock, flags); +#ifdef CONFIG_IPIPE + else if (is_chained) { + desc->ipipe_ack = handle; + desc->ipipe_end = &__ipipe_noend_irq; + handle = &__ipipe_noack_irq; + } else { + desc->ipipe_ack = &__ipipe_ack_bad_irq; + desc->ipipe_end = &__ipipe_noend_irq; + } +#endif /* CONFIG_IPIPE */ /* Uninstall? */ if (handle == handle_bad_irq) { @@ -580,9 +697,17 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, mask_ack_irq(desc, irq); desc->status |= IRQ_DISABLED; desc->depth = 1; +#ifdef CONFIG_IPIPE + desc->ipipe_ack = &__ipipe_ack_bad_irq; + desc->ipipe_end = &__ipipe_noend_irq; +#endif /* CONFIG_IPIPE */ } desc->handle_irq = handle; desc->name = name; +#ifdef CONFIG_IPIPE + /* Suppress intermediate trampoline routine. */ + ipipe_root_domain->irqs[irq].acknowledge = desc->ipipe_ack; +#endif /* CONFIG_IPIPE */ if (handle != handle_bad_irq && is_chained) { desc->status &= ~IRQ_DISABLED; @@ -590,10 +715,28 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, desc->depth = 0; desc->chip->startup(irq); } +} + +void +__set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, + const char *name) +{ + struct irq_desc *desc = irq_to_desc(irq); + unsigned long flags; + + if (!desc) { + printk(KERN_ERR + "Trying to install type control for IRQ%d\n", irq); + return; + } + + spin_lock_irqsave(&desc->lock, flags); + ___set_irq_handler(irq, handle, is_chained, name); spin_unlock_irqrestore(&desc->lock, flags); } EXPORT_SYMBOL_GPL(__set_irq_handler); + void set_irq_chip_and_handler(unsigned int irq, struct irq_chip *chip, irq_flow_handler_t handle) diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 065205b..ceedb90 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -458,8 +458,10 @@ unsigned int __do_IRQ(unsigned int irq) /* * No locking required for CPU-local interrupts: */ +#ifndef CONFIG_IPIPE if (desc->chip->ack) desc->chip->ack(irq); +#endif if (likely(!(desc->status & IRQ_DISABLED))) { action_ret = handle_IRQ_event(irq, desc->action); if (!noirqdebug) @@ -470,8 +472,10 @@ unsigned int __do_IRQ(unsigned int irq) } spin_lock(&desc->lock); +#ifndef CONFIG_IPIPE if (desc->chip->ack) desc->chip->ack(irq); +#endif /* * REPLAY is when Linux resends an IRQ that was dropped earlier * WAITING is used by probe to mark irqs that are being tested diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 8bbeef9..021043d 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -2135,7 +2135,7 @@ void trace_hardirqs_on_caller(unsigned long ip) /* we'll do an OFF -> ON transition: */ curr->hardirqs_enabled = 1; - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw())) return; if (DEBUG_LOCKS_WARN_ON(current->hardirq_context)) return; @@ -2178,7 +2178,7 @@ void trace_hardirqs_off_caller(unsigned long ip) if (unlikely(!debug_locks || current->lockdep_recursion)) return; - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw())) return; if (curr->hardirqs_enabled) { diff --git a/kernel/panic.c b/kernel/panic.c index 512ab73..1469357 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -22,6 +22,7 @@ #include #include #include +#include int panic_on_oops; static unsigned long tainted_mask; @@ -303,6 +304,8 @@ void oops_enter(void) { tracing_off(); /* can't trust the integrity of the kernel anymore: */ + ipipe_trace_panic_freeze(); + ipipe_disable_context_check(ipipe_processor_id()); debug_locks_off(); do_oops_enter_exit(); } diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 81d2e74..e8d5ca3 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -238,6 +238,7 @@ static int create_image(int platform_mode) goto Enable_cpus; local_irq_disable(); + local_irq_disable_hw_cond(); error = sysdev_suspend(PMSG_FREEZE); if (error) { @@ -267,6 +268,7 @@ static int create_image(int platform_mode) */ Enable_irqs: + local_irq_enable_hw_cond(); local_irq_enable(); Enable_cpus: @@ -355,6 +357,7 @@ static int resume_target_kernel(bool platform_mode) goto Enable_cpus; local_irq_disable(); + local_irq_disable_hw_cond(); error = sysdev_suspend(PMSG_QUIESCE); if (error) @@ -386,6 +389,7 @@ static int resume_target_kernel(bool platform_mode) sysdev_resume(); Enable_irqs: + local_irq_enable_hw_cond(); local_irq_enable(); Enable_cpus: @@ -467,6 +471,7 @@ int hibernation_platform_enter(void) goto Platofrm_finish; local_irq_disable(); + local_irq_disable_hw_cond(); sysdev_suspend(PMSG_HIBERNATE); hibernation_ops->enter(); /* We should never get here */ diff --git a/kernel/printk.c b/kernel/printk.c index b4d97b5..fad9fee 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -551,6 +551,41 @@ static int have_callable_console(void) return 0; } +#ifdef CONFIG_IPIPE + +static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED; + +static int __ipipe_printk_fill; + +static char __ipipe_printk_buf[__LOG_BUF_LEN]; + +void __ipipe_flush_printk (unsigned virq, void *cookie) +{ + char *p = __ipipe_printk_buf; + int len, lmax, out = 0; + unsigned long flags; + + goto start; + + do { + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + start: + lmax = __ipipe_printk_fill; + while (out < lmax) { + len = strlen(p) + 1; + printk("%s",p); + p += len; + out += len; + } + spin_lock_irqsave(&__ipipe_printk_lock, flags); + } + while (__ipipe_printk_fill != lmax); + + __ipipe_printk_fill = 0; + + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); +} + /** * printk - print a kernel message * @fmt: format string @@ -575,6 +610,65 @@ static int have_callable_console(void) asmlinkage int printk(const char *fmt, ...) { + int r, fbytes, oldcount; + unsigned long flags; + int sprintk = 1; + int cs = -1; + va_list args; + + va_start(args, fmt); + + local_irq_save_hw(flags); + + if (test_bit(IPIPE_SPRINTK_FLAG, &__ipipe_current_domain->flags) || + oops_in_progress) + cs = ipipe_disable_context_check(ipipe_processor_id()); + else if (__ipipe_current_domain == ipipe_root_domain) { + struct ipipe_domain *dom; + + list_for_each_entry(dom, &__ipipe_pipeline, p_link) { + if (dom == ipipe_root_domain) + break; + if (test_bit(IPIPE_STALL_FLAG, + &ipipe_cpudom_var(dom, status))) + sprintk = 0; + } + } else + sprintk = 0; + + local_irq_restore_hw(flags); + + if (sprintk) { + r = vprintk(fmt, args); + if (cs != -1) + ipipe_restore_context_check(ipipe_processor_id(), cs); + goto out; + } + + spin_lock_irqsave(&__ipipe_printk_lock, flags); + + oldcount = __ipipe_printk_fill; + fbytes = __LOG_BUF_LEN - oldcount; + + if (fbytes > 1) { + r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill, + fbytes, fmt, args) + 1; /* account for the null byte */ + __ipipe_printk_fill += r; + } else + r = 0; + + spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + + if (oldcount == 0) + ipipe_trigger_irq(__ipipe_printk_virq); +out: + va_end(args); + + return r; +} +#else /* !CONFIG_IPIPE */ +asmlinkage int printk(const char *fmt, ...) +{ va_list args; int r; @@ -584,6 +678,7 @@ asmlinkage int printk(const char *fmt, ...) return r; } +#endif /* CONFIG_IPIPE */ /* cpu currently holding logbuf_lock */ static volatile unsigned int printk_cpu = UINT_MAX; diff --git a/kernel/sched.c b/kernel/sched.c index 1b59e26..4bc5823 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2442,7 +2442,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync) rq = task_rq_lock(p, &flags); update_rq_clock(rq); old_state = p->state; - if (!(old_state & state)) + if (!(old_state & state) || (old_state & (TASK_NOWAKEUP|TASK_ATOMICSWITCH))) goto out; if (p->se.on_rq) @@ -2854,13 +2854,15 @@ asmlinkage void schedule_tail(struct task_struct *prev) #endif if (current->set_child_tid) put_user(task_pid_vnr(current), current->set_child_tid); + + ipipe_init_notify(current); } /* * context_switch - switch to the new MM and the new * thread's register state. */ -static inline void +static inline int context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { @@ -2902,12 +2904,23 @@ context_switch(struct rq *rq, struct task_struct *prev, switch_to(prev, next, prev); barrier(); + +#ifdef CONFIG_IPIPE_DELAYED_ATOMICSW + current->state &= ~TASK_ATOMICSWITCH; +#else + prev->state &= ~TASK_ATOMICSWITCH; +#endif + if (task_hijacked(prev)) + return 1; + /* * this_rq must be evaluated again because prev may have moved * CPUs since it called schedule(), thus the 'rq' on its stack * frame will be invalid. */ finish_task_switch(this_rq(), prev); + + return 0; } /* @@ -5168,6 +5181,7 @@ notrace unsigned long get_parent_ip(unsigned long addr) void __kprobes add_preempt_count(int val) { + ipipe_check_context(ipipe_root_domain); #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? @@ -5190,6 +5204,7 @@ EXPORT_SYMBOL(add_preempt_count); void __kprobes sub_preempt_count(int val) { + ipipe_check_context(ipipe_root_domain); #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? @@ -5238,6 +5253,7 @@ static noinline void __schedule_bug(struct task_struct *prev) */ static inline void schedule_debug(struct task_struct *prev) { + ipipe_check_context(ipipe_root_domain); /* * Test if we are atomic. Since do_exit() needs to call into * schedule() atomically, we ignore that path for now. @@ -5314,7 +5330,7 @@ pick_next_task(struct rq *rq) /* * schedule() is the main scheduler function. */ -asmlinkage void __sched schedule(void) +asmlinkage int __sched schedule(void) { struct task_struct *prev, *next; unsigned long *switch_count; @@ -5328,6 +5344,9 @@ need_resched: rcu_qsctr_inc(cpu); prev = rq->curr; switch_count = &prev->nivcsw; + if (unlikely(prev->state & TASK_ATOMICSWITCH)) + /* Pop one disable level -- one still remains. */ + preempt_enable(); release_kernel_lock(prev); need_resched_nonpreemptible: @@ -5368,15 +5387,18 @@ need_resched_nonpreemptible: rq->curr = next; ++*switch_count; - context_switch(rq, prev, next); /* unlocks the rq */ + if (context_switch(rq, prev, next)) /* unlocks the rq */ + return 1; /* task hijacked by higher domain */ /* * the context switch might have flipped the stack from under * us, hence refresh the local variables. */ cpu = smp_processor_id(); rq = cpu_rq(cpu); - } else + } else { + prev->state &= ~TASK_ATOMICSWITCH; spin_unlock_irq(&rq->lock); + } if (unlikely(reacquire_kernel_lock(current) < 0)) goto need_resched_nonpreemptible; @@ -5384,6 +5406,8 @@ need_resched_nonpreemptible: preempt_enable_no_resched(); if (need_resched()) goto need_resched; + + return 0; } EXPORT_SYMBOL(schedule); @@ -6222,6 +6246,7 @@ recheck: oldprio = p->prio; __setscheduler(rq, p, policy, param->sched_priority); + ipipe_setsched_notify(p); if (running) p->sched_class->set_curr_task(rq); @@ -6574,6 +6599,7 @@ static void __cond_resched(void) #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP __might_sleep(__FILE__, __LINE__); #endif + ipipe_check_context(ipipe_root_domain); /* * The BKS might be reacquired before we have dropped * PREEMPT_ACTIVE, which could trigger a second @@ -10581,3 +10607,58 @@ struct cgroup_subsys cpuacct_subsys = { .subsys_id = cpuacct_subsys_id, }; #endif /* CONFIG_CGROUP_CPUACCT */ + +#ifdef CONFIG_IPIPE + +int ipipe_setscheduler_root(struct task_struct *p, int policy, int prio) +{ + const struct sched_class *prev_class = p->sched_class; + int oldprio, on_rq, running; + unsigned long flags; + struct rq *rq; + + spin_lock_irqsave(&p->pi_lock, flags); + rq = __task_rq_lock(p); + update_rq_clock(rq); + on_rq = p->se.on_rq; + running = task_running(rq, p); + + if (on_rq) + deactivate_task(rq, p, 0); + if (running) + p->sched_class->put_prev_task(rq, p); + + oldprio = p->prio; + __setscheduler(rq, p, policy, prio); + ipipe_setsched_notify(p); + + if (running) + p->sched_class->set_curr_task(rq); + if (on_rq) { + activate_task(rq, p, 0); + check_class_changed(rq, p, prev_class, oldprio, running); + } + __task_rq_unlock(rq); + spin_unlock_irqrestore(&p->pi_lock, flags); + + rt_mutex_adjust_pi(p); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_setscheduler_root); + +int ipipe_reenter_root(struct task_struct *prev, int policy, int prio) +{ + finish_task_switch(this_rq(), prev); + + (void)reacquire_kernel_lock(current); + preempt_enable_no_resched(); + + if (current->policy != policy || current->rt_priority != prio) + return ipipe_setscheduler_root(current, policy, prio); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_reenter_root); + +#endif /* CONFIG_IPIPE */ diff --git a/kernel/signal.c b/kernel/signal.c index 64c5dee..cbc5a62 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -518,6 +518,7 @@ void signal_wake_up(struct task_struct *t, int resume) unsigned int mask; set_tsk_thread_flag(t, TIF_SIGPENDING); + ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */ /* * For SIGKILL, we want to wake it up in the stopped/traced/killable diff --git a/kernel/spinlock.c b/kernel/spinlock.c index 7932653..517d599 100644 --- a/kernel/spinlock.c +++ b/kernel/spinlock.c @@ -87,7 +87,7 @@ unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock) * _raw_spin_lock_flags() code, because lockdep assumes * that interrupts are not re-enabled during lock-acquire: */ -#ifdef CONFIG_LOCKDEP +#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE) LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock); #else _raw_spin_lock_flags(lock, &flags); diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 83c4417..782a209 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -69,7 +69,7 @@ static void tick_periodic(int cpu) write_sequnlock(&xtime_lock); } - update_process_times(user_mode(get_irq_regs())); + update_root_process_times(get_irq_regs()); profile_tick(CPU_PROFILING); } @@ -177,6 +177,10 @@ static void tick_setup_device(struct tick_device *td, td->evtdev = newdev; + /* I-pipe: derive global tick IRQ from CPU 0 */ + if (cpu == 0) + ipipe_update_tick_evtdev(newdev); + /* * When the device is not per cpu, pin the interrupt to the * current cpu: diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index e0f59a2..0132f56 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -520,7 +520,7 @@ static void tick_nohz_handler(struct clock_event_device *dev) ts->idle_jiffies++; } - update_process_times(user_mode(regs)); + update_root_process_times(regs); profile_tick(CPU_PROFILING); while (tick_nohz_reprogram(ts, now)) { @@ -671,7 +671,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer) touch_softlockup_watchdog(); ts->idle_jiffies++; } - update_process_times(user_mode(regs)); + update_root_process_times(regs); profile_tick(CPU_PROFILING); } diff --git a/kernel/timer.c b/kernel/timer.c index a7f07d5..9bebec5 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -1163,6 +1163,26 @@ void update_process_times(int user_tick) run_posix_cpu_timers(p); } +#ifdef CONFIG_IPIPE + +void update_root_process_times(struct pt_regs *regs) +{ + int cpu, user_tick = user_mode(regs); + + if (__ipipe_root_tick_p(regs)) { + update_process_times(user_tick); + return; + } + + run_local_timers(); + cpu = smp_processor_id(); + if (rcu_pending(cpu)) + rcu_check_callbacks(cpu, user_tick); + run_posix_cpu_timers(current); +} + +#endif + /* * This function runs timers and the timer-tq in bottom half context. */ diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 019f380..1d65eb3 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -415,6 +415,7 @@ config DYNAMIC_FTRACE bool "enable/disable ftrace tracepoints dynamically" depends on FUNCTION_TRACER depends on HAVE_DYNAMIC_FTRACE + depends on !IPIPE_TRACE_MCOUNT default y help This option will modify all the calls to ftrace dynamically diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index 25edd5c..9ca1e65 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -1179,6 +1180,9 @@ static int __ftrace_modify_code(void *data) static void ftrace_run_update_code(int command) { +#ifdef CONFIG_IPIPE + unsigned long flags; +#endif /* CONFIG_IPIPE */ int ret; ret = ftrace_arch_code_modify_prepare(); @@ -1186,7 +1190,13 @@ static void ftrace_run_update_code(int command) if (ret) return; +#ifdef CONFIG_IPIPE + flags = ipipe_critical_enter(NULL); + __ftrace_modify_code(&command); + ipipe_critical_exit(flags); +#else /* !CONFIG_IPIPE */ stop_machine(__ftrace_modify_code, &command, NULL); +#endif /* !CONFIG_IPIPE */ ret = ftrace_arch_code_modify_post_process(); FTRACE_WARN_ON(ret); @@ -2793,7 +2803,9 @@ static int ftrace_convert_nops(struct module *mod, /* disable interrupts to prevent kstop machine */ local_irq_save(flags); + local_irq_disable_hw(); ftrace_update_code(mod); + local_irq_enable_hw(); local_irq_restore(flags); mutex_unlock(&ftrace_lock); @@ -2878,7 +2890,9 @@ void __init ftrace_init(void) addr = (unsigned long)ftrace_stub; local_irq_save(flags); + local_irq_disable_hw(); ftrace_dyn_arch_init(&addr); + local_irq_enable_hw(); local_irq_restore(flags); /* ftrace_dyn_arch_init places the return code in addr */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 12327b2..abb7093 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -128,6 +128,8 @@ config DEBUG_SECTION_MISMATCH - Enable verbose reporting from modpost to help solving the section mismatches reported. +source "kernel/ipipe/Kconfig.debug" + config DEBUG_KERNEL bool "Kernel debugging" help diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c index 9681d54..2dba50c 100644 --- a/lib/bust_spinlocks.c +++ b/lib/bust_spinlocks.c @@ -13,6 +13,7 @@ #include #include #include +#include void __attribute__((weak)) bust_spinlocks(int yes) @@ -24,6 +25,7 @@ void __attribute__((weak)) bust_spinlocks(int yes) unblank_screen(); #endif console_unblank(); + ipipe_trace_panic_dump(); if (--oops_in_progress == 0) wake_up_klogd(); } diff --git a/lib/ioremap.c b/lib/ioremap.c index 14c6078..a275469 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -85,8 +85,8 @@ int ioremap_page_range(unsigned long addr, if (err) break; } while (pgd++, addr = next, addr != end); - - flush_cache_vmap(start, end); + __ipipe_pin_range_globally(start, end); + flush_cache_vmap(start, end); return err; } diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c index 4689cb0..3d12764 100644 --- a/lib/smp_processor_id.c +++ b/lib/smp_processor_id.c @@ -12,10 +12,13 @@ notrace unsigned int debug_smp_processor_id(void) unsigned long preempt_count = preempt_count(); int this_cpu = raw_smp_processor_id(); + if (!ipipe_root_domain_p) + goto out; + if (likely(preempt_count)) goto out; - if (irqs_disabled()) + if (irqs_disabled() || irqs_disabled_hw()) goto out; /* diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c index 9c4b025..08f096b 100644 --- a/lib/spinlock_debug.c +++ b/lib/spinlock_debug.c @@ -133,6 +133,8 @@ void _raw_spin_lock(spinlock_t *lock) debug_spin_lock_after(lock); } +EXPORT_SYMBOL(_raw_spin_lock); + int _raw_spin_trylock(spinlock_t *lock) { int ret = __raw_spin_trylock(&lock->raw_lock); @@ -148,12 +150,16 @@ int _raw_spin_trylock(spinlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_spin_trylock); + void _raw_spin_unlock(spinlock_t *lock) { debug_spin_unlock(lock); __raw_spin_unlock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_spin_unlock); + static void rwlock_bug(rwlock_t *lock, const char *msg) { if (!debug_locks_off()) @@ -199,6 +205,8 @@ void _raw_read_lock(rwlock_t *lock) __raw_read_lock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_read_lock); + int _raw_read_trylock(rwlock_t *lock) { int ret = __raw_read_trylock(&lock->raw_lock); @@ -212,12 +220,16 @@ int _raw_read_trylock(rwlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_read_trylock); + void _raw_read_unlock(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); __raw_read_unlock(&lock->raw_lock); } +EXPORT_SYMBOL(_raw_read_unlock); + static inline void debug_write_lock_before(rwlock_t *lock) { RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic"); @@ -275,6 +287,8 @@ void _raw_write_lock(rwlock_t *lock) debug_write_lock_after(lock); } +EXPORT_SYMBOL(_raw_write_lock); + int _raw_write_trylock(rwlock_t *lock) { int ret = __raw_write_trylock(&lock->raw_lock); @@ -290,8 +304,12 @@ int _raw_write_trylock(rwlock_t *lock) return ret; } +EXPORT_SYMBOL(_raw_write_trylock); + void _raw_write_unlock(rwlock_t *lock) { debug_write_unlock(lock); __raw_write_unlock(&lock->raw_lock); } + +EXPORT_SYMBOL(_raw_write_unlock); diff --git a/mm/memory.c b/mm/memory.c index aede2ce..561b457 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -55,6 +55,7 @@ #include #include #include +#include #include #include @@ -533,6 +534,32 @@ out: return pfn_to_page(pfn); } +static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) +{ + /* + * If the source page was a PFN mapping, we don't have + * a "struct page" for it. We do a best-effort copy by + * just copying from the original user address. If that + * fails, we just zero-fill it. Live with it. + */ + if (unlikely(!src)) { + void *kaddr = kmap_atomic(dst, KM_USER0); + void __user *uaddr = (void __user *)(va & PAGE_MASK); + + /* + * This really shouldn't fail, because the page is there + * in the page tables. But it might just be unreadable, + * in which case we just give up and fill the result with + * zeroes. + */ + if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) + memset(kaddr, 0, PAGE_SIZE); + kunmap_atomic(kaddr, KM_USER0); + flush_dcache_page(dst); + } else + copy_user_highpage(dst, src, va, vma); +} + /* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range @@ -541,8 +568,8 @@ out: static inline void copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, - unsigned long addr, int *rss) + pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, + unsigned long addr, int *rss, struct page *uncow_page) { unsigned long vm_flags = vma->vm_flags; pte_t pte = *src_pte; @@ -581,6 +608,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, * in the parent and the child */ if (is_cow_mapping(vm_flags)) { +#ifdef CONFIG_IPIPE + if (uncow_page) { + struct page *old_page = vm_normal_page(vma, addr, pte); + cow_user_page(uncow_page, old_page, addr, vma); + pte = mk_pte(uncow_page, vma->vm_page_prot); + + if (vm_flags & VM_SHARED) + pte = pte_mkclean(pte); + pte = pte_mkold(pte); + + page_add_new_anon_rmap(uncow_page, vma, addr); + rss[!!PageAnon(uncow_page)]++; + goto out_set_pte; + } +#endif /* CONFIG_IPIPE */ ptep_set_wrprotect(src_mm, addr, src_pte); pte = pte_wrprotect(pte); } @@ -611,13 +653,27 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte_t *src_pte, *dst_pte; spinlock_t *src_ptl, *dst_ptl; int progress = 0; + struct page *uncow_page = NULL; int rss[2]; - +#ifdef CONFIG_IPIPE + int do_cow_break = 0; +again: + if (do_cow_break) { + uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr); + if (!uncow_page) + return -ENOMEM; + do_cow_break = 0; + } +#else again: +#endif rss[1] = rss[0] = 0; dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); - if (!dst_pte) + if (!dst_pte) { + if (uncow_page) + page_cache_release(uncow_page); return -ENOMEM; + } src_pte = pte_offset_map_nested(src_pmd, addr); src_ptl = pte_lockptr(src_mm, src_pmd); spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); @@ -638,7 +694,26 @@ again: progress++; continue; } - copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss); +#ifdef CONFIG_IPIPE + if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) { + if (is_cow_mapping(vma->vm_flags)) { + if (((vma->vm_flags|src_mm->def_flags) & (VM_LOCKED|VM_PINNED)) + == (VM_LOCKED|VM_PINNED)) { + arch_leave_lazy_mmu_mode(); + spin_unlock(src_ptl); + pte_unmap_nested(src_pte); + add_mm_rss(dst_mm, rss[0], rss[1]); + pte_unmap_unlock(dst_pte, dst_ptl); + cond_resched(); + do_cow_break = 1; + goto again; + } + } + } +#endif + copy_one_pte(dst_mm, src_mm, dst_pte, + src_pte, vma, addr, rss, uncow_page); + uncow_page = NULL; progress += 8; } while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end); @@ -1900,32 +1975,6 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) return pte; } -static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) -{ - /* - * If the source page was a PFN mapping, we don't have - * a "struct page" for it. We do a best-effort copy by - * just copying from the original user address. If that - * fails, we just zero-fill it. Live with it. - */ - if (unlikely(!src)) { - void *kaddr = kmap_atomic(dst, KM_USER0); - void __user *uaddr = (void __user *)(va & PAGE_MASK); - - /* - * This really shouldn't fail, because the page is there - * in the page tables. But it might just be unreadable, - * in which case we just give up and fill the result with - * zeroes. - */ - if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) - memset(kaddr, 0, PAGE_SIZE); - kunmap_atomic(kaddr, KM_USER0); - flush_dcache_page(dst); - } else - copy_user_highpage(dst, src, va, vma); -} - /* * This routine handles present pages, when users try to write * to a shared page. It is done by copying the page to a new address @@ -3350,3 +3399,111 @@ void might_fault(void) } EXPORT_SYMBOL(might_fault); #endif + +#ifdef CONFIG_IPIPE + +static inline int ipipe_pin_pte_range(struct mm_struct *mm, pmd_t *pmd, + struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + spinlock_t *ptl; + pte_t *pte; + + do { + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + if (!pte) + continue; + + if (!pte_present(*pte) || pte_write(*pte)) { + pte_unmap_unlock(pte, ptl); + continue; + } + + if (do_wp_page(mm, vma, addr, pte, pmd, ptl, *pte) == VM_FAULT_OOM) + return -ENOMEM; + } while (addr += PAGE_SIZE, addr != end); + return 0; +} + +static inline int ipipe_pin_pmd_range(struct mm_struct *mm, pud_t *pud, + struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + unsigned long next; + pmd_t *pmd; + + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none_or_clear_bad(pmd)) + continue; + if (ipipe_pin_pte_range(mm, pmd, vma, addr, next)) + return -ENOMEM; + } while (pmd++, addr = next, addr != end); + return 0; +} + +static inline int ipipe_pin_pud_range(struct mm_struct *mm, pgd_t *pgd, + struct vm_area_struct *vma, + unsigned long addr, unsigned long end) +{ + unsigned long next; + pud_t *pud; + + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + if (ipipe_pin_pmd_range(mm, pud, vma, addr, next)) + return -ENOMEM; + } while (pud++, addr = next, addr != end); + return 0; +} + +int ipipe_disable_ondemand_mappings(struct task_struct *tsk) +{ + unsigned long addr, next, end; + struct vm_area_struct *vma; + struct mm_struct *mm; + int result = 0; + pgd_t *pgd; + + mm = get_task_mm(tsk); + if (!mm) + return -EPERM; + + down_write(&mm->mmap_sem); + if (mm->def_flags & VM_PINNED) + goto done_mm; + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (!is_cow_mapping(vma->vm_flags) + || !(vma->vm_flags & VM_WRITE)) + continue; + + addr = vma->vm_start; + end = vma->vm_end; + + pgd = pgd_offset(mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + if (ipipe_pin_pud_range(mm, pgd, vma, addr, next)) { + result = -ENOMEM; + goto done_mm; + } + } while (pgd++, addr = next, addr != end); + } + mm->def_flags |= VM_PINNED; + + done_mm: + up_write(&mm->mmap_sem); + mmput(mm); + return result; +} + +EXPORT_SYMBOL(ipipe_disable_ondemand_mappings); + +#endif diff --git a/mm/mlock.c b/mm/mlock.c index 45eb650..f19dffb 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -511,10 +511,10 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len) static int do_mlockall(int flags) { struct vm_area_struct * vma, * prev = NULL; - unsigned int def_flags = 0; + unsigned int def_flags = current->mm->def_flags & VM_PINNED; if (flags & MCL_FUTURE) - def_flags = VM_LOCKED; + def_flags |= VM_LOCKED; current->mm->def_flags = def_flags; if (flags == MCL_FUTURE) goto out; diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f8189a4..848da92 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -173,6 +173,9 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end, if (unlikely(err)) return err; + + __ipipe_pin_range_globally(start, end); + return nr; }