(Note: Patch also attached because the inline version is certain to get line wrapped.) Make the double fault handler use CPU-specific stacks, add some abstraction to simplify future change of other exception handlers to go through task gates. Change the pointer validity checks in the double fault handler to account for the fact that both GDT and TSS aren't in static kernel space anymore. This patch depends upon the presence of THREAD_ORDER, as added in a previously submitted patch. Signed-off-by: Jan Beulich <[email protected]> diff -Npru 2.6.13/arch/i386/kernel/cpu/common.c 2.6.13-i386-double-fault/arch/i386/kernel/cpu/common.c --- 2.6.13/arch/i386/kernel/cpu/common.c 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/arch/i386/kernel/cpu/common.c 2005-09-08 15:00:54.000000000 +0200 @@ -4,6 +4,7 @@ #include <linux/smp.h> #include <linux/module.h> #include <linux/percpu.h> +#include <linux/bootmem.h> #include <asm/semaphore.h> #include <asm/processor.h> #include <asm/i387.h> @@ -571,6 +572,7 @@ void __init early_cpu_init(void) void __devinit cpu_init(void) { int cpu = smp_processor_id(); + unsigned i; struct tss_struct * t = &per_cpu(init_tss, cpu); struct thread_struct *thread = ¤t->thread; __u32 stk16_off = (__u32)&per_cpu(cpu_16bit_stack, cpu); @@ -635,8 +637,57 @@ void __devinit cpu_init(void) load_TR_desc(); load_LDT(&init_mm.context); - /* Set up doublefault TSS pointer in the GDT */ - __set_tss_desc(cpu, GDT_ENTRY_DOUBLEFAULT_TSS, &doublefault_tss); +#if EXCEPTION_STACK_ORDER > THREAD_ORDER +# error Assertion failed: EXCEPTION_STACK_ORDER <= THREAD_ORDER +#endif + for (i = 0; i < N_EXCEPTION_TSS; ++i) { + unsigned long stack; + + /* Set up exception handling TSS */ + exception_tss[cpu][i].esp2 = cpu; + exception_tss[cpu][i].esi = (unsigned long)&exception_tss[cpu][i]; + + /* Set up exception handling stacks */ +#ifdef CONFIG_SMP + if (cpu) { + stack = __get_free_pages(GFP_ATOMIC, THREAD_ORDER); + if (!stack) + panic("Cannot allocate exception stack %u %d\n", + i, + cpu); + } + else +#endif + stack = (unsigned long)__alloc_bootmem(EXCEPTION_STKSZ, + THREAD_SIZE, + __pa(MAX_DMA_ADDRESS)); + stack += EXCEPTION_STKSZ; + exception_tss[cpu][i].esp = exception_tss[cpu][i].esp0 = stack; +#ifdef CONFIG_SMP + if (cpu) { + unsigned j; + + for (j = EXCEPTION_STACK_ORDER; j < THREAD_ORDER; ++j) { + /* set_page_refs sets the page count only for the first + page, but since we split the larger-order page here, + we need to adjust the page count before freeing the + pieces. */ + struct page * page = virt_to_page((void *)stack); + + BUG_ON(page_count(page)); + set_page_count(page, 1); + free_pages(stack, j); + stack += (PAGE_SIZE << j); + } + } + else +#endif + if (EXCEPTION_STACK_ORDER > THREAD_ORDER) + free_bootmem(stack, THREAD_SIZE - EXCEPTION_STKSZ); + + /* Set up exception handling TSS pointer in the GDT */ + __set_tss_desc(cpu, GDT_ENTRY_EXCEPTION_TSS + i, &exception_tss[cpu][i]); + } /* Clear %fs and %gs. */ asm volatile ("xorl %eax, %eax; movl %eax, %fs; movl %eax, %gs"); diff -Npru 2.6.13/arch/i386/kernel/doublefault.c 2.6.13-i386-double-fault/arch/i386/kernel/doublefault.c --- 2.6.13/arch/i386/kernel/doublefault.c 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/arch/i386/kernel/doublefault.c 2005-09-09 09:37:02.000000000 +0200 @@ -9,13 +9,11 @@ #include <asm/processor.h> #include <asm/desc.h> -#define DOUBLEFAULT_STACKSIZE (1024) -static unsigned long doublefault_stack[DOUBLEFAULT_STACKSIZE]; -#define STACK_START (unsigned long)(doublefault_stack+DOUBLEFAULT_STACKSIZE) +extern unsigned long max_low_pfn; +#define ptr_ok(x, l) ((x) >= PAGE_OFFSET \ + && (x) + (l) <= PAGE_OFFSET + max_low_pfn * PAGE_SIZE - 1) -#define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + 0x1000000) - -static void doublefault_fn(void) +void doublefault_fn(void) { struct Xgt_desc_struct gdt_desc = {0, 0}; unsigned long gdt, tss; @@ -23,43 +21,26 @@ static void doublefault_fn(void) __asm__ __volatile__("sgdt %0": "=m" (gdt_desc): :"memory"); gdt = gdt_desc.address; - printk("double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size); + printk("double fault, gdt at %08lx [%d bytes]\n", gdt, gdt_desc.size + 1); - if (ptr_ok(gdt)) { + if (ptr_ok(gdt, gdt_desc.size)) { gdt += GDT_ENTRY_TSS << 3; tss = *(u16 *)(gdt+2); tss += *(u8 *)(gdt+4) << 16; tss += *(u8 *)(gdt+7) << 24; printk("double fault, tss at %08lx\n", tss); - if (ptr_ok(tss)) { + if (ptr_ok(tss, *(u16 *)gdt)) { struct tss_struct *t = (struct tss_struct *)tss; printk("eip = %08lx, esp = %08lx\n", t->eip, t->esp); printk("eax = %08lx, ebx = %08lx, ecx = %08lx, edx = %08lx\n", t->eax, t->ebx, t->ecx, t->edx); - printk("esi = %08lx, edi = %08lx\n", - t->esi, t->edi); + printk("esi = %08lx, edi = %08lx, ebp = %08lx\n", + t->esi, t->edi, t->ebp); } } for (;;) /* nothing */; } - -struct tss_struct doublefault_tss __cacheline_aligned = { - .esp0 = STACK_START, - .ss0 = __KERNEL_DS, - .ldt = 0, - .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, - - .eip = (unsigned long) doublefault_fn, - .eflags = X86_EFLAGS_SF | 0x2, /* 0x2 bit is always set */ - .esp = STACK_START, - .es = __USER_DS, - .cs = __KERNEL_CS, - .ss = __KERNEL_DS, - .ds = __USER_DS, - - .__cr3 = __pa(swapper_pg_dir) -}; diff -Npru 2.6.13/arch/i386/kernel/head.S 2.6.13-i386-double-fault/arch/i386/kernel/head.S --- 2.6.13/arch/i386/kernel/head.S 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/arch/i386/kernel/head.S 2005-09-01 13:06:01.000000000 +0200 @@ -475,9 +475,9 @@ ENTRY(boot_gdt_table) .quad 0x00cf92000000ffff /* kernel 4GB data at 0x00000000 */ /* - * The Global Descriptor Table contains 28 quadwords, per-CPU. + * The Global Descriptor Table contains at least 31 quadwords, per-CPU. */ - .align PAGE_SIZE_asm + .align L1_CACHE_BYTES ENTRY(cpu_gdt_table) .quad 0x0000000000000000 /* NULL descriptor */ .quad 0x0000000000000000 /* 0x0b reserved */ @@ -515,9 +515,5 @@ ENTRY(cpu_gdt_table) .quad 0x0040920000000000 /* 0xc8 APM DS data */ .quad 0x0000920000000000 /* 0xd0 - ESPFIX 16-bit SS */ - .quad 0x0000000000000000 /* 0xd8 - unused */ - .quad 0x0000000000000000 /* 0xe0 - unused */ - .quad 0x0000000000000000 /* 0xe8 - unused */ - .quad 0x0000000000000000 /* 0xf0 - unused */ - .quad 0x0000000000000000 /* 0xf8 - GDT entry 31: double-fault TSS */ - + /* remaining entries run-time initialized */ + .fill GDT_ENTRIES - (. - cpu_gdt_table) / 8, 8, 0 diff -Npru 2.6.13/arch/i386/kernel/traps.c 2.6.13-i386-double-fault/arch/i386/kernel/traps.c --- 2.6.13/arch/i386/kernel/traps.c 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/arch/i386/kernel/traps.c 2005-09-07 15:08:32.000000000 +0200 @@ -62,6 +62,24 @@ asmlinkage int system_call(void); struct desc_struct default_ldt[] = { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 }, { 0, 0 } }; +void doublefault_fn(void); + +struct tss_struct exception_tss[NR_CPUS][N_EXCEPTION_TSS] __cacheline_aligned = { + [0 ... NR_CPUS-1] = { + [0 ... N_EXCEPTION_TSS-1] = { + .cs = __KERNEL_CS, + .ss = __KERNEL_DS, + .ss0 = __KERNEL_DS, + .__cr3 = __pa(swapper_pg_dir), + .io_bitmap_base = INVALID_IO_BITMAP_OFFSET, + .ds = __USER_DS, + .es = __USER_DS, + .eflags = X86_EFLAGS_SF | 0x2, /* 0x2 bit is always set */ + }, + [DOUBLEFAULT_TSS].eip = (unsigned long)doublefault_fn + } +}; + /* Do we ignore FPU interrupts ? */ char ignore_fpu_irq = 0; @@ -1083,7 +1101,7 @@ void __init trap_init(void) set_system_gate(5,&bounds); set_trap_gate(6,&invalid_op); set_trap_gate(7,&device_not_available); - set_task_gate(8,GDT_ENTRY_DOUBLEFAULT_TSS); + set_task_gate(8,GDT_ENTRY_EXCEPTION_TSS + DOUBLEFAULT_TSS); set_trap_gate(9,&coprocessor_segment_overrun); set_trap_gate(10,&invalid_TSS); set_trap_gate(11,&segment_not_present); diff -Npru 2.6.13/include/asm-i386/processor.h 2.6.13-i386-double-fault/include/asm-i386/processor.h --- 2.6.13/include/asm-i386/processor.h 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/include/asm-i386/processor.h 2005-09-01 13:14:48.000000000 +0200 @@ -86,7 +86,6 @@ struct cpuinfo_x86 { extern struct cpuinfo_x86 boot_cpu_data; extern struct cpuinfo_x86 new_cpu_data; -extern struct tss_struct doublefault_tss; DECLARE_PER_CPU(struct tss_struct, init_tss); #ifdef CONFIG_SMP @@ -479,6 +478,12 @@ struct thread_struct { .io_bitmap = { [ 0 ... IO_BITMAP_LONGS] = ~0 }, \ } +#define DOUBLEFAULT_TSS 0 +#define EXCEPTION_STACK_ORDER 0 +#define EXCEPTION_STKSZ (PAGE_SIZE << EXCEPTION_STACK_ORDER) + +extern struct tss_struct exception_tss[NR_CPUS][N_EXCEPTION_TSS]; + static inline void load_esp0(struct tss_struct *tss, struct thread_struct *thread) { tss->esp0 = thread->esp0; diff -Npru 2.6.13/include/asm-i386/segment.h 2.6.13-i386-double-fault/include/asm-i386/segment.h --- 2.6.13/include/asm-i386/segment.h 2005-08-29 01:41:01.000000000 +0200 +++ 2.6.13-i386-double-fault/include/asm-i386/segment.h 2005-09-01 11:32:11.000000000 +0200 @@ -1,6 +1,8 @@ #ifndef _ASM_SEGMENT_H #define _ASM_SEGMENT_H +#include <asm/cache.h> + /* * The layout of the per-CPU GDT under Linux: * @@ -43,7 +45,8 @@ * 28 - unused * 29 - unused * 30 - unused - * 31 - TSS for double fault handler + * 31 - TSS for first exception handler (double fault) + * 32+ TSSes for further exception handlers */ #define GDT_ENTRY_TLS_ENTRIES 3 #define GDT_ENTRY_TLS_MIN 6 @@ -74,12 +77,20 @@ #define GDT_ENTRY_ESPFIX_SS (GDT_ENTRY_KERNEL_BASE + 14) #define __ESPFIX_SS (GDT_ENTRY_ESPFIX_SS * 8) -#define GDT_ENTRY_DOUBLEFAULT_TSS 31 +#define GDT_ENTRY_EXCEPTION_TSS 31 +#define DOUBLEFAULT_TSS 0 +#define N_EXCEPTION_TSS 1 /* - * The GDT has 32 entries + * The GDT has 32+ entries */ -#define GDT_ENTRIES 32 +#if L1_CACHE_BYTES < 8 +# error Assertion failed: L1_CACHE_BYTES >= 8 +#endif +#if L1_CACHE_BYTES & (L1_CACHE_BYTES - 1) +# error Assertion failed: L1_CACHE_BYTES is power of two +#endif +#define GDT_ENTRIES ((31 + N_EXCEPTION_TSS + L1_CACHE_BYTES / 8 - 1) & ~(L1_CACHE_BYTES / 8 - 1)) #define GDT_SIZE (GDT_ENTRIES * 8) @@ -92,9 +103,7 @@ #define __BOOT_DS (GDT_ENTRY_BOOT_DS * 8) /* - * The interrupt descriptor table has room for 256 idt's, - * the global descriptor table is dependent on the number - * of tasks we can have.. + * The interrupt descriptor table has room for 256 idt's. */ #define IDT_ENTRIES 256
Attachment:
linux-2.6.13-i386-double-fault.patch
Description: Binary data
- Prev by Date: Re: [discuss] [PATCH] add and handle NMI_VECTOR II
- Next by Date: Re: [discuss] [PATCH] add and handle NMI_VECTOR II
- Previous by thread: [PATCH] [2.6.13-mm2] set IBM ThinkPad extras to default n in Kconfig
- Next by thread: Re: [PATCH] fix i386 double fault handler
- Index(es):
