diff -up linux-2.6.18.x86_64/arch/i386/kernel/time-xen.c.vcpus linux-2.6.18.x86_64/arch/i386/kernel/time-xen.c --- linux-2.6.18.x86_64/arch/i386/kernel/time-xen.c.vcpus 2011-07-07 13:24:56.000000000 +0800 +++ linux-2.6.18.x86_64/arch/i386/kernel/time-xen.c 2011-07-07 13:26:39.000000000 +0800 @@ -237,7 +237,7 @@ void init_cpu_khz(void) { u64 __cpu_khz = 1000000ULL << US_SCALE; struct vcpu_time_info *info; - info = &HYPERVISOR_shared_info->vcpu_info[0].time; + info = &((vcpu_infop(0))->time); do_div(__cpu_khz, info->tsc_to_system_mul); if (info->tsc_shift < 0) cpu_khz = __cpu_khz << -info->tsc_shift; @@ -307,11 +307,10 @@ static void update_wallclock(void) */ static void get_time_values_from_xen(void) { - shared_info_t *s = HYPERVISOR_shared_info; struct vcpu_time_info *src; struct shadow_time_info *dst; - src = &s->vcpu_info[smp_processor_id()].time; + src = &(vcpu_infop(smp_processor_id())->time); dst = &per_cpu(shadow_time, smp_processor_id()); do { @@ -332,7 +331,7 @@ static inline int time_values_up_to_date struct vcpu_time_info *src; struct shadow_time_info *dst; - src = &HYPERVISOR_shared_info->vcpu_info[cpu].time; + src = &(vcpu_infop(cpu)->time); dst = &per_cpu(shadow_time, cpu); rmb(); diff --git a/arch/x86_64/kernel/irqflags-xen.c b/arch/x86_64/kernel/irqflags-xen.c index e3b7ab5..2826dc6 100644 --- a/arch/x86_64/kernel/irqflags-xen.c +++ b/arch/x86_64/kernel/irqflags-xen.c @@ -17,7 +17,7 @@ unsigned long __raw_local_save_flags(void) unsigned long flags; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); flags = _vcpu->evtchn_upcall_mask; preempt_enable(); @@ -29,7 +29,7 @@ void raw_local_irq_restore(unsigned long flags) { struct vcpu_info *_vcpu; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); if ((_vcpu->evtchn_upcall_mask = flags) == 0) { barrier(); /* unmask then check (avoid races) */ if ( unlikely(_vcpu->evtchn_upcall_pending) ) @@ -45,7 +45,7 @@ void raw_local_irq_disable(void) struct vcpu_info *_vcpu; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); _vcpu->evtchn_upcall_mask = 1; preempt_enable_no_resched(); } @@ -56,7 +56,7 @@ void raw_local_irq_enable(void) struct vcpu_info *_vcpu; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); _vcpu->evtchn_upcall_mask = 0; barrier(); /* unmask then check (avoid races) */ if ( unlikely(_vcpu->evtchn_upcall_pending) ) @@ -75,7 +75,7 @@ unsigned long __raw_local_irq_save(void) unsigned long flags; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); flags = _vcpu->evtchn_upcall_mask; _vcpu->evtchn_upcall_mask = 1; preempt_enable_no_resched(); @@ -91,7 +91,7 @@ int raw_irqs_disabled(void) int disabled; preempt_disable(); - _vcpu = &HYPERVISOR_shared_info->vcpu_info[__vcpu_id]; + _vcpu = per_cpu(xen_vcpup, __vcpu_id); disabled = (_vcpu->evtchn_upcall_mask != 0); preempt_enable_no_resched(); diff --git a/arch/x86_64/kernel/xen_entry.S b/arch/x86_64/kernel/xen_entry.S index b3d7f19..8f2d77d 100644 --- a/arch/x86_64/kernel/xen_entry.S +++ b/arch/x86_64/kernel/xen_entry.S @@ -12,19 +12,18 @@ //#define preempt_enable(reg) decl threadinfo_preempt_count(reg) #define preempt_disable(reg) #define preempt_enable(reg) -#define XEN_GET_VCPU_INFO(reg) preempt_disable(%rbp) ; \ - movq %gs:pda_cpunumber,reg ; \ - shl $32, reg ; \ - shr $32-sizeof_vcpu_shift,reg ; \ - addq HYPERVISOR_shared_info,reg #define XEN_PUT_VCPU_INFO(reg) preempt_enable(%rbp) ; \ #define XEN_PUT_VCPU_INFO_fixup .byte 0xff,0xff,0xff #else -#define XEN_GET_VCPU_INFO(reg) movq HYPERVISOR_shared_info,reg #define XEN_PUT_VCPU_INFO(reg) #define XEN_PUT_VCPU_INFO_fixup #endif +#define XEN_GET_VCPU_INFO(reg) preempt_disable(%rbp) ; \ + movq %gs:pda_data_offset,reg ; \ + addq $per_cpu__xen_vcpup,reg ; \ + movq (reg), reg + #define XEN_LOCKED_BLOCK_EVENTS(reg) movb $1,evtchn_upcall_mask(reg) #define XEN_LOCKED_UNBLOCK_EVENTS(reg) movb $0,evtchn_upcall_mask(reg) #define XEN_BLOCK_EVENTS(reg) XEN_GET_VCPU_INFO(reg) ; \ diff -up linux-2.6.18.x86_64/arch/x86_64/mm/fault-xen.c.vcpus linux-2.6.18.x86_64/arch/x86_64/mm/fault-xen.c --- linux-2.6.18.x86_64/arch/x86_64/mm/fault-xen.c.vcpus 2011-07-07 13:34:00.000000000 +0800 +++ linux-2.6.18.x86_64/arch/x86_64/mm/fault-xen.c 2011-07-07 13:34:16.000000000 +0800 @@ -675,8 +675,7 @@ asmlinkage void __kprobes do_page_fault( unsigned long address; /* get the address */ - address = HYPERVISOR_shared_info->vcpu_info[ - smp_processor_id()].arch.cr2; + address = (__get_cpu_var(xen_vcpup))->arch.cr2; __do_page_fault(regs, address, error_code); diff -up linux-2.6.18.x86_64/arch/x86_64/mm/init-xen.c.vcpus linux-2.6.18.x86_64/arch/x86_64/mm/init-xen.c --- linux-2.6.18.x86_64/arch/x86_64/mm/init-xen.c.vcpus 2011-07-07 13:36:08.000000000 +0800 +++ linux-2.6.18.x86_64/arch/x86_64/mm/init-xen.c 2011-07-07 13:36:18.000000000 +0800 @@ -802,6 +802,8 @@ void __init paging_init(void) __set_fixmap(FIX_ISAMAP_BEGIN - i, virt_to_mfn(empty_zero_page) << PAGE_SHIFT, PAGE_KERNEL_RO); + + per_cpu(xen_vcpup, 0) = &HYPERVISOR_shared_info->vcpu_info[0]; } #endif diff -up linux-2.6.18.x86_64/drivers/xen/core/evtchn.c.vcpus linux-2.6.18.x86_64/drivers/xen/core/evtchn.c --- linux-2.6.18.x86_64/drivers/xen/core/evtchn.c.vcpus 2011-07-07 13:37:31.000000000 +0800 +++ linux-2.6.18.x86_64/drivers/xen/core/evtchn.c 2011-07-07 13:38:30.000000000 +0800 @@ -219,7 +219,7 @@ asmlinkage void evtchn_do_upcall(struct unsigned int l1i, l2i, port, count; int irq, cpu = smp_processor_id(); shared_info_t *s = HYPERVISOR_shared_info; - vcpu_info_t *vcpu_info = &s->vcpu_info[cpu]; + vcpu_info_t *vcpu_info = vcpu_infop(cpu); exit_idle(); irq_enter(); @@ -790,7 +790,7 @@ void unmask_evtchn(int port) { shared_info_t *s = HYPERVISOR_shared_info; unsigned int cpu = smp_processor_id(); - vcpu_info_t *vcpu_info = &s->vcpu_info[cpu]; + vcpu_info_t *vcpu_info = vcpu_infop(cpu); BUG_ON(!irqs_disabled()); diff --git a/include/asm-i386/mach-xen/asm/hypervisor.h b/include/asm-i386/mach-xen/asm/hypervisor.h index 89cde62..7568584 100644 --- a/include/asm-i386/mach-xen/asm/hypervisor.h +++ b/include/asm-i386/mach-xen/asm/hypervisor.h @@ -103,6 +103,14 @@ void xen_tlb_flush_mask(cpumask_t *mask); void xen_invlpg_mask(cpumask_t *mask, unsigned long ptr); #endif +#ifdef CONFIG_X86_64 +#include +DECLARE_PER_CPU(struct vcpu_info *, xen_vcpup); +#define vcpu_infop(cpu) (per_cpu(xen_vcpup, cpu)) +#else +#define vcpu_infop(cpu) (HYPERVISOR_shared_info->vcpu_info + (cpu)) +#endif + /* Returns zero on success else negative errno. */ int xen_create_contiguous_region( unsigned long vstart, unsigned int order, unsigned int address_bits); diff --git a/include/xen/interface/vcpu.h b/include/xen/interface/vcpu.h index eb8b5a1..42a131a 100644 --- a/include/xen/interface/vcpu.h +++ b/include/xen/interface/vcpu.h @@ -108,7 +108,23 @@ struct vcpu_register_runstate_memory_area { }; typedef struct vcpu_register_runstate_memory_area vcpu_register_runstate_memory_area_t; +/* + * Register a memory location in the guest address space for the + * vcpu_info structure. This allows the guest to place the vcpu_info + * structure in a convenient place, such as in a per-cpu data area. + * The pointer need not be page aligned, but the structure must not + * cross a page boundary. + * + * This may be called only once per vcpu. + */ #define VCPUOP_register_vcpu_info 10 /* arg == vcpu_register_vcpu_info_t */ +struct vcpu_register_vcpu_info { + uint64_t mfn; /* mfn of page to place vcpu_info */ + uint32_t offset; /* offset within page */ + uint32_t rsvd; /* unused */ +}; +typedef struct vcpu_register_vcpu_info vcpu_register_vcpu_info_t; +DEFINE_XEN_GUEST_HANDLE(vcpu_register_vcpu_info_t); /* * Get the physical ID information for a pinned vcpu's underlying physical diff -up linux-2.6.18.x86_64/drivers/xen/core/smpboot.c.vcpus linux-2.6.18.x86_64/drivers/xen/core/smpboot.c --- linux-2.6.18.x86_64/drivers/xen/core/smpboot.c.vcpus 2011-07-07 13:42:56.000000000 +0800 +++ linux-2.6.18.x86_64/drivers/xen/core/smpboot.c 2011-07-08 15:20:02.000000000 +0800 @@ -361,8 +361,51 @@ void __init smp_prepare_cpus(unsigned in #endif } +#ifdef __x86_64__ +DEFINE_PER_CPU(struct vcpu_info, xen_vcpu_info) __attribute__((aligned(64))); +DEFINE_PER_CPU(struct vcpu_info *, xen_vcpup) = + { (struct vcpu_info *)empty_zero_page }; +EXPORT_PER_CPU_SYMBOL(xen_vcpup); + +static void check_relocate_vcpus(void) +{ + struct vcpu_register_vcpu_info info; + struct vcpu_info *vcpup; + int rc, cpu, relocate=0; + + if (num_possible_cpus() > MAX_VIRT_CPUS) + relocate = 1; + + for_each_possible_cpu (cpu) { + if (relocate) { + vcpup = &per_cpu(xen_vcpu_info, cpu); + info.mfn = virt_to_mfn(vcpup); + info.offset = offset_in_page(vcpup); + rc = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, cpu, + &info); + if (rc == ENOSYS) { + printk(KERN_ERR "Current xen does not support " + "more than 32 VCPUs\n"); + } else if (rc) + printk(KERN_ERR "VCPUOP failed. rc:%d\n", rc); + + BUG_ON(rc); + } else { + /* use shared page so we can run on older xen without + * VCPUOP_register_vcpu_info */ + vcpup = &HYPERVISOR_shared_info->vcpu_info[cpu]; + } + + per_cpu(xen_vcpup, cpu) = vcpup; + } +} +#endif + void __devinit smp_prepare_boot_cpu(void) { +#ifdef __x86_64__ + check_relocate_vcpus(); +#endif } #ifdef CONFIG_HOTPLUG_CPU