VMM: Add kernel support for VM contexts [1/2]
[akaros.git] / kern / arch / x86 / trap.c
index 6bc1079..6cb72cb 100644 (file)
@@ -1,13 +1,9 @@
-#ifdef __SHARC__
-#pragma nosharc
-#define SINIT(x) x
-#endif
-
 #include <arch/mmu.h>
 #include <arch/x86.h>
 #include <arch/arch.h>
 #include <arch/console.h>
 #include <arch/apic.h>
+#include <arch/perfmon.h>
 #include <ros/common.h>
 #include <smp.h>
 #include <assert.h>
 #include <syscall.h>
 #include <kdebug.h>
 #include <kmalloc.h>
+#include <ex_table.h>
+#include <arch/mptables.h>
 
-taskstate_t RO ts;
+taskstate_t ts;
 
 /* Interrupt descriptor table.  64 bit needs 16 byte alignment (i think). */
 gatedesc_t __attribute__((aligned (16))) idt[256] = { { 0 } };
@@ -33,14 +31,23 @@ pseudodesc_t idt_pd;
 struct irq_handler *irq_handlers[NUM_IRQS];
 spinlock_t irq_handler_wlock = SPINLOCK_INITIALIZER_IRQSAVE;
 
-/* Which pci devices hang off of which irqs */
-/* TODO: make this an array of SLISTs (pain from ioapic.c, etc...) */
-struct pci_device *irq_pci_map[NUM_IRQS] = {0};
+static bool try_handle_exception_fixup(struct hw_trapframe *hw_tf)
+{
+       if (in_kernel(hw_tf)) {
+               uintptr_t fixup_ip = get_fixup_ip(hw_tf->tf_rip);
+
+               if (fixup_ip != 0) {
+                       hw_tf->tf_rip = fixup_ip;
+                       return true;
+               }
+       }
+
+       return false;
+}
 
 const char *x86_trapname(int trapno)
 {
-    // zra: excnames is SREADONLY because Ivy doesn't trust const
-       static const char *NT const (RO excnames)[] = {
+       static const char *const excnames[] = {
                "Divide error",
                "Debug",
                "Non-Maskable Interrupt",
@@ -138,8 +145,8 @@ void idt_init(void)
                (uintptr_t)idt[T_SYSCALL].gd_off_15_0));
        /* turn on trap-based syscall handling and other user-accessible ints
         * DPL 3 means this can be triggered by the int instruction */
-       idt[T_SYSCALL].gd_dpl = SINIT(3);
-       idt[T_BRKPT].gd_dpl = SINIT(3);
+       idt[T_SYSCALL].gd_dpl = 3;
+       idt[T_BRKPT].gd_dpl = 3;
 
        /* Set up our kernel stack when changing rings */
        /* Note: we want 16 byte aligned kernel stack frames (AMD 2:8.9.3) */
@@ -168,29 +175,28 @@ void idt_init(void)
        pic_remap();
        pic_mask_all();
 
-#ifdef CONFIG_ENABLE_MPTABLES
-       int ncleft;
-       int mpsinit(int maxcores);
+       int ncleft = MAX_NUM_CORES;
+       int num_cores_mpacpi;
 
-       ncleft = mpsinit(MAX_NUM_CPUS);
-       /* NEVER printd here ... */
-       printk("mpacpi is %d\n", mpacpi(ncleft));
+       ncleft = mpsinit(ncleft);
+       ncleft = mpacpi(ncleft);
+       num_cores_mpacpi = MAX_NUM_CORES - ncleft;
+       printk("MP and ACPI found %d cores\n", num_cores_mpacpi);
+       if (num_cores != num_cores_mpacpi)
+               warn("Topology (%d) and MP/ACPI (%d) differ on num_cores!", num_cores,
+                    num_cores_mpacpi);
 
-       void ioapiconline(void);
-       void apiconline(void);
-       apiconline(); /* TODO: do this this for all cores*/
+       apiconline();
        ioapiconline();
-#else
-       // set LINT0 to receive ExtINTs (KVM's default).  At reset they are 0x1000.
-       write_mmreg32(LAPIC_LVT_LINT0, 0x700);
-       lapic_enable();
-       unmask_lapic_lvt(LAPIC_LVT_LINT0);
-#endif
-
-       /* register the generic timer_interrupt() handler for the per-core timers */
-       register_raw_irq(IdtLAPIC_TIMER, timer_interrupt, NULL);
-       /* register the kernel message handler */
-       register_raw_irq(I_KERNEL_MSG, handle_kmsg_ipi, NULL);
+
+       /* the lapic IRQs need to be unmasked on a per-core basis */
+       register_irq(IdtLAPIC_TIMER, timer_interrupt, NULL,
+                    MKBUS(BusLAPIC, 0, 0, 0));
+       register_irq(IdtLAPIC_ERROR, handle_lapic_error, NULL,
+                    MKBUS(BusLAPIC, 0, 0, 0));
+       register_irq(IdtLAPIC_PCINT, perfmon_interrupt, NULL,
+                    MKBUS(BusLAPIC, 0, 0, 0));
+       register_irq(I_KERNEL_MSG, handle_kmsg_ipi, NULL, MKBUS(BusIPI, 0, 0, 0));
 }
 
 static void handle_fperr(struct hw_trapframe *hw_tf)
@@ -226,66 +232,94 @@ static void handle_fperr(struct hw_trapframe *hw_tf)
        if (fpsw & ~fpcw & FP_EXCP_PE)
                printk("\tInexact result (precision)\n");
        printk("Killing the process.\n");
-       enable_irq();
        proc_destroy(current);
 }
 
-void backtrace_kframe(struct hw_trapframe *hw_tf)
+static bool __handler_user_page_fault(struct hw_trapframe *hw_tf,
+                                      uintptr_t fault_va, int prot)
 {
        struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
-       pcpui->__lock_checking_enabled--;
-       printk("\nBacktrace of faulting kernel context on Core %d:\n", core_id());
-       backtrace_frame(x86_get_hwtf_pc(hw_tf), x86_get_hwtf_fp(hw_tf));
-       pcpui->__lock_checking_enabled++;
+       int err;
+
+       assert(pcpui->owning_proc == pcpui->cur_proc);
+       enable_irq();
+       err = handle_page_fault(pcpui->owning_proc, fault_va, prot);
+       disable_irq();
+       if (err) {
+               if (err == -EAGAIN)
+                       hw_tf->tf_err |= PF_VMR_BACKED;
+               return FALSE;
+       }
+       return TRUE;
 }
 
-static bool __handle_page_fault(struct hw_trapframe *hw_tf, unsigned long *aux)
+static bool __handler_kernel_page_fault(struct hw_trapframe *hw_tf,
+                                        uintptr_t fault_va, int prot)
 {
-       uintptr_t fault_va = rcr2();
-       int prot = hw_tf->tf_err & PF_ERROR_WRITE ? PROT_WRITE : PROT_READ;
+       struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
        int err;
 
-       /* TODO - handle kernel page faults */
-       if ((hw_tf->tf_cs & 3) == 0) {
+       /* In general, if there's no cur_proc, a KPF is a bug. */
+       if (!pcpui->cur_proc) {
+               /* This only runs from test_uaccess(), where it is expected to fail. */
+               if (try_handle_exception_fixup(hw_tf))
+                       return TRUE;
                print_trapframe(hw_tf);
-               backtrace_kframe(hw_tf);
-               panic("Page Fault in the Kernel at %p!", fault_va);
-               /* if we want to do something like kill a process or other code, be
-                * aware we are in a sort of irq-like context, meaning the main kernel
-                * code we 'interrupted' could be holding locks - even irqsave locks. */
+               backtrace_hwtf(hw_tf);
+               panic("Proc-less Page Fault in the Kernel at %p!", fault_va);
        }
-       /* safe to reenable after rcr2 */
-       enable_irq();
-       if ((err = handle_page_fault(current, fault_va, prot))) {
-               if (err == -EAGAIN)
-                       hw_tf->tf_err |= PF_VMR_BACKED;
-               *aux = fault_va;
-               return FALSE;
-               /* useful debugging */
-               printk("[%08x] user %s fault va %p ip %p on core %d with err %d\n",
-                      current->pid, prot & PROT_READ ? "READ" : "WRITE", fault_va,
-                      hw_tf->tf_rip, core_id(), err);
+       /* TODO - handle kernel page faults.  This is dangerous, since we might be
+        * holding locks in the kernel and could deadlock when we HPF.  For now, I'm
+        * just disabling the lock checker, since it'll flip out when it sees there
+        * is a kernel trap.  Will need to think about this a bit, esp when we
+        * properly handle bad addrs and whatnot. */
+       pcpui->__lock_checking_enabled--;
+       /* It is a bug for the kernel to access user memory while holding locks that
+        * are used by handle_page_fault.  At a minimum, this includes p->vmr_lock
+        * and memory allocation locks.
+        *
+        * In an effort to reduce the number of locks (both now and in the future),
+        * the kernel will not attempt to handle faults on file-back VMRs.  We
+        * probably can turn that on in the future, but I'd rather keep things safe
+        * for now.
+        *
+        * Note that we do not enable IRQs here, unlike in the user case.  Again,
+        * this is to limit the locks we could be grabbing. */
+       err = handle_page_fault_nofile(pcpui->cur_proc, fault_va, prot);
+       pcpui->__lock_checking_enabled++;
+       if (err) {
+               if (try_handle_exception_fixup(hw_tf))
+                       return TRUE;
                print_trapframe(hw_tf);
+               backtrace_hwtf(hw_tf);
                /* Turn this on to help debug bad function pointers */
-#ifdef CONFIG_X86_64
                printd("rsp %p\n\t 0(rsp): %p\n\t 8(rsp): %p\n\t 16(rsp): %p\n"
                       "\t24(rsp): %p\n", hw_tf->tf_rsp,
                       *(uintptr_t*)(hw_tf->tf_rsp +  0),
                       *(uintptr_t*)(hw_tf->tf_rsp +  8),
                       *(uintptr_t*)(hw_tf->tf_rsp + 16),
                       *(uintptr_t*)(hw_tf->tf_rsp + 24));
-#else
-               printd("esp %p\n\t 0(esp): %p\n\t 4(esp): %p\n\t 8(esp): %p\n"
-                      "\t12(esp): %p\n", hw_tf->tf_esp,
-                      *(uintptr_t*)(hw_tf->tf_esp +  0),
-                      *(uintptr_t*)(hw_tf->tf_esp +  4),
-                      *(uintptr_t*)(hw_tf->tf_esp +  8),
-                      *(uintptr_t*)(hw_tf->tf_esp + 12));
-#endif
+               panic("Proc-ful Page Fault in the Kernel at %p!", fault_va);
+               /* if we want to do something like kill a process or other code, be
+                * aware we are in a sort of irq-like context, meaning the main
+                * kernel code we 'interrupted' could be holding locks - even
+                * irqsave locks. */
        }
        return TRUE;
 }
 
+static bool __handle_page_fault(struct hw_trapframe *hw_tf, unsigned long *aux)
+{
+       uintptr_t fault_va = rcr2();
+       int prot = hw_tf->tf_err & PF_ERROR_WRITE ? PROT_WRITE : PROT_READ;
+
+       *aux = fault_va;
+       if (in_kernel(hw_tf))
+               return __handler_kernel_page_fault(hw_tf, fault_va, prot);
+       else
+               return __handler_user_page_fault(hw_tf, fault_va, prot);
+}
+
 /* Certain traps want IRQs enabled, such as the syscall.  Others can't handle
  * it, like the page fault handler.  Turn them on on a case-by-case basis. */
 static void trap_dispatch(struct hw_trapframe *hw_tf)
@@ -293,6 +327,8 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
        struct per_cpu_info *pcpui;
        bool handled = TRUE;
        unsigned long aux = 0;
+       uintptr_t fixup_ip;
+
        // Handle processor exceptions.
        switch(hw_tf->tf_trapno) {
                case T_NMI:
@@ -304,10 +340,11 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
                        extern bool mon_verbose_trace;
                        if (mon_verbose_trace) {
                                print_trapframe(hw_tf);
-                               backtrace_kframe(hw_tf);
+                               backtrace_hwtf(hw_tf);
                        }
-                       char *fn_name = get_fn_name(x86_get_ip_hw(hw_tf));
-                       printk("Core %d is at %p (%s)\n", core_id(), x86_get_ip_hw(hw_tf),
+                       char *fn_name = get_fn_name(get_hwtf_pc(hw_tf));
+
+                       printk("Core %d is at %p (%s)\n", core_id(), get_hwtf_pc(hw_tf),
                               fn_name);
                        kfree(fn_name);
                        print_kmsgs(core_id());
@@ -316,11 +353,12 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
                case T_BRKPT:
                        enable_irq();
                        monitor(hw_tf);
+                       disable_irq();
                        break;
                case T_ILLOP:
                {
                        /* TODO: this can PF if there is a concurrent unmap/PM removal. */
-                       uintptr_t ip = x86_get_ip_hw(hw_tf);
+                       uintptr_t ip = get_hwtf_pc(hw_tf);
                        pcpui = &per_cpu_info[core_id()];
                        pcpui->__lock_checking_enabled--;               /* for print debugging */
                        /* We will muck with the actual TF.  If we're dealing with
@@ -343,6 +381,7 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
                        }
                        enable_irq();
                        monitor(hw_tf);
+                       disable_irq();
                        pcpui->__lock_checking_enabled++;               /* for print debugging */
                        break;
                }
@@ -350,7 +389,9 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
                        handled = __handle_page_fault(hw_tf, &aux);
                        break;
                case T_FPERR:
-                       handle_fperr(hw_tf);
+                       handled = try_handle_exception_fixup(hw_tf);
+                       if (!handled)
+                               handle_fperr(hw_tf);
                        break;
                case T_SYSCALL:
                        enable_irq();
@@ -361,15 +402,20 @@ static void trap_dispatch(struct hw_trapframe *hw_tf)
                        prep_syscalls(current,
                                      (struct syscall*)x86_get_systrap_arg0(hw_tf),
                                                  (unsigned int)x86_get_systrap_arg1(hw_tf));
+                       disable_irq();
                        break;
                default:
                        if (hw_tf->tf_cs == GD_KT) {
-                               print_trapframe(hw_tf);
-                               panic("Damn Damn!  Unhandled trap in the kernel!");
+                               handled = try_handle_exception_fixup(hw_tf);
+                               if (!handled) {
+                                       print_trapframe(hw_tf);
+                                       panic("Damn Damn!  Unhandled trap in the kernel!");
+                               }
                        } else {
                                handled = FALSE;
                        }
        }
+
        if (!handled)
                reflect_unhandled_trap(hw_tf->tf_trapno, hw_tf->tf_err, aux);
 }
@@ -390,7 +436,6 @@ static void set_current_ctx_hw(struct per_cpu_info *pcpui,
                                struct hw_trapframe *hw_tf)
 {
        assert(!irq_is_enabled());
-       assert(!pcpui->cur_ctx);
        pcpui->actual_ctx.type = ROS_HW_CTX;
        pcpui->actual_ctx.tf.hw_tf = *hw_tf;
        pcpui->cur_ctx = &pcpui->actual_ctx;
@@ -400,39 +445,31 @@ static void set_current_ctx_sw(struct per_cpu_info *pcpui,
                                struct sw_trapframe *sw_tf)
 {
        assert(!irq_is_enabled());
-       assert(!pcpui->cur_ctx);
        pcpui->actual_ctx.type = ROS_SW_CTX;
        pcpui->actual_ctx.tf.sw_tf = *sw_tf;
        pcpui->cur_ctx = &pcpui->actual_ctx;
 }
 
-/* If the interrupt interrupted a halt, we advance past it.  Made to work with
- * x86's custom cpu_halt() in arch/arch.h.  Note this nearly never gets called.
- * I needed to insert exactly one 'nop' in cpu_halt() (that isn't there now) to
- * get the interrupt to trip on the hlt, o/w the hlt will execute before the
- * interrupt arrives (even with a pending interrupt that should hit right after
- * an interrupt_enable (sti)).  This was on the i7. */
-static void abort_halt(struct hw_trapframe *hw_tf)
+static void set_current_ctx_vm(struct per_cpu_info *pcpui,
+                               struct vm_trapframe *vm_tf)
 {
-       /* Don't care about user TFs.  Incidentally, dereferencing user EIPs is
-        * reading userspace memory, which can be dangerous.  It can page fault,
-        * like immediately after a fork (which doesn't populate the pages). */
-       if (!in_kernel(hw_tf))
-               return;
-       /* the halt instruction in is 0xf4, and it's size is 1 byte */
-       if (*(uint8_t*)x86_get_ip_hw(hw_tf) == 0xf4)
-               x86_advance_ip(hw_tf, 1);
+       assert(!irq_is_enabled());
+       pcpui->actual_ctx.type = ROS_VM_CTX;
+       pcpui->actual_ctx.tf.vm_tf = *vm_tf;
+       pcpui->cur_ctx = &pcpui->actual_ctx;
 }
 
 void trap(struct hw_trapframe *hw_tf)
 {
        struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
        /* Copy out the TF for now */
-       if (!in_kernel(hw_tf))
+       if (!in_kernel(hw_tf)) {
                set_current_ctx_hw(pcpui, hw_tf);
-       else
+               /* ignoring state for nested kernel traps.  should be rare. */
+               __set_cpu_state(pcpui, CPU_STATE_KERNEL);
+       } else {
                inc_ktrap_depth(pcpui);
-
+       }
        printd("Incoming TRAP %d on core %d, TF at %p\n", hw_tf->tf_trapno,
               core_id(), hw_tf);
        if ((hw_tf->tf_cs & ~3) != GD_UT && (hw_tf->tf_cs & ~3) != GD_KT) {
@@ -451,35 +488,42 @@ void trap(struct hw_trapframe *hw_tf)
        assert(0);
 }
 
-/* Note IRQs are disabled unless explicitly turned on.
- *
- * In general, we should only get trapno's >= PIC1_OFFSET (32).  Anything else
- * should be a trap.  Even if we don't use the PIC, that should be the standard.
- * It is possible to get a spurious LAPIC IRQ with vector 15 (or similar), but
- * the spurious check should catch that.
- *
- * Note that from hardware's perspective (PIC, etc), IRQs start from 0, but they
- * are all mapped up at PIC1_OFFSET for the cpu / irq_handler. */
-void handle_irq(struct hw_trapframe *hw_tf)
+static bool vector_is_irq(int apic_vec)
+{
+       /* arguably, we could limit them to MaxIdtIOAPIC */
+       return (IdtPIC <= apic_vec) && (apic_vec <= IdtMAX);
+}
+
+static void irq_dispatch(struct hw_trapframe *hw_tf)
 {
        struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
        struct irq_handler *irq_h;
-       /* Copy out the TF for now */
-       if (!in_kernel(hw_tf))
-               set_current_ctx_hw(pcpui, hw_tf);
+
+       if (!in_irq_ctx(pcpui))
+               __set_cpu_state(pcpui, CPU_STATE_IRQ);
        inc_irq_depth(pcpui);
-       /* Coupled with cpu_halt() and smp_idle() */
-       abort_halt(hw_tf);
        //if (core_id())
        if (hw_tf->tf_trapno != IdtLAPIC_TIMER) /* timer irq */
-       if (hw_tf->tf_trapno != 255) /* kmsg */
-       if (hw_tf->tf_trapno != 36)     /* serial */
+       if (hw_tf->tf_trapno != I_KERNEL_MSG)
+       if (hw_tf->tf_trapno != 65)     /* qemu serial tends to get this one */
                printd("Incoming IRQ, ISR: %d on core %d\n", hw_tf->tf_trapno,
                       core_id());
        /* TODO: RCU read lock */
        irq_h = irq_handlers[hw_tf->tf_trapno];
-       if (!irq_h || irq_h->check_spurious(hw_tf->tf_trapno))
+       if (!irq_h) {
+               warn_once("Received IRQ %d, had no handler registered!",
+                         hw_tf->tf_trapno);
+               /* If we don't have an IRQ handler, we don't know how to EOI.  Odds are,
+                * it's a LAPIC IRQ, such as I_TESTING */
+               if (!lapic_check_spurious(hw_tf->tf_trapno))
+                       lapic_send_eoi(hw_tf->tf_trapno);
                goto out_no_eoi;
+       }
+       if (irq_h->check_spurious(hw_tf->tf_trapno))
+               goto out_no_eoi;
+       /* Can now be interrupted/nested by higher priority IRQs, but not by our
+        * current IRQ vector, til we EOI. */
+       enable_irq();
        while (irq_h) {
                irq_h->isr(hw_tf, irq_h->data);
                irq_h = irq_h->next;
@@ -489,11 +533,33 @@ void handle_irq(struct hw_trapframe *hw_tf)
        if ((I_SMP_CALL0 <= hw_tf->tf_trapno) &&
            (hw_tf->tf_trapno <= I_SMP_CALL_LAST))
                down_checklist(handler_wrappers[hw_tf->tf_trapno & 0x0f].cpu_list);
+       disable_irq();
        /* Keep in sync with ipi_is_pending */
        irq_handlers[hw_tf->tf_trapno]->eoi(hw_tf->tf_trapno);
        /* Fall-through */
 out_no_eoi:
        dec_irq_depth(pcpui);
+       if (!in_irq_ctx(pcpui))
+               __set_cpu_state(pcpui, CPU_STATE_KERNEL);
+}
+
+/* Note IRQs are disabled unless explicitly turned on.
+ *
+ * In general, we should only get trapno's >= PIC1_OFFSET (32).  Anything else
+ * should be a trap.  Even if we don't use the PIC, that should be the standard.
+ * It is possible to get a spurious LAPIC IRQ with vector 15 (or similar), but
+ * the spurious check should catch that.
+ *
+ * Note that from hardware's perspective (PIC, etc), IRQs start from 0, but they
+ * are all mapped up at PIC1_OFFSET for the cpu / irq_handler. */
+void handle_irq(struct hw_trapframe *hw_tf)
+{
+       struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
+
+       /* Copy out the TF for now */
+       if (!in_kernel(hw_tf))
+               set_current_ctx_hw(pcpui, hw_tf);
+       irq_dispatch(hw_tf);
        /* Return to the current process, which should be runnable.  If we're the
         * kernel, we should just return naturally.  Note that current and tf need
         * to still be okay (might not be after blocking) */
@@ -503,115 +569,102 @@ out_no_eoi:
        assert(0);
 }
 
-/* Tells us if an interrupt (trap_nr) came from the PIC or not */
-static bool irq_from_pic(uint32_t trap_nr)
-{
-       /* The 16 IRQs within the range [PIC1_OFFSET, PIC1_OFFSET + 15] came from
-        * the PIC.  [32-47] */
-       if (trap_nr < PIC1_OFFSET)
-               return FALSE;
-       if (trap_nr > PIC1_OFFSET + 15)
-               return FALSE;
-       return TRUE;
-}
-
-/* TODO: remove the distinction btw raw and device IRQs */
-void register_raw_irq(unsigned int vector, isr_t handler, void *data)
+/* The irq field may be ignored based on the type of Bus. */
+int register_irq(int irq, isr_t handler, void *irq_arg, uint32_t tbdf)
 {
        struct irq_handler *irq_h;
-       irq_h = kmalloc(sizeof(struct irq_handler), 0);
+       int vector;
+       irq_h = kzmalloc(sizeof(struct irq_handler), 0);
        assert(irq_h);
-       spin_lock_irqsave(&irq_handler_wlock);
-       irq_h->isr = handler;
-       irq_h->data = data;
-       /* TODO: better way to sort out LAPIC vs PIC */
-       if (irq_from_pic(vector)) {
-               irq_h->check_spurious = pic_check_spurious;
-               irq_h->eoi = pic_send_eoi;
-               irq_h->mask = pic_mask_irq;
-               irq_h->unmask = pic_unmask_irq;
-               irq_h->type = "pic";
-       } else {
-               irq_h->check_spurious = lapic_check_spurious;
-               irq_h->eoi = lapic_send_eoi;
-               /* TODO: which mask we pick also depends on source: IOAPIC, LINT, etc */
-               irq_h->mask = 0;
-               irq_h->unmask = 0;
-               irq_h->type = "lapic";
+       irq_h->dev_irq = irq;
+       irq_h->tbdf = tbdf;
+       vector = bus_irq_setup(irq_h);
+       if (vector == -1) {
+               kfree(irq_h);
+               return -1;
        }
+       printk("IRQ %d, vector %d (0x%x), type %s\n", irq, vector, vector,
+              irq_h->type);
+       assert(irq_h->check_spurious && irq_h->eoi);
+       irq_h->isr = handler;
+       irq_h->data = irq_arg;
        irq_h->apic_vector = vector;
+       /* RCU write lock */
+       spin_lock_irqsave(&irq_handler_wlock);
        irq_h->next = irq_handlers[vector];
        wmb();  /* make sure irq_h is done before publishing to readers */
        irq_handlers[vector] = irq_h;
        spin_unlock_irqsave(&irq_handler_wlock);
+       /* Most IRQs other than the BusIPI should need their irq unmasked.
+        * Might need to pass the irq_h, in case unmask needs more info.
+        * The lapic IRQs need to be unmasked on a per-core basis */
+       if (irq_h->unmask && strcmp(irq_h->type, "lapic"))
+               irq_h->unmask(irq_h, vector);
+       return 0;
 }
 
-void unregister_raw_irq(unsigned int vector, isr_t handler, void *data)
+/* These routing functions only allow the routing of an irq to a single core.
+ * If we want to route to multiple cores, we'll probably need to set up logical
+ * groups or something and take some additional parameters. */
+static int route_irq_h(struct irq_handler *irq_h, int os_coreid)
 {
-       /* TODO: RCU */
-       printk("Unregistering not supported\n");
+       int hw_coreid;
+       if (!irq_h->route_irq) {
+               printk("[kernel] apic_vec %d, type %s cannot be routed\n",
+                      irq_h->apic_vector, irq_h->type);
+               return -1;
+       }
+       if (os_coreid >= MAX_NUM_CORES) {
+               printk("[kernel] os_coreid %d out of range!\n", os_coreid);
+               return -1;
+       }
+       hw_coreid = get_hw_coreid(os_coreid);
+       if (hw_coreid == -1) {
+               printk("[kernel] os_coreid %d not a valid hw core!\n", os_coreid);
+               return -1;
+       }
+       irq_h->route_irq(irq_h, irq_h->apic_vector, hw_coreid);
+       return 0;
 }
 
-/* The devno is arbitrary data. Normally, however, it will be a
- * PCI type-bus-dev.func. It is required for ioapics.
- */
-int register_dev_irq(int irq, isr_t handler, void *irq_arg, uint32_t tbdf)
+/* Routes all irqs for a given apic_vector to os_coreid.  Returns 0 if all of
+ * them succeeded.  -1 if there were none or if any of them failed.  We don't
+ * share IRQs often (if ever anymore), so this shouldn't be an issue. */
+int route_irqs(int apic_vec, int os_coreid)
 {
-       /* TODO: whenever we sort out the ACPI/IOAPIC business, we'll probably want
-        * a helper to reroute an irq? */
-#ifdef CONFIG_ENABLE_MPTABLES
-       /* TODO: dirty hack to get the IOAPIC vector */
-extern int intrenable(int irq, void (*f) (void *, void *), void *a, int tbdf);
-int x =        intrenable(irq, handler, irq_arg, tbdf);
-       if (x > 0)
-               register_raw_irq(x, handler, irq_arg);
-#else
-       register_raw_irq(irq + IdtPIC, handler, irq_arg);
-       pic_unmask_irq(irq + IdtPIC);
-#endif
-       return 0;
+       struct irq_handler *irq_h;
+       int ret = -1;
+       if (!vector_is_irq(apic_vec)) {
+               printk("[kernel] vector %d is not an IRQ vector!\n", apic_vec);
+               return -1;
+       }
+       irq_h = irq_handlers[apic_vec];
+       while (irq_h) {
+               assert(irq_h->apic_vector == apic_vec);
+               ret = route_irq_h(irq_h, os_coreid);
+               irq_h = irq_h->next;
+       }
+       return ret;
 }
 
 /* It's a moderate pain in the ass to put these in bit-specific files (header
  * hell with the set_current_ helpers) */
-#ifdef CONFIG_X86_64
 void sysenter_callwrapper(struct syscall *sysc, unsigned long count,
                           struct sw_trapframe *sw_tf)
 {
        struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
        set_current_ctx_sw(pcpui, sw_tf);
+       __set_cpu_state(pcpui, CPU_STATE_KERNEL);
        /* Once we've set_current_ctx, we can enable interrupts.  This used to be
         * mandatory (we had immediate KMSGs that would muck with cur_ctx).  Now it
         * should only help for sanity/debugging. */
        enable_irq();
-       /* Set up and run the async calls */
+       /* Set up and run the async calls.  This may block, and we could migrate to
+        * another core.  If you use pcpui again, you need to reread it. */
        prep_syscalls(current, sysc, count);
-       /* If you use pcpui again, reread it, since you might have migrated */
-       proc_restartcore();
-}
-
-#else
-
-/* This is called from sysenter's asm, with the tf on the kernel stack. */
-/* TODO: use a sw_tf for sysenter */
-void sysenter_callwrapper(struct hw_trapframe *hw_tf)
-{
-       struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
-       assert(!in_kernel(hw_tf));
-       set_current_ctx_hw(pcpui, hw_tf);
-       /* Once we've set_current_ctx, we can enable interrupts.  This used to be
-        * mandatory (we had immediate KMSGs that would muck with cur_ctx).  Now it
-        * should only help for sanity/debugging. */
-       enable_irq();
-
-       /* Set up and run the async calls */
-       prep_syscalls(current,
-                                 (struct syscall*)x86_get_sysenter_arg0(hw_tf),
-                                 (unsigned int)x86_get_sysenter_arg1(hw_tf));
-       /* If you use pcpui again, reread it, since you might have migrated */
+       disable_irq();
        proc_restartcore();
 }
-#endif
 
 /* Declared in x86/arch.h */
 void send_ipi(uint32_t os_coreid, uint8_t vector)
@@ -623,3 +676,191 @@ void send_ipi(uint32_t os_coreid, uint8_t vector)
        }
        __send_ipi(hw_coreid, vector);
 }
+
+/****************** VM exit handling ******************/
+
+static bool handle_vmexit_cpuid(struct vm_trapframe *tf)
+{
+       uint32_t eax, ebx, ecx, edx;
+
+       cpuid(tf->tf_rax, tf->tf_rcx, &eax, &ebx, &ecx, &edx);
+       tf->tf_rax = eax;
+       tf->tf_rbx = ebx;
+       tf->tf_rcx = ecx;
+       tf->tf_rdx = edx;
+       tf->tf_rip += 2;
+       return TRUE;
+}
+
+static bool handle_vmexit_ept_fault(struct vm_trapframe *tf)
+{
+       int prot = 0;
+       int ret;
+
+       prot |= tf->tf_exit_qual & VMX_EPT_FAULT_READ ? PROT_READ : 0;
+       prot |= tf->tf_exit_qual & VMX_EPT_FAULT_WRITE ? PROT_WRITE : 0;
+       prot |= tf->tf_exit_qual & VMX_EPT_FAULT_INS ? PROT_EXEC : 0;
+       ret = handle_page_fault(current, tf->tf_guest_pa, prot);
+       if (ret) {
+               /* TODO: maybe put ret in the TF somewhere */
+               return FALSE;
+       }
+       return TRUE;
+}
+
+static bool handle_vmexit_nmi(struct vm_trapframe *tf)
+{
+       /* Sanity checks, make sure we really got an NMI.  Feel free to remove. */
+       assert((tf->tf_intrinfo2 & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR);
+       assert((tf->tf_intrinfo2 & INTR_INFO_VECTOR_MASK) == T_NMI);
+       /* our NMI handler from trap.c won't run.  but we don't need the lock
+        * disabling stuff. */
+       extern bool mon_verbose_trace;
+
+       if (mon_verbose_trace) {
+               print_vmtrapframe(tf);
+               /* TODO: a backtrace of the guest would be nice here. */
+       }
+       printk("Core %d is at %p\n", core_id(), get_vmtf_pc(tf));
+       return TRUE;
+}
+
+bool handle_vmexit_msr(struct vm_trapframe *tf)
+{
+       bool ret;
+
+       ret = vmm_emulate_msr(&tf->tf_rcx, &tf->tf_rdx, &tf->tf_rax,
+                             (tf->tf_exit_reason == EXIT_REASON_MSR_READ
+                                                  ? VMM_MSR_EMU_READ : VMM_MSR_EMU_WRITE));
+       if (ret)
+               tf->tf_rip += 2;
+       return ret;
+}
+
+bool handle_vmexit_extirq(struct vm_trapframe *tf)
+{
+       struct hw_trapframe hw_tf;
+
+       /* For now, we just handle external IRQs.  I think guest traps should go to
+        * the guest, based on our vmctls */
+       assert((tf->tf_intrinfo2 & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_EXT_INTR);
+       /* TODO: Our IRQ handlers all expect TFs.  Let's fake one.  A bunch of
+        * handlers (e.g. backtrace/perf) will probably be unhappy about a user TF
+        * that is really a VM, so this all needs work. */
+       hw_tf.tf_gsbase = 0;
+       hw_tf.tf_fsbase = 0;
+       hw_tf.tf_rax = tf->tf_rax;
+       hw_tf.tf_rbx = tf->tf_rbx;
+       hw_tf.tf_rcx = tf->tf_rcx;
+       hw_tf.tf_rdx = tf->tf_rdx;
+       hw_tf.tf_rbp = tf->tf_rbp;
+       hw_tf.tf_rsi = tf->tf_rsi;
+       hw_tf.tf_rdi = tf->tf_rdi;
+       hw_tf.tf_r8 = tf->tf_r8;
+       hw_tf.tf_r9 = tf->tf_r9;
+       hw_tf.tf_r10 = tf->tf_r10;
+       hw_tf.tf_r11 = tf->tf_r11;
+       hw_tf.tf_r12 = tf->tf_r12;
+       hw_tf.tf_r13 = tf->tf_r13;
+       hw_tf.tf_r14 = tf->tf_r14;
+       hw_tf.tf_r15 = tf->tf_r15;
+       hw_tf.tf_trapno = tf->tf_intrinfo2 & INTR_INFO_VECTOR_MASK;
+       hw_tf.tf_err = 0;
+       hw_tf.tf_rip = tf->tf_rip;
+       hw_tf.tf_cs = GD_UT;    /* faking a user TF, even though it's a VM */
+       hw_tf.tf_rflags = tf->tf_rflags;
+       hw_tf.tf_rsp = tf->tf_rsp;
+       hw_tf.tf_ss = GD_UD;
+
+       irq_dispatch(&hw_tf);
+       /* Consider returning whether or not there was a handler registered */
+       return TRUE;
+}
+
+static void vmexit_dispatch(struct vm_trapframe *tf)
+{
+       bool handled = FALSE;
+
+       /* Do not block in any of these functions.
+        *
+        * If we block, we'll probably need to finalize the context.  If we do, then
+        * there's a chance the guest pcore can start somewhere else, and then we
+        * can't get the GPC loaded again.  Plus, they could be running a GPC with
+        * an unresolved vmexit.  It's just mess.
+        *
+        * If we want to enable IRQs, we can do so on a case-by-case basis.  Don't
+        * do it for external IRQs - the irq_dispatch code will handle it. */
+       switch (tf->tf_exit_reason) {
+       case EXIT_REASON_VMCALL:
+               if (current->vmm.flags & VMM_VMCALL_PRINTF) {
+                       printk("%c", tf->tf_rdi);
+                       tf->tf_rip += 3;
+                       handled = TRUE;
+               }
+               break;
+       case EXIT_REASON_CPUID:
+               handled = handle_vmexit_cpuid(tf);
+               break;
+       case EXIT_REASON_EPT_VIOLATION:
+               handled = handle_vmexit_ept_fault(tf);
+               break;
+       case EXIT_REASON_EXCEPTION_NMI:
+               handled = handle_vmexit_nmi(tf);
+               break;
+       case EXIT_REASON_MSR_READ:
+       case EXIT_REASON_MSR_WRITE:
+               handled = handle_vmexit_msr(tf);
+               break;
+       case EXIT_REASON_EXTERNAL_INTERRUPT:
+               handled = handle_vmexit_extirq(tf);
+               break;
+       default:
+               printd("Unhandled vmexit: reason 0x%x, exit qualification 0x%x\n",
+                      tf->tf_exit_reason, tf->tf_exit_qual);
+       }
+       if (!handled) {
+               tf->tf_flags |= VMCTX_FL_HAS_FAULT;
+               if (reflect_current_context()) {
+                       /* VM contexts shouldn't be in vcore context, so this should be
+                        * pretty rare (unlike SCPs or VC ctx page faults). */
+                       printk("[kernel] Unable to reflect VM Exit\n");
+                       print_vmtrapframe(tf);
+                       proc_destroy(current);
+               }
+       }
+}
+
+void handle_vmexit(struct vm_trapframe *tf)
+{
+       struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
+
+       tf->tf_rip = vmcs_read(GUEST_RIP);
+       tf->tf_rflags = vmcs_read(GUEST_RFLAGS);
+       tf->tf_rsp = vmcs_read(GUEST_RSP);
+       tf->tf_cr2 = rcr2();
+       tf->tf_cr3 = vmcs_read(GUEST_CR3);
+       tf->tf_guest_pcoreid = pcpui->guest_pcoreid;
+       tf->tf_flags |= VMCTX_FL_PARTIAL;
+       tf->tf_exit_reason = vmcs_read(VM_EXIT_REASON);
+       tf->tf_exit_qual = vmcs_read(EXIT_QUALIFICATION);
+       tf->tf_intrinfo1 = vmcs_read(GUEST_INTERRUPTIBILITY_INFO);
+       tf->tf_intrinfo2 = vmcs_read(VM_EXIT_INTR_INFO);
+       tf->tf_guest_va = vmcs_read(GUEST_LINEAR_ADDRESS);
+       tf->tf_guest_pa = vmcs_read(GUEST_PHYSICAL_ADDRESS);
+
+       set_current_ctx_vm(pcpui, tf);
+       tf = &pcpui->cur_ctx->tf.vm_tf;
+       vmexit_dispatch(tf);
+       /* We're either restarting a partial VM ctx (vmcs was launched, loaded on
+        * the core, etc) or a SW vc ctx for the reflected trap.  Or the proc is
+        * dying and we'll handle a __death KMSG shortly. */
+       proc_restartcore();
+}
+
+void x86_finalize_vmtf(struct vm_trapframe *tf)
+{
+       struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
+
+       x86_vmtf_clear_partial(tf);
+       unload_guest_pcore(pcpui->cur_proc, pcpui->guest_pcoreid);
+}