VMX: only check the PB VM EC2 if EC1 is ok
[akaros.git] / kern / arch / x86 / vmm / intel / vmx.c
1 //#define DEBUG
2 /**
3  *  vmx.c - The Intel VT-x driver for Dune
4  *
5  * This file is derived from Linux KVM VT-x support.
6  * Copyright (C) 2006 Qumranet, Inc.
7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
8  *
9  * Original Authors:
10  *   Avi Kivity   <avi@qumranet.com>
11  *   Yaniv Kamay  <yaniv@qumranet.com>
12  *
13  * This modified version is simpler because it avoids the following
14  * features that are not requirements for Dune:
15  *  * Real-mode emulation
16  *  * Nested VT-x support
17  *  * I/O hardware emulation
18  *  * Any of the more esoteric X86 features and registers
19  *  * KVM-specific functionality
20  *
21  * In essence we provide only the minimum functionality needed to run
22  * a process in vmx non-root mode rather than the full hardware emulation
23  * needed to support an entire OS.
24  *
25  * This driver is a research prototype and as such has the following
26  * limitations:
27  *
28  * FIXME: Backward compatability is currently a non-goal, and only recent
29  * full-featured (EPT, PCID, VPID, etc.) Intel hardware is supported by this
30  * driver.
31  *
32  * FIXME: Eventually we should handle concurrent user's of VT-x more
33  * gracefully instead of requiring exclusive access. This would allow
34  * Dune to interoperate with KVM and other HV solutions.
35  *
36  * FIXME: We need to support hotplugged physical CPUs.
37  *
38  * Authors:
39  *   Adam Belay   <abelay@stanford.edu>
40  */
41
42 /* Basic flow.
43  * Yep, it's confusing. This is in part because the vmcs is used twice, for two different things.
44  * You're left with the feeling that they got part way through and realized they had to have one for
45  *
46  * 1) your CPU is going to be capable of running VMs, and you need state for that.
47  *
48  * 2) you're about to start a guest, and you need state for that.
49  *
50  * So there is get cpu set up to be able to run VMs stuff, and now
51  * let's start a guest stuff.  In Akaros, CPUs will always be set up
52  * to run a VM if that is possible. Processes can flip themselves into
53  * a VM and that will require another VMCS.
54  *
55  * So: at kernel startup time, the SMP boot stuff calls
56  * k/a/x86/vmm/vmm.c:vmm_init, which calls arch-dependent bits, which
57  * in the case of this file is intel_vmm_init. That does some code
58  * that sets up stuff for ALL sockets, based on the capabilities of
59  * the socket it runs on. If any cpu supports vmx, it assumes they all
60  * do. That's a realistic assumption. So the call_function_all is kind
61  * of stupid, really; it could just see what's on the current cpu and
62  * assume it's on all. HOWEVER: there are systems in the wilde that
63  * can run VMs on some but not all CPUs, due to BIOS mistakes, so we
64  * might as well allow for the chance that wel'll only all VMMCPs on a
65  * subset (not implemented yet however).  So: probe all CPUs, get a
66  * count of how many support VMX and, for now, assume they all do
67  * anyway.
68  *
69  * Next, call setup_vmcs_config to configure the GLOBAL vmcs_config struct,
70  * which contains all the naughty bits settings for all the cpus that can run a VM.
71  * Realistically, all VMX-capable cpus in a system will have identical configurations.
72  * So: 0 or more cpus can run VMX; all cpus which can run VMX will have the same configuration.
73  *
74  * configure the msr_bitmap. This is the bitmap of MSRs which the
75  * guest can manipulate.  Currently, we only allow GS and FS base.
76  *
77  * Reserve bit 0 in the vpid bitmap as guests can not use that
78  *
79  * Set up the what we call the vmxarea. The vmxarea is per-cpu, not
80  * per-guest. Once set up, it is left alone.  The ONLY think we set in
81  * there is the revision area. The VMX is page-sized per cpu and
82  * page-aligned. Note that it can be smaller, but why bother? We know
83  * the max size and alightment, and it's convenient.
84  *
85  * Now that it is set up, enable vmx on all cpus. This involves
86  * testing VMXE in cr4, to see if we've been here before (TODO: delete
87  * this test), then testing MSR_IA32_FEATURE_CONTROL to see if we can
88  * do a VM, the setting the VMXE in cr4, calling vmxon (does a vmxon
89  * instruction), and syncing vpid's and ept's.  Now the CPU is ready
90  * to host guests.
91  *
92  * Setting up a guest.
93  * We divide this into two things: vmm_proc_init and vm_run.
94  * Currently, on Intel, vmm_proc_init does nothing.
95  *
96  * vm_run is really complicated. It is called with a coreid, rip, rsp,
97  * cr3, and flags.  On intel, it calls vmx_launch. vmx_launch is set
98  * up for a few test cases. If rip is 1, it sets the guest rip to
99  * a function which will deref 0 and should exit with failure 2. If rip is 0,
100  * it calls an infinite loop in the guest.
101  *
102  * The sequence of operations:
103  * create a vcpu
104  * while (1) {
105  * get a vcpu
106  * disable irqs (required or you can't enter the VM)
107  * vmx_run_vcpu()
108  * enable irqs
109  * manage the vm exit
110  * }
111  *
112  * get a vcpu
113  * See if the current cpu has a vcpu. If so, and is the same as the vcpu we want,
114  * vmcs_load(vcpu->vmcs) -- i.e. issue a VMPTRLD.
115  *
116  * If it's not the same, see if the vcpu thinks it is on the core. If it is not, call
117  * __vmx_get_cpu_helper on the other cpu, to free it up. Else vmcs_clear the one
118  * attached to this cpu. Then vmcs_load the vmcs for vcpu on this this cpu,
119  * call __vmx_setup_cpu, mark this vcpu as being attached to this cpu, done.
120  *
121  * vmx_run_vcpu this one gets messy, mainly because it's a giant wad
122  * of inline assembly with embedded CPP crap. I suspect we'll want to
123  * un-inline it someday, but maybe not.  It's called with a vcpu
124  * struct from which it loads guest state, and to which it stores
125  * non-virtualized host state. It issues a vmlaunch or vmresume
126  * instruction depending, and on return, it evaluates if things the
127  * launch/resume had an error in that operation. Note this is NOT the
128  * same as an error while in the virtual machine; this is an error in
129  * startup due to misconfiguration. Depending on whatis returned it's
130  * either a failed vm startup or an exit for lots of many reasons.
131  *
132  */
133
134 /* basically: only rename those globals that might conflict
135  * with existing names. Leave all else the same.
136  * this code is more modern than the other code, yet still
137  * well encapsulated, it seems.
138  */
139 #include <kmalloc.h>
140 #include <string.h>
141 #include <stdio.h>
142 #include <assert.h>
143 #include <error.h>
144 #include <pmap.h>
145 #include <sys/queue.h>
146 #include <smp.h>
147 #include <kref.h>
148 #include <atomic.h>
149 #include <alarm.h>
150 #include <event.h>
151 #include <umem.h>
152 #include <bitops.h>
153 #include <arch/types.h>
154 #include <syscall.h>
155
156 #include "vmx.h"
157 #include "../vmm.h"
158 #include <ros/vmm.h>
159
160 #include "cpufeature.h"
161
162 #define currentcpu (&per_cpu_info[core_id()])
163
164 static unsigned long *msr_bitmap;
165
166 int x86_ept_pte_fix_ups = 0;
167
168 struct vmx_capability vmx_capability;
169 struct vmcs_config vmcs_config;
170
171 static int autoloaded_msrs[] = {
172         MSR_KERNEL_GS_BASE,
173         MSR_LSTAR,
174         MSR_STAR,
175         MSR_SFMASK,
176 };
177
178 static char *cr_access_type[] = {
179         "move to cr",
180         "move from cr",
181         "clts",
182         "lmsw"
183 };
184
185 static char *cr_gpr[] = {
186         "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
187         "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
188 };
189
190 static int guest_cr_num[16] = {
191         GUEST_CR0,
192         -1,
193         -1,
194         GUEST_CR3,
195         GUEST_CR4,
196         -1,
197         -1,
198         -1,
199         -1, /* 8? */
200         -1, -1, -1, -1, -1, -1, -1
201 };
202 __always_inline unsigned long vmcs_readl(unsigned long field);
203 /* See section 24-3 of The Good Book */
204 void show_cr_access(uint64_t val) {
205         int crnr = val & 0xf;
206         int type = (val>>4) & 3;
207         int reg = (val >> 11) & 0xf;
208         printk("%s: %d: ", cr_access_type[type], crnr);
209         if (type < 2) {
210                 printk("%s", cr_gpr[reg]);
211                 if (guest_cr_num[crnr] > -1) {
212                         printk(": 0x%x", vmcs_readl(guest_cr_num[crnr]));
213                 }
214         }
215         printk("\n");
216 }
217
218 void ept_flush(uint64_t eptp)
219 {
220         ept_sync_context(eptp);
221 }
222
223 static void vmcs_clear(struct vmcs *vmcs)
224 {
225         uint64_t phys_addr = PADDR(vmcs);
226         uint8_t error;
227
228         asm volatile (ASM_VMX_VMCLEAR_RAX "; setna %0"
229                       : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
230                       : "cc", "memory");
231         if (error)
232                 printk("vmclear fail: %p/%llx\n",
233                        vmcs, phys_addr);
234 }
235
236 static void vmcs_load(struct vmcs *vmcs)
237 {
238         uint64_t phys_addr = PADDR(vmcs);
239         uint8_t error;
240
241         asm volatile (ASM_VMX_VMPTRLD_RAX "; setna %0"
242                         : "=qm"(error) : "a"(&phys_addr), "m"(phys_addr)
243                         : "cc", "memory");
244         if (error)
245                 printk("vmptrld %p/%llx failed\n",
246                        vmcs, phys_addr);
247 }
248
249 /* Returns the paddr pointer of the current CPU's VMCS region, or -1 if none. */
250 static physaddr_t vmcs_get_current(void)
251 {
252         physaddr_t vmcs_paddr;
253         /* RAX contains the addr of the location to store the VMCS pointer.  The
254          * compiler doesn't know the ASM will deref that pointer, hence the =m */
255         asm volatile (ASM_VMX_VMPTRST_RAX : "=m"(vmcs_paddr) : "a"(&vmcs_paddr));
256         return vmcs_paddr;
257 }
258
259 __always_inline unsigned long vmcs_readl(unsigned long field)
260 {
261         unsigned long value;
262
263         asm volatile (ASM_VMX_VMREAD_RDX_RAX
264                       : "=a"(value) : "d"(field) : "cc");
265         return value;
266 }
267
268 __always_inline uint16_t vmcs_read16(unsigned long field)
269 {
270         return vmcs_readl(field);
271 }
272
273 static __always_inline uint32_t vmcs_read32(unsigned long field)
274 {
275         return vmcs_readl(field);
276 }
277
278 static __always_inline uint64_t vmcs_read64(unsigned long field)
279 {
280         return vmcs_readl(field);
281 }
282
283 void vmwrite_error(unsigned long field, unsigned long value)
284 {
285         printk("vmwrite error: reg %lx value %lx (err %d)\n",
286                field, value, vmcs_read32(VM_INSTRUCTION_ERROR));
287 }
288
289 void vmcs_writel(unsigned long field, unsigned long value)
290 {
291         uint8_t error;
292
293         asm volatile (ASM_VMX_VMWRITE_RAX_RDX "; setna %0"
294                        : "=q"(error) : "a"(value), "d"(field) : "cc");
295         if (error)
296                 vmwrite_error(field, value);
297 }
298
299 static void vmcs_write16(unsigned long field, uint16_t value)
300 {
301         vmcs_writel(field, value);
302 }
303
304 static void vmcs_write32(unsigned long field, uint32_t value)
305 {
306         vmcs_writel(field, value);
307 }
308
309 static void vmcs_write64(unsigned long field, uint64_t value)
310 {
311         vmcs_writel(field, value);
312 }
313
314 /*
315  * A note on Things You Can't Make Up.
316  * or
317  * "George, you can type this shit, but you can't say it" -- Harrison Ford
318  *
319  * There are 5 VMCS 32-bit words that control guest permissions. If
320  * you set these correctly, you've got a guest that will behave. If
321  * you get even one bit wrong, you've got a guest that will chew your
322  * leg off. Some bits must be 1, some must be 0, and some can be set
323  * either way. To add to the fun, the docs are sort of a docudrama or,
324  * as the quote goes, "interesting if true."
325  *
326  * To determine what bit can be set in what VMCS 32-bit control word,
327  * there are 5 corresponding 64-bit MSRs.  And, to make it even more
328  * fun, the standard set of MSRs have errors in them, i.e. report
329  * incorrect values, for legacy reasons, and so you are supposed to
330  * "look around" to another set, which have correct bits in
331  * them. There are four such 'correct' registers, and they have _TRUE_
332  * in the names as you can see below. We test for the value of VMCS
333  * control bits in the _TRUE_ registers if possible. The fifth
334  * register, CPU Secondary Exec Controls, which came later, needs no
335  * _TRUE_ variant.
336  *
337  * For each MSR, the high 32 bits tell you what bits can be "1" by a
338  * "1" in that position; the low 32 bits tell you what bit can be "0"
339  * by a "0" in that position. So, for each of 32 bits in a given VMCS
340  * control word, there is a pair of bits in an MSR that tells you what
341  * values it can take. The two bits, of which there are *four*
342  * combinations, describe the *three* possible operations on a
343  * bit. The two bits, taken together, form an untruth table: There are
344  * three possibilities: The VMCS bit can be set to 0 or 1, or it can
345  * only be 0, or only 1. The fourth combination is not supposed to
346  * happen.
347  *
348  * So: there is the 1 bit from the upper 32 bits of the msr.
349  * If this bit is set, then the bit can be 1. If clear, it can not be 1.
350  *
351  * Then there is the 0 bit, from low 32 bits. If clear, the VMCS bit
352  * can be 0. If 1, the VMCS bit can not be 0.
353  *
354  * SO, let's call the 1 bit R1, and the 0 bit R0, we have:
355  *  R1 R0
356  *  0 0 -> must be 0
357  *  1 0 -> can be 1, can be 0
358  *  0 1 -> can not be 1, can not be 0. --> JACKPOT! Not seen yet.
359  *  1 1 -> must be one.
360  *
361  * It's also pretty hard to know what you can and can't set, and
362  * that's led to inadvertant opening of permissions at times.  Because
363  * of this complexity we've decided on the following: the driver must
364  * define EVERY bit, UNIQUELY, for each of the 5 registers, that it wants
365  * set. Further, for any bit that's settable, the driver must specify
366  * a setting; for any bit that's reserved, the driver settings must
367  * match that bit. If there are reserved bits we don't specify, that's
368  * ok; we'll take them as is.
369  *
370  * We use a set-means-set, and set-means-clear model, i.e. we use a
371  * 32-bit word to contain the bits we want to be 1, indicated by one;
372  * and another 32-bit word in which a bit we want to be 0 is indicated
373  * by a 1. This allows us to easily create masks of all bits we're
374  * going to set, for example.
375  *
376  * We have two 32-bit numbers for each 32-bit VMCS field: bits we want
377  * set and bits we want clear.  If you read the MSR for that field,
378  * compute the reserved 0 and 1 settings, and | them together, they
379  * need to result in 0xffffffff. You can see that we can create other
380  * tests for conflicts (i.e. overlap).
381  *
382  * At this point, I've tested check_vmx_controls in every way
383  * possible, beause I kept screwing the bitfields up. You'll get a nice
384  * error it won't work at all, which is what we want: a
385  * failure-prone setup, where even errors that might result in correct
386  * values are caught -- "right answer, wrong method, zero credit." If there's
387  * weirdness in the bits, we don't want to run.
388  */
389
390 static bool check_vmxec_controls(struct vmxec const *v, bool have_true_msr,
391                                  uint32_t *result)
392 {
393         bool err = false;
394         uint32_t vmx_msr_low, vmx_msr_high;
395         uint32_t reserved_0, reserved_1, changeable_bits;
396
397         if (have_true_msr)
398                 rdmsr(v->truemsr, vmx_msr_low, vmx_msr_high);
399         else
400                 rdmsr(v->msr, vmx_msr_low, vmx_msr_high);
401
402         if (vmx_msr_low & ~vmx_msr_high)
403                 warn("JACKPOT: Conflicting VMX ec ctls for %s, high 0x%08x low 0x%08x",
404                      v->name, vmx_msr_high, vmx_msr_low);
405
406         reserved_0 = (~vmx_msr_low) & (~vmx_msr_high);
407         reserved_1 = vmx_msr_low & vmx_msr_high;
408         changeable_bits = ~(reserved_0 | reserved_1);
409
410         /*
411          * this is very much as follows:
412          * accept the things I cannot change,
413          * change the things I can,
414          * know the difference.
415          */
416
417         /* Conflict. Don't try to both set and reset bits. */
418         if (v->set_to_0 & v->set_to_1) {
419                 printk("%s: set to 0 (0x%x) and set to 1 (0x%x) overlap: 0x%x\n",
420                        v->name, v->set_to_0, v->set_to_1, v->set_to_0 & v->set_to_1);
421                 err = true;
422         }
423
424         /* coverage */
425         if (((v->set_to_0 | v->set_to_1) & changeable_bits) !=
426             changeable_bits) {
427                 printk("%s: Need to cover 0x%x and have 0x%x,0x%x\n",
428                        v->name, changeable_bits, v->set_to_0,  v->set_to_1);
429                 err = true;
430         }
431
432         if ((v->set_to_0 | v->set_to_1 | reserved_0 | reserved_1) !=
433             0xffffffff) {
434                 printk("%s: incomplete coverage: have 0x%x, want 0x%x\n",
435                        v->name, v->set_to_0 | v->set_to_1 |
436                        reserved_0 | reserved_1, 0xffffffff);
437                 err = true;
438         }
439
440         /* Don't try to change bits that can't be changed. */
441         if ((v->set_to_0 & (reserved_0 | changeable_bits)) != v->set_to_0) {
442                 printk("%s: set to 0 (0x%x) can't be done\n", v->name,
443                         v->set_to_0);
444                 err = true;
445         }
446
447         if ((v->set_to_1 & (reserved_1 | changeable_bits)) != v->set_to_1) {
448                 printk("%s: set to 1 (0x%x) can't be done\n",
449                        v->name, v->set_to_1);
450                 err = true;
451         }
452
453         /* If there's been any error at all, spill our guts and return. */
454         if (err) {
455                 printk("%s: vmx_msr_high 0x%x, vmx_msr_low 0x%x, ",
456                        v->name, vmx_msr_high, vmx_msr_low);
457                 printk("set_to_1 0x%x,set_to_0 0x%x,reserved_1 0x%x",
458                        v->set_to_1, v->set_to_0, reserved_1);
459                 printk(" reserved_0 0x%x", reserved_0);
460                 printk(" changeable_bits 0x%x\n", changeable_bits);
461                 return false;
462         }
463
464         *result = v->set_to_1 | reserved_1;
465
466         printd("%s: check_vmxec_controls succeeds with result 0x%x\n",
467                v->name, *result);
468         return true;
469 }
470
471 /*
472  * We're trying to make this as readable as possible. Realistically, it will
473  * rarely if ever change, if the past is any guide.
474  */
475 static const struct vmxec pbec = {
476         .name = "Pin Based Execution Controls",
477         .msr = MSR_IA32_VMX_PINBASED_CTLS,
478         .truemsr = MSR_IA32_VMX_TRUE_PINBASED_CTLS,
479
480         .set_to_1 = (PIN_BASED_EXT_INTR_MASK |
481                      PIN_BASED_NMI_EXITING |
482                      PIN_BASED_VIRTUAL_NMIS),
483
484         .set_to_0 = (PIN_BASED_VMX_PREEMPTION_TIMER |
485                      PIN_BASED_POSTED_INTR),
486 };
487
488 static const struct vmxec cbec = {
489         .name = "CPU Based Execution Controls",
490         .msr = MSR_IA32_VMX_PROCBASED_CTLS,
491         .truemsr = MSR_IA32_VMX_TRUE_PROCBASED_CTLS,
492
493         .set_to_1 = (CPU_BASED_HLT_EXITING |
494                      CPU_BASED_INVLPG_EXITING |
495                      CPU_BASED_MWAIT_EXITING |
496                      CPU_BASED_RDPMC_EXITING |
497                      CPU_BASED_CR8_LOAD_EXITING |
498                      CPU_BASED_CR8_STORE_EXITING |
499                      CPU_BASED_MOV_DR_EXITING |
500                      CPU_BASED_UNCOND_IO_EXITING |
501                      CPU_BASED_USE_MSR_BITMAPS |
502                      CPU_BASED_MONITOR_EXITING |
503                      CPU_BASED_ACTIVATE_SECONDARY_CONTROLS),
504
505         .set_to_0 = (CPU_BASED_VIRTUAL_INTR_PENDING |
506                      CPU_BASED_USE_TSC_OFFSETING |
507                      CPU_BASED_RDTSC_EXITING |
508                      CPU_BASED_CR3_LOAD_EXITING |
509                      CPU_BASED_CR3_STORE_EXITING |
510                      CPU_BASED_TPR_SHADOW |
511                      CPU_BASED_VIRTUAL_NMI_PENDING |
512                      CPU_BASED_MONITOR_TRAP |
513                      CPU_BASED_PAUSE_EXITING |
514                      CPU_BASED_USE_IO_BITMAPS),
515 };
516
517 static const struct vmxec cb2ec = {
518         .name = "CPU Based 2nd Execution Controls",
519         .msr = MSR_IA32_VMX_PROCBASED_CTLS2,
520         .truemsr = MSR_IA32_VMX_PROCBASED_CTLS2,
521
522         .set_to_1 = (SECONDARY_EXEC_ENABLE_EPT |
523                      SECONDARY_EXEC_WBINVD_EXITING),
524
525         .set_to_0 = (SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
526                      SECONDARY_EXEC_DESCRIPTOR_EXITING |
527                      SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
528                      SECONDARY_EXEC_ENABLE_VPID |
529                      SECONDARY_EXEC_UNRESTRICTED_GUEST |
530                      SECONDARY_EXEC_APIC_REGISTER_VIRT |
531                      SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
532                      SECONDARY_EXEC_PAUSE_LOOP_EXITING |
533                      SECONDARY_EXEC_RDRAND_EXITING |
534                      SECONDARY_EXEC_ENABLE_INVPCID |
535                      SECONDARY_EXEC_ENABLE_VMFUNC |
536                      SECONDARY_EXEC_SHADOW_VMCS |
537                      SECONDARY_EXEC_RDSEED_EXITING |
538                      SECONDARY_EPT_VE |
539                      /* TODO: re enable this via a "Want" struct
540                         member at some point */
541                      SECONDARY_EXEC_RDTSCP |
542                      SECONDARY_ENABLE_XSAV_RESTORE)
543 };
544
545 static const struct vmxec vmentry = {
546         .name = "VMENTRY controls",
547         .msr = MSR_IA32_VMX_ENTRY_CTLS,
548         .truemsr = MSR_IA32_VMX_TRUE_ENTRY_CTLS,
549         /* exact order from vmx.h; only the first two are enabled. */
550
551         .set_to_1 =  (VM_ENTRY_LOAD_DEBUG_CONTROLS | /* can't set to 0 */
552                       VM_ENTRY_LOAD_IA32_EFER |
553                       VM_ENTRY_IA32E_MODE),
554
555         .set_to_0 = (VM_ENTRY_SMM |
556                      VM_ENTRY_DEACT_DUAL_MONITOR |
557                      VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL |
558                      VM_ENTRY_LOAD_IA32_PAT),
559 };
560
561 static const struct vmxec vmexit = {
562         .name = "VMEXIT controls",
563         .msr = MSR_IA32_VMX_EXIT_CTLS,
564         .truemsr = MSR_IA32_VMX_TRUE_EXIT_CTLS,
565
566         .set_to_1 = (VM_EXIT_SAVE_DEBUG_CONTROLS | /* can't set to 0 */
567                      VM_EXIT_SAVE_IA32_EFER |
568                      VM_EXIT_LOAD_IA32_EFER |
569                      VM_EXIT_HOST_ADDR_SPACE_SIZE), /* 64 bit */
570
571         .set_to_0 = (VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL |
572                      VM_EXIT_ACK_INTR_ON_EXIT |
573                      VM_EXIT_SAVE_IA32_PAT |
574                      VM_EXIT_LOAD_IA32_PAT |
575                      VM_EXIT_SAVE_VMX_PREEMPTION_TIMER),
576 };
577
578 static void setup_vmcs_config(void *p)
579 {
580         int *ret = p;
581         struct vmcs_config *vmcs_conf = &vmcs_config;
582         uint32_t vmx_msr_high;
583         uint64_t vmx_msr;
584         bool have_true_msrs = false;
585         bool ok;
586
587         *ret = -EIO;
588
589         vmx_msr = read_msr(MSR_IA32_VMX_BASIC);
590         vmx_msr_high = vmx_msr >> 32;
591
592         /*
593          * If bit 55 (VMX_BASIC_HAVE_TRUE_MSRS) is set, then we
594          * can go for the true MSRs.  Else, we ask you to get a better CPU.
595          */
596         if (vmx_msr & VMX_BASIC_TRUE_CTLS) {
597                 have_true_msrs = true;
598                 printd("Running with TRUE MSRs\n");
599         } else {
600                 printk("Running with non-TRUE MSRs, this is old hardware\n");
601         }
602
603         /*
604          * Don't worry that one or more of these might fail and leave
605          * the VMCS in some kind of incomplete state. If one of these
606          * fails, the caller is going to discard the VMCS.
607          * It is written this way to ensure we get results of all tests and avoid
608          * BMAFR behavior.
609          */
610         ok = check_vmxec_controls(&pbec, have_true_msrs,
611                                   &vmcs_conf->pin_based_exec_ctrl);
612         ok = check_vmxec_controls(&cbec, have_true_msrs,
613                                   &vmcs_conf->cpu_based_exec_ctrl) && ok;
614         /* Only check cb2ec if we're still ok, o/w we may GPF */
615         ok = ok && check_vmxec_controls(&cb2ec, have_true_msrs,
616                                         &vmcs_conf->cpu_based_2nd_exec_ctrl);
617         ok = check_vmxec_controls(&vmentry, have_true_msrs,
618                                   &vmcs_conf->vmentry_ctrl) && ok;
619         ok = check_vmxec_controls(&vmexit, have_true_msrs,
620                                   &vmcs_conf->vmexit_ctrl) && ok;
621         if (! ok) {
622                 printk("vmxexec controls is no good.\n");
623                 return;
624         }
625
626         /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
627         if ((vmx_msr_high & 0x1fff) > PGSIZE) {
628                 printk("vmx_msr_high & 0x1fff) is 0x%x, > PAGE_SIZE 0x%x\n",
629                        vmx_msr_high & 0x1fff, PGSIZE);
630                 return;
631         }
632
633         /* IA-32 SDM Vol 3B: 64-bit CPUs always have VMX_BASIC_MSR[48]==0. */
634         if (vmx_msr & VMX_BASIC_64) {
635                 printk("VMX doesn't support 64 bit width!\n");
636                 return;
637         }
638
639         if (((vmx_msr & VMX_BASIC_MEM_TYPE_MASK) >> VMX_BASIC_MEM_TYPE_SHIFT)
640             != VMX_BASIC_MEM_TYPE_WB) {
641                 printk("VMX doesn't support WB memory for VMCS accesses!\n");
642                 return;
643         }
644
645         vmcs_conf->size = vmx_msr_high & 0x1fff;
646         vmcs_conf->order = LOG2_UP(nr_pages(vmcs_config.size));
647         vmcs_conf->revision_id = (uint32_t)vmx_msr;
648
649         /* Read in the caps for runtime checks.  This MSR is only available if
650          * secondary controls and ept or vpid is on, which we check earlier */
651         rdmsr(MSR_IA32_VMX_EPT_VPID_CAP, vmx_capability.ept, vmx_capability.vpid);
652
653         *ret = 0;
654 }
655
656 static struct vmcs *__vmx_alloc_vmcs(int node)
657 {
658         struct vmcs *vmcs;
659
660         vmcs = get_cont_pages_node(node, vmcs_config.order, KMALLOC_WAIT);
661         if (!vmcs)
662                 return 0;
663         memset(vmcs, 0, vmcs_config.size);
664         vmcs->revision_id = vmcs_config.revision_id;    /* vmcs revision id */
665         printd("%d: set rev id %d\n", core_id(), vmcs->revision_id);
666         return vmcs;
667 }
668
669 /**
670  * vmx_alloc_vmcs - allocates a VMCS region
671  *
672  * NOTE: Assumes the new region will be used by the current CPU.
673  *
674  * Returns a valid VMCS region.
675  */
676 static struct vmcs *vmx_alloc_vmcs(void)
677 {
678         return __vmx_alloc_vmcs(numa_id());
679 }
680
681 /**
682  * vmx_free_vmcs - frees a VMCS region
683  */
684 static void vmx_free_vmcs(struct vmcs *vmcs)
685 {
686   //free_pages((unsigned long)vmcs, vmcs_config.order);
687 }
688
689 /*
690  * Set up the vmcs's constant host-state fields, i.e., host-state fields that
691  * will not change in the lifetime of the guest.
692  * Note that host-state that does change is set elsewhere. E.g., host-state
693  * that is set differently for each CPU is set in vmx_vcpu_load(), not here.
694  */
695 static void vmx_setup_constant_host_state(void)
696 {
697         uint32_t low32, high32;
698         unsigned long tmpl;
699         pseudodesc_t dt;
700
701         vmcs_writel(HOST_CR0, rcr0() & ~X86_CR0_TS);  /* 22.2.3 */
702         vmcs_writel(HOST_CR4, rcr4());  /* 22.2.3, 22.2.5 */
703         vmcs_writel(HOST_CR3, rcr3());  /* 22.2.3 */
704
705         vmcs_write16(HOST_CS_SELECTOR, GD_KT);  /* 22.2.4 */
706         vmcs_write16(HOST_DS_SELECTOR, GD_KD);  /* 22.2.4 */
707         vmcs_write16(HOST_ES_SELECTOR, GD_KD);  /* 22.2.4 */
708         vmcs_write16(HOST_SS_SELECTOR, GD_KD);  /* 22.2.4 */
709         vmcs_write16(HOST_TR_SELECTOR, GD_TSS);  /* 22.2.4 */
710
711         native_store_idt(&dt);
712         vmcs_writel(HOST_IDTR_BASE, dt.pd_base);   /* 22.2.4 */
713
714         asm("mov $.Lkvm_vmx_return, %0" : "=r"(tmpl));
715         vmcs_writel(HOST_RIP, tmpl); /* 22.2.5 */
716
717         rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
718         vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
719         rdmsrl(MSR_IA32_SYSENTER_EIP, tmpl);
720         vmcs_writel(HOST_IA32_SYSENTER_EIP, tmpl);   /* 22.2.3 */
721
722         rdmsr(MSR_EFER, low32, high32);
723         vmcs_write32(HOST_IA32_EFER, low32);
724
725         if (vmcs_config.vmexit_ctrl & VM_EXIT_LOAD_IA32_PAT) {
726                 rdmsr(MSR_IA32_CR_PAT, low32, high32);
727                 vmcs_write64(HOST_IA32_PAT, low32 | ((uint64_t) high32 << 32));
728         }
729
730         vmcs_write16(HOST_FS_SELECTOR, 0);            /* 22.2.4 */
731         vmcs_write16(HOST_GS_SELECTOR, 0);            /* 22.2.4 */
732
733         /* TODO: This (at least gs) is per cpu */
734         rdmsrl(MSR_FS_BASE, tmpl);
735         vmcs_writel(HOST_FS_BASE, tmpl); /* 22.2.4 */
736         rdmsrl(MSR_GS_BASE, tmpl);
737         vmcs_writel(HOST_GS_BASE, tmpl); /* 22.2.4 */
738 }
739
740 static inline uint16_t vmx_read_ldt(void)
741 {
742         uint16_t ldt;
743         asm("sldt %0" : "=g"(ldt));
744         return ldt;
745 }
746
747 static unsigned long segment_base(uint16_t selector)
748 {
749         pseudodesc_t *gdt = &currentcpu->host_gdt;
750         struct desc_struct *d;
751         unsigned long table_base;
752         unsigned long v;
753
754         if (!(selector & ~3)) {
755                 return 0;
756         }
757
758         table_base = gdt->pd_base;
759
760         if (selector & 4) {           /* from ldt */
761                 uint16_t ldt_selector = vmx_read_ldt();
762
763                 if (!(ldt_selector & ~3)) {
764                         return 0;
765                 }
766
767                 table_base = segment_base(ldt_selector);
768         }
769         d = (struct desc_struct *)(table_base + (selector & ~7));
770         v = get_desc_base(d);
771         if (d->s == 0 && (d->type == 2 || d->type == 9 || d->type == 11))
772                 v |= ((unsigned long)((struct ldttss_desc64 *)d)->base3) << 32;
773         return v;
774 }
775
776 static inline unsigned long vmx_read_tr_base(void)
777 {
778         uint16_t tr;
779         asm("str %0" : "=g"(tr));
780         return segment_base(tr);
781 }
782
783 static void __vmx_setup_cpu(void)
784 {
785         pseudodesc_t *gdt = &currentcpu->host_gdt;
786         unsigned long sysenter_esp;
787         unsigned long tmpl;
788
789         /*
790          * Linux uses per-cpu TSS and GDT, so set these when switching
791          * processors.
792          */
793         vmcs_writel(HOST_TR_BASE, vmx_read_tr_base()); /* 22.2.4 */
794         vmcs_writel(HOST_GDTR_BASE, gdt->pd_base);   /* 22.2.4 */
795
796         rdmsrl(MSR_IA32_SYSENTER_ESP, sysenter_esp);
797         vmcs_writel(HOST_IA32_SYSENTER_ESP, sysenter_esp); /* 22.2.3 */
798
799         rdmsrl(MSR_FS_BASE, tmpl);
800         vmcs_writel(HOST_FS_BASE, tmpl); /* 22.2.4 */
801         rdmsrl(MSR_GS_BASE, tmpl);
802         vmcs_writel(HOST_GS_BASE, tmpl); /* 22.2.4 */
803 }
804
805 /**
806  * vmx_get_cpu - called before using a cpu
807  * @vcpu: VCPU that will be loaded.
808  *
809  * Disables preemption. Call vmx_put_cpu() when finished.
810  */
811 static void vmx_get_cpu(struct vmx_vcpu *vcpu)
812 {
813         int cur_cpu = core_id();
814         handler_wrapper_t *w;
815
816         if (currentcpu->local_vcpu)
817                 panic("get_cpu: currentcpu->localvcpu was non-NULL");
818         if (currentcpu->local_vcpu != vcpu) {
819                 currentcpu->local_vcpu = vcpu;
820
821                 if (vcpu->cpu != cur_cpu) {
822                         if (vcpu->cpu >= 0) {
823                                 panic("vcpu->cpu is not -1, it's %d\n", vcpu->cpu);
824                         } else
825                                 vmcs_clear(vcpu->vmcs);
826
827                         ept_sync_context(vcpu_get_eptp(vcpu));
828
829                         vcpu->launched = 0;
830                         vmcs_load(vcpu->vmcs);
831                         __vmx_setup_cpu();
832                         vcpu->cpu = cur_cpu;
833                 } else {
834                         vmcs_load(vcpu->vmcs);
835                 }
836         }
837 }
838
839 /**
840  * vmx_put_cpu - called after using a cpu
841  * @vcpu: VCPU that was loaded.
842  */
843 static void vmx_put_cpu(struct vmx_vcpu *vcpu)
844 {
845         if (core_id() != vcpu->cpu)
846                 panic("%s: core_id() %d != vcpu->cpu %d\n",
847                       __func__, core_id(), vcpu->cpu);
848
849         if (currentcpu->local_vcpu != vcpu)
850                 panic("vmx_put_cpu: asked to clear something not ours");
851
852         ept_sync_context(vcpu_get_eptp(vcpu));
853         vmcs_clear(vcpu->vmcs);
854         vcpu->cpu = -1;
855         currentcpu->local_vcpu = NULL;
856         //put_cpu();
857 }
858
859 /**
860  * vmx_dump_cpu - prints the CPU state
861  * @vcpu: VCPU to print
862  */
863 static void vmx_dump_cpu(struct vmx_vcpu *vcpu)
864 {
865
866         unsigned long flags;
867
868         vmx_get_cpu(vcpu);
869         vcpu->regs.tf_rip = vmcs_readl(GUEST_RIP);
870         vcpu->regs.tf_rsp = vmcs_readl(GUEST_RSP);
871         flags = vmcs_readl(GUEST_RFLAGS);
872         vmx_put_cpu(vcpu);
873
874         printk("--- Begin VCPU Dump ---\n");
875         printk("CPU %d VPID %d\n", vcpu->cpu, 0);
876         printk("RIP 0x%016lx RFLAGS 0x%08lx\n",
877                vcpu->regs.tf_rip, flags);
878         printk("RAX 0x%016lx RCX 0x%016lx\n",
879                 vcpu->regs.tf_rax, vcpu->regs.tf_rcx);
880         printk("RDX 0x%016lx RBX 0x%016lx\n",
881                 vcpu->regs.tf_rdx, vcpu->regs.tf_rbx);
882         printk("RSP 0x%016lx RBP 0x%016lx\n",
883                 vcpu->regs.tf_rsp, vcpu->regs.tf_rbp);
884         printk("RSI 0x%016lx RDI 0x%016lx\n",
885                 vcpu->regs.tf_rsi, vcpu->regs.tf_rdi);
886         printk("R8  0x%016lx R9  0x%016lx\n",
887                 vcpu->regs.tf_r8, vcpu->regs.tf_r9);
888         printk("R10 0x%016lx R11 0x%016lx\n",
889                 vcpu->regs.tf_r10, vcpu->regs.tf_r11);
890         printk("R12 0x%016lx R13 0x%016lx\n",
891                 vcpu->regs.tf_r12, vcpu->regs.tf_r13);
892         printk("R14 0x%016lx R15 0x%016lx\n",
893                 vcpu->regs.tf_r14, vcpu->regs.tf_r15);
894         printk("--- End VCPU Dump ---\n");
895
896 }
897
898 uint64_t construct_eptp(physaddr_t root_hpa)
899 {
900         uint64_t eptp;
901
902         /* set WB memory and 4 levels of walk.  we checked these in ept_init */
903         eptp = VMX_EPT_MEM_TYPE_WB |
904                (VMX_EPT_GAW_4_LVL << VMX_EPT_GAW_EPTP_SHIFT);
905         if (cpu_has_vmx_ept_ad_bits())
906                 eptp |= VMX_EPT_AD_ENABLE_BIT;
907         eptp |= (root_hpa & PAGE_MASK);
908
909         return eptp;
910 }
911
912 /**
913  * vmx_setup_initial_guest_state - configures the initial state of guest registers
914  */
915 static void vmx_setup_initial_guest_state(void)
916 {
917         unsigned long tmpl;
918         unsigned long cr4 = X86_CR4_PAE | X86_CR4_VMXE | X86_CR4_OSXMMEXCPT |
919                             X86_CR4_PGE | X86_CR4_OSFXSR;
920         uint32_t protected_mode = X86_CR0_PG | X86_CR0_PE;
921 #if 0
922         do we need it
923         if (boot_cpu_has(X86_FEATURE_PCID))
924                 cr4 |= X86_CR4_PCIDE;
925         if (boot_cpu_has(X86_FEATURE_OSXSAVE))
926                 cr4 |= X86_CR4_OSXSAVE;
927 #endif
928         /* we almost certainly have this */
929         /* we'll go sour if we don't. */
930         if (1) //boot_cpu_has(X86_FEATURE_FSGSBASE))
931                 cr4 |= X86_CR4_RDWRGSFS;
932
933         /* configure control and data registers */
934         vmcs_writel(GUEST_CR0, protected_mode | X86_CR0_WP |
935                                X86_CR0_MP | X86_CR0_ET | X86_CR0_NE);
936         vmcs_writel(CR0_READ_SHADOW, protected_mode | X86_CR0_WP |
937                                      X86_CR0_MP | X86_CR0_ET | X86_CR0_NE);
938         vmcs_writel(GUEST_CR3, rcr3());
939         vmcs_writel(GUEST_CR4, cr4);
940         vmcs_writel(CR4_READ_SHADOW, cr4);
941         vmcs_writel(GUEST_IA32_EFER, EFER_LME | EFER_LMA |
942                     EFER_SCE /*| EFER_FFXSR*/);
943         vmcs_writel(GUEST_GDTR_BASE, 0);
944         vmcs_writel(GUEST_GDTR_LIMIT, 0);
945         vmcs_writel(GUEST_IDTR_BASE, 0);
946         vmcs_writel(GUEST_IDTR_LIMIT, 0);
947         vmcs_writel(GUEST_RIP, 0xdeadbeef);
948         vmcs_writel(GUEST_RSP, 0xdeadbeef);
949         vmcs_writel(GUEST_RFLAGS, 0x02);
950         vmcs_writel(GUEST_DR7, 0);
951
952         /* guest segment bases */
953         vmcs_writel(GUEST_CS_BASE, 0);
954         vmcs_writel(GUEST_DS_BASE, 0);
955         vmcs_writel(GUEST_ES_BASE, 0);
956         vmcs_writel(GUEST_GS_BASE, 0);
957         vmcs_writel(GUEST_SS_BASE, 0);
958         rdmsrl(MSR_FS_BASE, tmpl);
959         vmcs_writel(GUEST_FS_BASE, tmpl);
960
961         /* guest segment access rights */
962         vmcs_writel(GUEST_CS_AR_BYTES, 0xA09B);
963         vmcs_writel(GUEST_DS_AR_BYTES, 0xA093);
964         vmcs_writel(GUEST_ES_AR_BYTES, 0xA093);
965         vmcs_writel(GUEST_FS_AR_BYTES, 0xA093);
966         vmcs_writel(GUEST_GS_AR_BYTES, 0xA093);
967         vmcs_writel(GUEST_SS_AR_BYTES, 0xA093);
968
969         /* guest segment limits */
970         vmcs_write32(GUEST_CS_LIMIT, 0xFFFFFFFF);
971         vmcs_write32(GUEST_DS_LIMIT, 0xFFFFFFFF);
972         vmcs_write32(GUEST_ES_LIMIT, 0xFFFFFFFF);
973         vmcs_write32(GUEST_FS_LIMIT, 0xFFFFFFFF);
974         vmcs_write32(GUEST_GS_LIMIT, 0xFFFFFFFF);
975         vmcs_write32(GUEST_SS_LIMIT, 0xFFFFFFFF);
976
977         /* configure segment selectors */
978         vmcs_write16(GUEST_CS_SELECTOR, 0);
979         vmcs_write16(GUEST_DS_SELECTOR, 0);
980         vmcs_write16(GUEST_ES_SELECTOR, 0);
981         vmcs_write16(GUEST_FS_SELECTOR, 0);
982         vmcs_write16(GUEST_GS_SELECTOR, 0);
983         vmcs_write16(GUEST_SS_SELECTOR, 0);
984         vmcs_write16(GUEST_TR_SELECTOR, 0);
985
986         /* guest LDTR */
987         vmcs_write16(GUEST_LDTR_SELECTOR, 0);
988         vmcs_writel(GUEST_LDTR_AR_BYTES, 0x0082);
989         vmcs_writel(GUEST_LDTR_BASE, 0);
990         vmcs_writel(GUEST_LDTR_LIMIT, 0);
991
992         /* guest TSS */
993         vmcs_writel(GUEST_TR_BASE, 0);
994         vmcs_writel(GUEST_TR_AR_BYTES, 0x0080 | AR_TYPE_BUSY_64_TSS);
995         vmcs_writel(GUEST_TR_LIMIT, 0xff);
996
997         /* initialize sysenter */
998         vmcs_write32(GUEST_SYSENTER_CS, 0);
999         vmcs_writel(GUEST_SYSENTER_ESP, 0);
1000         vmcs_writel(GUEST_SYSENTER_EIP, 0);
1001
1002         /* other random initialization */
1003         vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
1004         vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
1005         vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0);
1006         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
1007         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);  /* 22.2.1 */
1008 }
1009
1010 static void __vmx_disable_intercept_for_msr(unsigned long *msr_bitmap, uint32_t msr)
1011 {
1012         int f = sizeof(unsigned long);
1013         /*
1014          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
1015          * have the write-low and read-high bitmap offsets the wrong way round.
1016          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
1017          */
1018         if (msr <= 0x1fff) {
1019                 __clear_bit(msr, msr_bitmap + 0x000 / f); /* read-low */
1020                 __clear_bit(msr, msr_bitmap + 0x800 / f); /* write-low */
1021         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
1022                 msr &= 0x1fff;
1023                 __clear_bit(msr, msr_bitmap + 0x400 / f); /* read-high */
1024                 __clear_bit(msr, msr_bitmap + 0xc00 / f); /* write-high */
1025         }
1026 }
1027
1028 static void vcpu_print_autoloads(struct vmx_vcpu *vcpu)
1029 {
1030         struct vmx_msr_entry *e;
1031         int sz = sizeof(autoloaded_msrs) / sizeof(*autoloaded_msrs);
1032         printk("Host Autoloads:\n-------------------\n");
1033         for (int i = 0; i < sz; i++) {
1034                 e = &vcpu->msr_autoload.host[i];
1035                 printk("\tMSR 0x%08x: %p\n", e->index, e->value);
1036         }
1037         printk("Guest Autoloads:\n-------------------\n");
1038         for (int i = 0; i < sz; i++) {
1039                 e = &vcpu->msr_autoload.guest[i];
1040                 printk("\tMSR 0x%08x %p\n", e->index, e->value);
1041         }
1042 }
1043
1044 static void dumpmsrs(void)
1045 {
1046         int i;
1047         int set[] = {
1048                 MSR_LSTAR,
1049                 MSR_FS_BASE,
1050                 MSR_GS_BASE,
1051                 MSR_KERNEL_GS_BASE,
1052                 MSR_SFMASK,
1053                 MSR_IA32_PEBS_ENABLE
1054         };
1055         for(i = 0; i < ARRAY_SIZE(set); i++) {
1056                 printk("%p: %p\n", set[i], read_msr(set[i]));
1057         }
1058         printk("core id %d\n", core_id());
1059 }
1060
1061 /* emulated msr. For now, an msr value and a pointer to a helper that
1062  * performs the requested operation.
1063  */
1064 struct emmsr {
1065         uint32_t reg;
1066         int (*f)(struct vmx_vcpu *vcpu, struct emmsr *, uint32_t, uint32_t);
1067         bool written;
1068         uint32_t edx, eax;
1069 };
1070
1071 int emsr_misc_enable(struct vmx_vcpu *vcpu, struct emmsr *, uint32_t, uint32_t);
1072 int emsr_readonly(struct vmx_vcpu *vcpu, struct emmsr *, uint32_t, uint32_t);
1073 int emsr_fakewrite(struct vmx_vcpu *vcpu, struct emmsr *, uint32_t, uint32_t);
1074
1075 struct emmsr emmsrs[] = {
1076         {MSR_IA32_MISC_ENABLE, emsr_misc_enable},
1077         {MSR_IA32_UCODE_REV, emsr_fakewrite},
1078 };
1079
1080 #define set_low32(hi,lo) (((hi) & 0xffffffff00000000ULL ) | (lo))
1081 int emsr_misc_enable(struct vmx_vcpu *vcpu, struct emmsr *msr, uint32_t opcode, uint32_t qual)
1082 {
1083         uint32_t eax, edx;
1084         rdmsr(MSR_IA32_MISC_ENABLE, eax, edx);
1085         /* we just let them read the misc msr for now. */
1086         if (opcode == EXIT_REASON_MSR_READ) {
1087                 vcpu->regs.tf_rax = set_low32(vcpu->regs.tf_rax, eax);
1088                 vcpu->regs.tf_rdx = set_low32(vcpu->regs.tf_rdx, edx);
1089                 return 0;
1090         } else {
1091                 /* if they are writing what is already written, that's ok. */
1092                 if (((uint32_t)vcpu->regs.tf_rax == eax) && ((uint32_t)vcpu->regs.tf_rdx == edx))
1093                         return 0;
1094         }
1095         return SHUTDOWN_UNHANDLED_EXIT_REASON;
1096 }
1097
1098 /* return what's there. Let them think they are writing it if they are not changing anything. */
1099 int emsr_readonly(struct vmx_vcpu *vcpu, struct emmsr *msr, uint32_t opcode, uint32_t qual)
1100 {
1101         uint32_t eax, edx;
1102         rdmsr((uint32_t)vcpu->regs.tf_rcx, eax, edx);
1103         /* we just let them read the misc msr for now. */
1104         if (opcode == EXIT_REASON_MSR_READ) {
1105                 vcpu->regs.tf_rax = set_low32(vcpu->regs.tf_rax, eax);
1106                 vcpu->regs.tf_rdx = set_low32(vcpu->regs.tf_rdx, edx);
1107                 return 0;
1108         } else {
1109                 /* if they are writing what is already written, that's ok. */
1110                 if (((uint32_t)vcpu->regs.tf_rax == eax) && ((uint32_t)vcpu->regs.tf_rdx == edx))
1111                         return 0;
1112                 printk("%s write 0x%lx failed: msr is (0x%lx,0x%lx) and wanted (0x%lx,0x%lx)\n",
1113                         __func__, vcpu->regs.tf_rcx, edx, eax, (uint32_t)vcpu->regs.tf_rdx, (uint32_t)vcpu->regs.tf_rax);
1114         }
1115         return SHUTDOWN_UNHANDLED_EXIT_REASON;
1116 }
1117
1118 /* pretend to write it, but don't write it. */
1119 int emsr_fakewrite(struct vmx_vcpu *vcpu, struct emmsr *msr, uint32_t opcode, uint32_t qual)
1120 {
1121         uint32_t eax, edx;
1122         if (! msr->written) {
1123                 rdmsr(MSR_IA32_MISC_ENABLE, eax, edx);
1124         } else {
1125                 edx = msr->edx;
1126                 eax = msr->eax;
1127         }
1128         /* we just let them read the misc msr for now. */
1129         if (opcode == EXIT_REASON_MSR_READ) {
1130                 vcpu->regs.tf_rax = set_low32(vcpu->regs.tf_rax, eax);
1131                 vcpu->regs.tf_rdx = set_low32(vcpu->regs.tf_rdx, edx);
1132                 return 0;
1133         } else {
1134                 /* if they are writing what is already written, that's ok. */
1135                 if (((uint32_t)vcpu->regs.tf_rax == eax) && ((uint32_t)vcpu->regs.tf_rdx == edx))
1136                         return 0;
1137                 msr->edx = vcpu->regs.tf_rdx;
1138                 msr->eax = vcpu->regs.tf_rax;
1139                 msr->written = true;
1140         }
1141         return 0;
1142 }
1143
1144 int
1145 msrio(struct vmx_vcpu *vcpu, uint32_t opcode, uint32_t qual)
1146 {
1147         int i;
1148         for (i = 0; i < ARRAY_SIZE(emmsrs); i++) {
1149                 if (emmsrs[i].reg != vcpu->regs.tf_rcx)
1150                         continue;
1151                 return emmsrs[i].f(vcpu, &emmsrs[i], opcode, qual);
1152         }
1153         printk("msrio for 0x%lx failed\n", vcpu->regs.tf_rcx);
1154         return SHUTDOWN_UNHANDLED_EXIT_REASON;
1155 }
1156 /* Notes on autoloading.  We can't autoload FS_BASE or GS_BASE, according to the
1157  * manual, but that's because they are automatically saved and restored when all
1158  * of the other architectural registers are saved and restored, such as cs, ds,
1159  * es, and other fun things. (See 24.4.1).  We need to make sure we don't
1160  * accidentally intercept them too, since they are magically autloaded..
1161  *
1162  * We'll need to be careful of any MSR we neither autoload nor intercept
1163  * whenever we vmenter/vmexit, and we intercept by default.
1164  *
1165  * Other MSRs, such as MSR_IA32_PEBS_ENABLE only work on certain architectures
1166  * only work on certain architectures. */
1167 static void setup_msr(struct vmx_vcpu *vcpu)
1168 {
1169         struct vmx_msr_entry *e;
1170         int sz = sizeof(autoloaded_msrs) / sizeof(*autoloaded_msrs);
1171         int i;
1172
1173         static_assert((sizeof(autoloaded_msrs) / sizeof(*autoloaded_msrs)) <=
1174                       NR_AUTOLOAD_MSRS);
1175
1176         vcpu->msr_autoload.nr = sz;
1177
1178         /* Since PADDR(msr_bitmap) is non-zero, and the bitmap is all 0xff, we now
1179          * intercept all MSRs */
1180         vmcs_write64(MSR_BITMAP, PADDR(msr_bitmap));
1181
1182         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, vcpu->msr_autoload.nr);
1183         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vcpu->msr_autoload.nr);
1184         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vcpu->msr_autoload.nr);
1185
1186         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, PADDR(vcpu->msr_autoload.host));
1187         vmcs_write64(VM_EXIT_MSR_STORE_ADDR, PADDR(vcpu->msr_autoload.guest));
1188         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, PADDR(vcpu->msr_autoload.guest));
1189
1190         for (i = 0; i < sz; i++) {
1191                 uint64_t val;
1192
1193                 e = &vcpu->msr_autoload.host[i];
1194                 e->index = autoloaded_msrs[i];
1195                 __vmx_disable_intercept_for_msr(msr_bitmap, e->index);
1196                 rdmsrl(e->index, val);
1197                 e->value = val;
1198                 printk("host index %p val %p\n", e->index, e->value);
1199
1200                 e = &vcpu->msr_autoload.guest[i];
1201                 e->index = autoloaded_msrs[i];
1202                 e->value = 0xDEADBEEF;
1203                 printk("guest index %p val %p\n", e->index, e->value);
1204         }
1205 }
1206
1207 /**
1208  *  vmx_setup_vmcs - configures the vmcs with starting parameters
1209  */
1210 static void vmx_setup_vmcs(struct vmx_vcpu *vcpu)
1211 {
1212         vmcs_write16(VIRTUAL_PROCESSOR_ID, 0);
1213         vmcs_write64(VMCS_LINK_POINTER, -1ull); /* 22.3.1.5 */
1214
1215         /* Control */
1216         vmcs_write32(PIN_BASED_VM_EXEC_CONTROL,
1217                 vmcs_config.pin_based_exec_ctrl);
1218
1219         vmcs_write32(CPU_BASED_VM_EXEC_CONTROL,
1220                 vmcs_config.cpu_based_exec_ctrl);
1221
1222         if (cpu_has_secondary_exec_ctrls()) {
1223                 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
1224                              vmcs_config.cpu_based_2nd_exec_ctrl);
1225         }
1226
1227         vmcs_write64(EPT_POINTER, vcpu_get_eptp(vcpu));
1228
1229         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, 0);
1230         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, 0);
1231         vmcs_write32(CR3_TARGET_COUNT, 0);           /* 22.2.1 */
1232
1233         setup_msr(vcpu);
1234
1235         vmcs_config.vmentry_ctrl |= VM_ENTRY_IA32E_MODE;
1236
1237         vmcs_write32(VM_EXIT_CONTROLS, vmcs_config.vmexit_ctrl);
1238         vmcs_write32(VM_ENTRY_CONTROLS, vmcs_config.vmentry_ctrl);
1239
1240         vmcs_writel(CR0_GUEST_HOST_MASK, 0); // ~0ul);
1241         vmcs_writel(CR4_GUEST_HOST_MASK, 0); // ~0ul);
1242
1243         //kvm_write_tsc(&vmx->vcpu, 0);
1244         vmcs_writel(TSC_OFFSET, 0);
1245
1246         vmx_setup_constant_host_state();
1247 }
1248
1249 /**
1250  * vmx_create_vcpu - allocates and initializes a new virtual cpu
1251  *
1252  * Returns: A new VCPU structure
1253  */
1254 struct vmx_vcpu *vmx_create_vcpu(struct proc *p)
1255 {
1256         struct vmx_vcpu *vcpu = kmalloc(sizeof(struct vmx_vcpu), KMALLOC_WAIT);
1257         if (!vcpu) {
1258                 return NULL;
1259         }
1260
1261         memset(vcpu, 0, sizeof(*vcpu));
1262
1263         vcpu->proc = p; /* uncounted (weak) reference */
1264         vcpu->vmcs = vmx_alloc_vmcs();
1265         printd("%d: vcpu->vmcs is %p\n", core_id(), vcpu->vmcs);
1266         if (!vcpu->vmcs)
1267                 goto fail_vmcs;
1268
1269         vcpu->cpu = -1;
1270
1271         vmx_get_cpu(vcpu);
1272         vmx_setup_vmcs(vcpu);
1273         vmx_setup_initial_guest_state();
1274         vmx_put_cpu(vcpu);
1275
1276         return vcpu;
1277
1278 fail_vmcs:
1279         kfree(vcpu);
1280         return NULL;
1281 }
1282
1283 /**
1284  * vmx_destroy_vcpu - destroys and frees an existing virtual cpu
1285  * @vcpu: the VCPU to destroy
1286  */
1287 void vmx_destroy_vcpu(struct vmx_vcpu *vcpu)
1288 {
1289         vmx_free_vmcs(vcpu->vmcs);
1290         kfree(vcpu);
1291 }
1292
1293 /**
1294  * vmx_current_vcpu - returns a pointer to the vcpu for the current task.
1295  *
1296  * In the contexts where this is used the vcpu pointer should never be NULL.
1297  */
1298 static inline struct vmx_vcpu *vmx_current_vcpu(void)
1299 {
1300         struct vmx_vcpu *vcpu = currentcpu->local_vcpu;
1301         if (!vcpu)
1302                 panic("Core has no vcpu!");
1303         return vcpu;
1304 }
1305
1306 /**
1307  * vmx_run_vcpu - launches the CPU into non-root mode
1308  * We ONLY support 64-bit guests.
1309  * @vcpu: the vmx instance to launch
1310  */
1311 static int vmx_run_vcpu(struct vmx_vcpu *vcpu)
1312 {
1313         asm(
1314                 /* Store host registers */
1315                 "push %%rdx; push %%rbp;"
1316                 "push %%rcx \n\t" /* placeholder for guest rcx */
1317                 "push %%rcx \n\t"
1318                 "cmp %%rsp, %c[host_rsp](%0) \n\t"
1319                 "je 1f \n\t"
1320                 "mov %%rsp, %c[host_rsp](%0) \n\t"
1321                 ASM_VMX_VMWRITE_RSP_RDX "\n\t"
1322                 "1: \n\t"
1323                 /* Reload cr2 if changed */
1324                 "mov %c[cr2](%0), %%rax \n\t"
1325                 "mov %%cr2, %%rdx \n\t"
1326                 "cmp %%rax, %%rdx \n\t"
1327                 "je 2f \n\t"
1328                 "mov %%rax, %%cr2 \n\t"
1329                 "2: \n\t"
1330                 /* Check if vmlaunch of vmresume is needed */
1331                 "cmpl $0, %c[launched](%0) \n\t"
1332                 /* Load guest registers.  Don't clobber flags. */
1333                 "mov %c[rax](%0), %%rax \n\t"
1334                 "mov %c[rbx](%0), %%rbx \n\t"
1335                 "mov %c[rdx](%0), %%rdx \n\t"
1336                 "mov %c[rsi](%0), %%rsi \n\t"
1337                 "mov %c[rdi](%0), %%rdi \n\t"
1338                 "mov %c[rbp](%0), %%rbp \n\t"
1339                 "mov %c[r8](%0),  %%r8  \n\t"
1340                 "mov %c[r9](%0),  %%r9  \n\t"
1341                 "mov %c[r10](%0), %%r10 \n\t"
1342                 "mov %c[r11](%0), %%r11 \n\t"
1343                 "mov %c[r12](%0), %%r12 \n\t"
1344                 "mov %c[r13](%0), %%r13 \n\t"
1345                 "mov %c[r14](%0), %%r14 \n\t"
1346                 "mov %c[r15](%0), %%r15 \n\t"
1347                 "mov %c[rcx](%0), %%rcx \n\t" /* kills %0 (ecx) */
1348
1349                 /* Enter guest mode */
1350                 "jne .Llaunched \n\t"
1351                 ASM_VMX_VMLAUNCH "\n\t"
1352                 "jmp .Lkvm_vmx_return \n\t"
1353                 ".Llaunched: " ASM_VMX_VMRESUME "\n\t"
1354                 ".Lkvm_vmx_return: "
1355                 /* Save guest registers, load host registers, keep flags */
1356                 "mov %0, %c[wordsize](%%rsp) \n\t"
1357                 "pop %0 \n\t"
1358                 "mov %%rax, %c[rax](%0) \n\t"
1359                 "mov %%rbx, %c[rbx](%0) \n\t"
1360                 "popq %c[rcx](%0) \n\t"
1361                 "mov %%rdx, %c[rdx](%0) \n\t"
1362                 "mov %%rsi, %c[rsi](%0) \n\t"
1363                 "mov %%rdi, %c[rdi](%0) \n\t"
1364                 "mov %%rbp, %c[rbp](%0) \n\t"
1365                 "mov %%r8,  %c[r8](%0) \n\t"
1366                 "mov %%r9,  %c[r9](%0) \n\t"
1367                 "mov %%r10, %c[r10](%0) \n\t"
1368                 "mov %%r11, %c[r11](%0) \n\t"
1369                 "mov %%r12, %c[r12](%0) \n\t"
1370                 "mov %%r13, %c[r13](%0) \n\t"
1371                 "mov %%r14, %c[r14](%0) \n\t"
1372                 "mov %%r15, %c[r15](%0) \n\t"
1373                 "mov %%rax, %%r10 \n\t"
1374                 "mov %%rdx, %%r11 \n\t"
1375
1376                 "mov %%cr2, %%rax   \n\t"
1377                 "mov %%rax, %c[cr2](%0) \n\t"
1378
1379                 "pop  %%rbp; pop  %%rdx \n\t"
1380                 "setbe %c[fail](%0) \n\t"
1381                 "mov $" STRINGIFY(GD_UD) ", %%rax \n\t"
1382                 "mov %%rax, %%ds \n\t"
1383                 "mov %%rax, %%es \n\t"
1384               : : "c"(vcpu), "d"((unsigned long)HOST_RSP),
1385                 [launched]"i"(offsetof(struct vmx_vcpu, launched)),
1386                 [fail]"i"(offsetof(struct vmx_vcpu, fail)),
1387                 [host_rsp]"i"(offsetof(struct vmx_vcpu, host_rsp)),
1388                 [rax]"i"(offsetof(struct vmx_vcpu, regs.tf_rax)),
1389                 [rbx]"i"(offsetof(struct vmx_vcpu, regs.tf_rbx)),
1390                 [rcx]"i"(offsetof(struct vmx_vcpu, regs.tf_rcx)),
1391                 [rdx]"i"(offsetof(struct vmx_vcpu, regs.tf_rdx)),
1392                 [rsi]"i"(offsetof(struct vmx_vcpu, regs.tf_rsi)),
1393                 [rdi]"i"(offsetof(struct vmx_vcpu, regs.tf_rdi)),
1394                 [rbp]"i"(offsetof(struct vmx_vcpu, regs.tf_rbp)),
1395                 [r8]"i"(offsetof(struct vmx_vcpu, regs.tf_r8)),
1396                 [r9]"i"(offsetof(struct vmx_vcpu, regs.tf_r9)),
1397                 [r10]"i"(offsetof(struct vmx_vcpu, regs.tf_r10)),
1398                 [r11]"i"(offsetof(struct vmx_vcpu, regs.tf_r11)),
1399                 [r12]"i"(offsetof(struct vmx_vcpu, regs.tf_r12)),
1400                 [r13]"i"(offsetof(struct vmx_vcpu, regs.tf_r13)),
1401                 [r14]"i"(offsetof(struct vmx_vcpu, regs.tf_r14)),
1402                 [r15]"i"(offsetof(struct vmx_vcpu, regs.tf_r15)),
1403                 [cr2]"i"(offsetof(struct vmx_vcpu, cr2)),
1404                 [wordsize]"i"(sizeof(unsigned long))
1405               : "cc", "memory"
1406                 , "rax", "rbx", "rdi", "rsi"
1407                 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
1408         );
1409
1410         vcpu->regs.tf_rip = vmcs_readl(GUEST_RIP);
1411         vcpu->regs.tf_rsp = vmcs_readl(GUEST_RSP);
1412         printd("RETURN. ip %016lx sp %016lx cr2 %016lx\n",
1413                vcpu->regs.tf_rip, vcpu->regs.tf_rsp, vcpu->cr2);
1414         /* FIXME: do we need to set up other flags? */
1415         vcpu->regs.tf_rflags = (vmcs_readl(GUEST_RFLAGS) & 0xFF) |
1416                       X86_EFLAGS_IF | 0x2;
1417
1418         vcpu->regs.tf_cs = GD_UT;
1419         vcpu->regs.tf_ss = GD_UD;
1420
1421         vcpu->launched = 1;
1422
1423         if (vcpu->fail) {
1424                 printk("failure detected (err %x)\n",
1425                        vmcs_read32(VM_INSTRUCTION_ERROR));
1426                 return VMX_EXIT_REASONS_FAILED_VMENTRY;
1427         }
1428
1429         return vmcs_read32(VM_EXIT_REASON);
1430
1431 #if 0
1432         vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
1433         vmx_complete_atomic_exit(vmx);
1434         vmx_recover_nmi_blocking(vmx);
1435         vmx_complete_interrupts(vmx);
1436 #endif
1437 }
1438
1439 static void vmx_step_instruction(void)
1440 {
1441         vmcs_writel(GUEST_RIP, vmcs_readl(GUEST_RIP) +
1442                                vmcs_read32(VM_EXIT_INSTRUCTION_LEN));
1443 }
1444
1445 static int vmx_handle_ept_violation(struct vmx_vcpu *vcpu)
1446 {
1447         unsigned long gva, gpa;
1448         int exit_qual, ret = -1;
1449         page_t *page;
1450
1451         vmx_get_cpu(vcpu);
1452         exit_qual = vmcs_read32(EXIT_QUALIFICATION);
1453         gva = vmcs_readl(GUEST_LINEAR_ADDRESS);
1454         gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS);
1455
1456         vmx_put_cpu(vcpu);
1457
1458         int prot = 0;
1459         prot |= exit_qual & VMX_EPT_FAULT_READ ? PROT_READ : 0;
1460         prot |= exit_qual & VMX_EPT_FAULT_WRITE ? PROT_WRITE : 0;
1461         prot |= exit_qual & VMX_EPT_FAULT_INS ? PROT_EXEC : 0;
1462         ret = handle_page_fault(current, gpa, prot);
1463
1464         if (ret) {
1465                 printk("EPT page fault failure %d, GPA: %p, GVA: %p\n", ret, gpa, gva);
1466                 vmx_dump_cpu(vcpu);
1467         }
1468
1469         return ret;
1470 }
1471
1472 static void vmx_handle_cpuid(struct vmx_vcpu *vcpu)
1473 {
1474         unsigned int eax, ebx, ecx, edx;
1475
1476         eax = vcpu->regs.tf_rax;
1477         ecx = vcpu->regs.tf_rcx;
1478         cpuid(eax, ecx, &eax, &ebx, &ecx, &edx);
1479         vcpu->regs.tf_rax = eax;
1480         vcpu->regs.tf_rbx = ebx;
1481         vcpu->regs.tf_rcx = ecx;
1482         vcpu->regs.tf_rdx = edx;
1483 }
1484
1485 static int vmx_handle_nmi_exception(struct vmx_vcpu *vcpu)
1486 {
1487         uint32_t intr_info;
1488
1489         vmx_get_cpu(vcpu);
1490         intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
1491         vmx_put_cpu(vcpu);
1492
1493         printk("vmx (vcpu %p): got an exception\n", vcpu);
1494         printk("vmx (vcpu %p): pid %d\n", vcpu, vcpu->proc->pid);
1495         if ((intr_info & INTR_INFO_INTR_TYPE_MASK) == INTR_TYPE_NMI_INTR) {
1496                 return 0;
1497         }
1498
1499         printk("unhandled nmi, intr_info %x\n", intr_info);
1500         return -EIO;
1501 }
1502
1503 /**
1504  * vmx_launch - the main loop for a VMX Dune process
1505  * @conf: the launch configuration
1506  */
1507 int vmx_launch(uint64_t rip, uint64_t rsp, uint64_t cr3)
1508 {
1509         int ret;
1510         struct vmx_vcpu *vcpu;
1511         int errors = 0;
1512         int advance;
1513
1514         printd("RUNNING: %s: rip %p rsp %p cr3 %p \n",
1515                __func__, rip, rsp, cr3);
1516         /* TODO: dirty hack til we have VMM contexts */
1517         vcpu = current->vmm.guest_pcores[0];
1518         if (!vcpu) {
1519                 printk("Failed to get a CPU!\n");
1520                 return -ENOMEM;
1521         }
1522
1523         /* We need to prep the host's autoload region for our current core.  Right
1524          * now, the only autoloaded MSR that varies at runtime (in this case per
1525          * core is the KERN_GS_BASE). */
1526         rdmsrl(MSR_KERNEL_GS_BASE, vcpu->msr_autoload.host[0].value);
1527         /* if cr3 is set, means 'set everything', else means 'start where you left off' */
1528         if (cr3) {
1529                 vmx_get_cpu(vcpu);
1530                 vmcs_writel(GUEST_RIP, rip);
1531                 vmcs_writel(GUEST_RSP, rsp);
1532                 vmcs_writel(GUEST_CR3, cr3);
1533                 vmx_put_cpu(vcpu);
1534         }
1535
1536         vcpu->ret_code = -1;
1537
1538         while (1) {
1539                 advance = 0;
1540                 vmx_get_cpu(vcpu);
1541
1542                 // TODO: manage the fpu when we restart.
1543
1544                 // TODO: see if we need to exit before we go much further.
1545                 disable_irq();
1546                 //dumpmsrs();
1547                 ret = vmx_run_vcpu(vcpu);
1548                 //dumpmsrs();
1549                 enable_irq();
1550                 vmx_put_cpu(vcpu);
1551
1552                 if (ret == EXIT_REASON_VMCALL) {
1553                         if (current->vmm.flags & VMM_VMCALL_PRINTF) {
1554                                 uint8_t byte = vcpu->regs.tf_rdi;
1555                                 printd("System call\n");
1556 #ifdef DEBUG
1557                                 vmx_dump_cpu(vcpu);
1558 #endif
1559                                 advance = 3;
1560                                 printk("%c", byte);
1561                                 // adjust the RIP
1562                         } else {
1563                                 vcpu->shutdown = SHUTDOWN_UNHANDLED_EXIT_REASON;
1564                                 uint8_t byte = vcpu->regs.tf_rdi;
1565                                 printk("%p %c\n", byte, vcpu->regs.tf_rdi);
1566                                 vmx_dump_cpu(vcpu);
1567                                 printd("system call! WTF\n");
1568                         }
1569                 } else if (ret == EXIT_REASON_CR_ACCESS) {
1570                         show_cr_access(vmcs_read32(EXIT_QUALIFICATION));
1571                         vmx_dump_cpu(vcpu);
1572                         vcpu->shutdown = SHUTDOWN_UNHANDLED_EXIT_REASON;
1573                 } else if (ret == EXIT_REASON_CPUID) {
1574                         vmx_handle_cpuid(vcpu);
1575                         vmx_get_cpu(vcpu);
1576                         vmcs_writel(GUEST_RIP, vcpu->regs.tf_rip + 2);
1577                         vmx_put_cpu(vcpu);
1578                 } else if (ret == EXIT_REASON_EPT_VIOLATION) {
1579                         if (vmx_handle_ept_violation(vcpu))
1580                                 vcpu->shutdown = SHUTDOWN_EPT_VIOLATION;
1581                 } else if (ret == EXIT_REASON_EXCEPTION_NMI) {
1582                         if (vmx_handle_nmi_exception(vcpu))
1583                                 vcpu->shutdown = SHUTDOWN_NMI_EXCEPTION;
1584                 } else if (ret == EXIT_REASON_EXTERNAL_INTERRUPT) {
1585                         printd("External interrupt\n");
1586                         vcpu->shutdown = SHUTDOWN_UNHANDLED_EXIT_REASON;
1587                 } else if (ret == EXIT_REASON_MSR_READ) {
1588                         printd("msr read\n");
1589                         vmx_dump_cpu(vcpu);
1590                         vcpu->shutdown = msrio(vcpu, ret, vmcs_read32(EXIT_QUALIFICATION));
1591                         advance = 2;
1592                 } else if (ret == EXIT_REASON_MSR_WRITE) {
1593                         printd("msr write\n");
1594                         vmx_dump_cpu(vcpu);
1595                         vcpu->shutdown = msrio(vcpu, ret, vmcs_read32(EXIT_QUALIFICATION));
1596                         advance = 2;
1597                 } else {
1598                         printk("unhandled exit: reason 0x%x, exit qualification 0x%x\n",
1599                                ret, vmcs_read32(EXIT_QUALIFICATION));
1600                         vmx_dump_cpu(vcpu);
1601                         vcpu->shutdown = SHUTDOWN_UNHANDLED_EXIT_REASON;
1602                 }
1603
1604                 /* TODO: we can't just return and relaunch the VMCS, in case we blocked.
1605                  * similar to how proc_restartcore/smp_idle only restart the pcpui
1606                  * cur_ctx, we need to do the same, via the VMCS resume business. */
1607                 if (vcpu->shutdown)
1608                         break;
1609
1610                 if (advance) {
1611                         vmx_get_cpu(vcpu);
1612                         vmcs_writel(GUEST_RIP, vcpu->regs.tf_rip + advance);
1613                         vmx_put_cpu(vcpu);
1614                 }
1615         }
1616
1617         printd("RETURN. ip %016lx sp %016lx\n",
1618                 vcpu->regs.tf_rip, vcpu->regs.tf_rsp);
1619 //      hexdump((void *)vcpu->regs.tf_rsp, 128 * 8);
1620         /*
1621          * Return both the reason for the shutdown and a status value.
1622          * The exit() and exit_group() system calls only need 8 bits for
1623          * the status but we allow 16 bits in case we might want to
1624          * return more information for one of the other shutdown reasons.
1625          */
1626         ret = (vcpu->shutdown << 16) | (vcpu->ret_code & 0xffff);
1627
1628         return ret;
1629 }
1630
1631 /**
1632  * __vmx_enable - low-level enable of VMX mode on the current CPU
1633  * @vmxon_buf: an opaque buffer for use as the VMXON region
1634  */
1635 static  int __vmx_enable(struct vmcs *vmxon_buf)
1636 {
1637         uint64_t phys_addr = PADDR(vmxon_buf);
1638         uint64_t old, test_bits;
1639
1640         if (rcr4() & X86_CR4_VMXE) {
1641                 panic("Should never have this happen");
1642                 return -EBUSY;
1643         }
1644
1645         rdmsrl(MSR_IA32_FEATURE_CONTROL, old);
1646
1647         test_bits = FEATURE_CONTROL_LOCKED;
1648         test_bits |= FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
1649
1650         if (0) // tboot_enabled())
1651                 test_bits |= FEATURE_CONTROL_VMXON_ENABLED_INSIDE_SMX;
1652
1653         if ((old & test_bits) != test_bits) {
1654                 /* If it's locked, then trying to set it will cause a GPF.
1655                  * No Dune for you!
1656                  */
1657                 if (old & FEATURE_CONTROL_LOCKED) {
1658                         printk("Dune: MSR_IA32_FEATURE_CONTROL is locked!\n");
1659                         return -1;
1660                 }
1661
1662                 /* enable and lock */
1663                 write_msr(MSR_IA32_FEATURE_CONTROL, old | test_bits);
1664         }
1665         lcr4(rcr4() | X86_CR4_VMXE);
1666
1667         __vmxon(phys_addr);
1668         vpid_sync_vcpu_global();        /* good idea, even if we aren't using vpids */
1669         ept_sync_global();
1670
1671         return 0;
1672 }
1673
1674 /**
1675  * vmx_enable - enables VMX mode on the current CPU
1676  * @unused: not used (required for on_each_cpu())
1677  *
1678  * Sets up necessary state for enable (e.g. a scratchpad for VMXON.)
1679  */
1680 static void vmx_enable(void)
1681 {
1682         struct vmcs *vmxon_buf = currentcpu->vmxarea;
1683         int ret;
1684
1685         ret = __vmx_enable(vmxon_buf);
1686         if (ret)
1687                 goto failed;
1688
1689         currentcpu->vmx_enabled = 1;
1690         // TODO: do we need this?
1691         store_gdt(&currentcpu->host_gdt);
1692
1693         printk("VMX enabled on CPU %d\n", core_id());
1694         return;
1695
1696 failed:
1697         printk("Failed to enable VMX on core %d, err = %d\n", core_id(), ret);
1698 }
1699
1700 /**
1701  * vmx_disable - disables VMX mode on the current CPU
1702  */
1703 static void vmx_disable(void *unused)
1704 {
1705         if (currentcpu->vmx_enabled) {
1706                 __vmxoff();
1707                 lcr4(rcr4() & ~X86_CR4_VMXE);
1708                 currentcpu->vmx_enabled = 0;
1709         }
1710 }
1711
1712 /* Probe the cpus to see which ones can do vmx.
1713  * Return -errno if it fails, and 1 if it succeeds.
1714  */
1715 static bool probe_cpu_vmx(void)
1716 {
1717         /* The best way to test this code is:
1718          * wrmsr -p <cpu> 0x3a 1
1719          * This will lock vmx off; then modprobe dune.
1720          * Frequently, however, systems have all 0x3a registers set to 5,
1721          * meaning testing is impossible, as vmx can not be disabled.
1722          * We have to simulate it being unavailable in most cases.
1723          * The 'test' variable provides an easy way to simulate
1724          * unavailability of vmx on some, none, or all cpus.
1725          */
1726         if (!cpu_has_vmx()) {
1727                 printk("Machine does not support VT-x\n");
1728                 return FALSE;
1729         } else {
1730                 printk("Machine supports VT-x\n");
1731                 return TRUE;
1732         }
1733 }
1734
1735 static void setup_vmxarea(void)
1736 {
1737                 struct vmcs *vmxon_buf;
1738                 printd("Set up vmxarea for cpu %d\n", core_id());
1739                 vmxon_buf = __vmx_alloc_vmcs(core_id());
1740                 if (!vmxon_buf) {
1741                         printk("setup_vmxarea failed on node %d\n", core_id());
1742                         return;
1743                 }
1744                 currentcpu->vmxarea = vmxon_buf;
1745 }
1746
1747 static int ept_init(void)
1748 {
1749         if (!cpu_has_vmx_ept()) {
1750                 printk("VMX doesn't support EPT!\n");
1751                 return -1;
1752         }
1753         if (!cpu_has_vmx_eptp_writeback()) {
1754                 printk("VMX EPT doesn't support WB memory!\n");
1755                 return -1;
1756         }
1757         if (!cpu_has_vmx_ept_4levels()) {
1758                 printk("VMX EPT doesn't support 4 level walks!\n");
1759                 return -1;
1760         }
1761         switch (arch_max_jumbo_page_shift()) {
1762                 case PML3_SHIFT:
1763                         if (!cpu_has_vmx_ept_1g_page()) {
1764                                 printk("VMX EPT doesn't support 1 GB pages!\n");
1765                                 return -1;
1766                         }
1767                         break;
1768                 case PML2_SHIFT:
1769                         if (!cpu_has_vmx_ept_2m_page()) {
1770                                 printk("VMX EPT doesn't support 2 MB pages!\n");
1771                                 return -1;
1772                         }
1773                         break;
1774                 default:
1775                         printk("Unexpected jumbo page size %d\n",
1776                                arch_max_jumbo_page_shift());
1777                         return -1;
1778         }
1779         if (!cpu_has_vmx_ept_ad_bits()) {
1780                 printk("VMX EPT doesn't support accessed/dirty!\n");
1781                 x86_ept_pte_fix_ups |= EPTE_A | EPTE_D;
1782         }
1783         if (!cpu_has_vmx_invept() || !cpu_has_vmx_invept_global()) {
1784                 printk("VMX EPT can't invalidate PTEs/TLBs!\n");
1785                 return -1;
1786         }
1787
1788         return 0;
1789 }
1790
1791 /**
1792  * vmx_init sets up physical core data areas that are required to run a vm at all.
1793  * These data areas are not connected to a specific user process in any way. Instead,
1794  * they are in some sense externalizing what would other wise be a very large ball of
1795  * state that would be inside the CPU.
1796  */
1797 int intel_vmm_init(void)
1798 {
1799         int r, cpu, ret;
1800
1801         if (! probe_cpu_vmx()) {
1802                 return -EOPNOTSUPP;
1803         }
1804
1805         setup_vmcs_config(&ret);
1806
1807         if (ret) {
1808                 printk("setup_vmcs_config failed: %d\n", ret);
1809                 return ret;
1810         }
1811
1812         msr_bitmap = (unsigned long *)kpage_zalloc_addr();
1813         if (!msr_bitmap) {
1814                 printk("Could not allocate msr_bitmap\n");
1815                 return -ENOMEM;
1816         }
1817         /* FIXME: do we need APIC virtualization (flexpriority?) */
1818
1819         memset(msr_bitmap, 0xff, PAGE_SIZE);
1820         /* These are the only MSRs that are not autoloaded and not intercepted */
1821         __vmx_disable_intercept_for_msr(msr_bitmap, MSR_FS_BASE);
1822         __vmx_disable_intercept_for_msr(msr_bitmap, MSR_GS_BASE);
1823         __vmx_disable_intercept_for_msr(msr_bitmap, MSR_EFER);
1824
1825         if ((ret = ept_init())) {
1826                 printk("EPT init failed, %d\n", ret);
1827                 return ret;
1828         }
1829         printk("VMX setup succeeded\n");
1830         return 0;
1831 }
1832
1833 int intel_vmm_pcpu_init(void)
1834 {
1835         setup_vmxarea();
1836         vmx_enable();
1837         return 0;
1838 }