vmm: Split VM creation into init and adding GPCs
[akaros.git] / kern / arch / x86 / vmm / vmm.c
1 /* Copyright 2015 Google Inc.
2  *
3  * See LICENSE for details.
4  */
5
6 /* We're not going to falll into the trap of only compiling support
7  * for AMD OR Intel for an image. It all gets compiled in, and which
8  * one you use depends on on cpuinfo, not a compile-time
9  * switch. That's proven to be the best strategy.  Conditionally
10  * compiling in support is the path to hell.
11  */
12 #include <assert.h>
13 #include <pmap.h>
14 #include <smp.h>
15 #include <kmalloc.h>
16
17 #include <ros/vmm.h>
18 #include "intel/vmx.h"
19 #include "vmm.h"
20 #include <trap.h>
21 #include <umem.h>
22
23 #include <arch/x86.h>
24 #include <ros/procinfo.h>
25
26
27 /* TODO: have better cpuid info storage and checks */
28 bool x86_supports_vmx = FALSE;
29
30 /* Figure out what kind of CPU we are on, and if it supports any reasonable
31  * virtualization. For now, if we're not some sort of newer intel, don't
32  * bother. This does all cores. Again, note, we make these decisions at runtime,
33  * to avoid getting into the problems that compile-time decisions can cause.
34  * At this point, of course, it's still all intel.
35  */
36 void vmm_init(void)
37 {
38         int ret;
39         /* Check first for intel capabilities. This is hence two back-to-back
40          * implementationd-dependent checks. That's ok, it's all msr dependent.
41          */
42         ret = intel_vmm_init();
43         if (! ret) {
44                 x86_supports_vmx = TRUE;
45                 return;
46         }
47
48         /* TODO: AMD. Will we ever care? It's not clear. */
49         printk("vmm_init failed, ret %d\n", ret);
50         return;
51 }
52
53 void vmm_pcpu_init(void)
54 {
55         struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
56
57         pcpui->guest_pcoreid = -1;
58         if (!x86_supports_vmx)
59                 return;
60         if (! intel_vmm_pcpu_init()) {
61                 printd("vmm_pcpu_init worked\n");
62                 return;
63         }
64         /* TODO: AMD. Will we ever care? It's not clear. */
65         printk("vmm_pcpu_init failed\n");
66 }
67
68 /* Ensures a process is ready to run virtual machines, though it may have no
69  * guest pcores yet.  Typically, this is called by other vmm functions.  Caller
70  * holds the qlock.  Throws on error. */
71 void __vmm_struct_init(struct proc *p)
72 {
73         struct vmm *vmm = &p->vmm;
74
75         if (vmm->vmmcp)
76                 return;
77         if (!x86_supports_vmx)
78                 error(ENODEV, "This CPU does not support VMX");
79         vmm->vmmcp = TRUE;
80         vmm->amd = 0;
81         vmx_setup_vmx_vmm(&vmm->vmx);
82         for (int i = 0; i < VMM_VMEXIT_NR_TYPES; i++)
83                 vmm->vmexits[i] = 0;
84         vmm->nr_guest_pcores = 0;
85         vmm->guest_pcores = NULL;
86         vmm->gpc_array_elem = 0;
87 }
88
89 /* Helper, grows the array of guest_pcores in vmm.  Concurrent readers
90  * (lookup_guest_pcore) need to use a seq-lock-style of concurrency.  They could
91  * read the old array even after we free it. */
92 static void __vmm_grow_gpc_array(struct vmm *vmm, unsigned int new_nr_gpcs)
93 {
94         struct guest_pcore **new_array, **old_array;
95         size_t new_nr_elem;
96
97         if (new_nr_gpcs <= vmm->gpc_array_elem)
98                 return;
99         /* TODO: (RCU) we could defer the free, maybe with an RCU-safe krealloc. */
100         old_array = vmm->guest_pcores;
101         new_nr_elem = MAX(vmm->gpc_array_elem * 2, new_nr_gpcs);
102         new_array = kzmalloc(new_nr_elem * sizeof(void*), MEM_WAIT);
103         memcpy(new_array, vmm->guest_pcores,
104                sizeof(void*) * vmm->nr_guest_pcores);
105         wmb();  /* all elements written before changing pointer */
106         vmm->guest_pcores = new_array;
107         wmb();  /* ptr written before potentially clobbering it. */
108         kfree(old_array);
109 }
110
111 /* Adds gpcs to the VMM.  Caller holds the qlock; throws on error. */
112 void __vmm_add_gpcs(struct proc *p, unsigned int nr_more_gpcs,
113                     struct vmm_gpcore_init *u_gpcis)
114 {
115         struct vmm *vmm = &p->vmm;
116         struct vmm_gpcore_init gpci;
117         unsigned int new_nr_gpcs;
118
119         if (!nr_more_gpcs)
120                 return;
121         new_nr_gpcs = vmm->nr_guest_pcores + nr_more_gpcs;
122         if ((new_nr_gpcs < vmm->nr_guest_pcores) || (new_nr_gpcs > 10000))
123                 error(EINVAL, "Can't add %u new gpcs", new_nr_gpcs);
124         __vmm_grow_gpc_array(vmm, new_nr_gpcs);
125         for (int i = 0; i < nr_more_gpcs; i++) {
126                 if (copy_from_user(&gpci, &u_gpcis[i], sizeof(struct vmm_gpcore_init)))
127                         error(EINVAL, "Bad pointer %p for gps", u_gpcis);
128                 vmm->guest_pcores[vmm->nr_guest_pcores] = create_guest_pcore(p, &gpci);
129                 wmb();  /* concurrent readers will check nr_guest_pcores first */
130                 vmm->nr_guest_pcores++;
131         }
132 }
133
134 /* Has no concurrency protection - only call this when you know you have the
135  * only ref to vmm.  For instance, from __proc_free, where there is only one ref
136  * to the proc (and thus proc.vmm). */
137 void __vmm_struct_cleanup(struct proc *p)
138 {
139         struct vmm *vmm = &p->vmm;
140         if (!vmm->vmmcp)
141                 return;
142         for (int i = 0; i < vmm->nr_guest_pcores; i++) {
143                 if (vmm->guest_pcores[i])
144                         destroy_guest_pcore(vmm->guest_pcores[i]);
145         }
146         kfree(vmm->guest_pcores);
147         ept_flush(p->env_pgdir.eptp);
148         vmm->vmmcp = FALSE;
149 }
150
151 int vmm_poke_guest(struct proc *p, int guest_pcoreid)
152 {
153         struct guest_pcore *gpc;
154         int pcoreid;
155
156         gpc = lookup_guest_pcore(p, guest_pcoreid);
157         if (!gpc) {
158                 set_error(ENOENT, "Bad guest_pcoreid %d", guest_pcoreid);
159                 return -1;
160         }
161         /* We're doing an unlocked peek; it could change immediately.  This is a
162          * best effort service. */
163         pcoreid = ACCESS_ONCE(gpc->cpu);
164         if (pcoreid == -1) {
165                 /* So we know that we'll miss the poke for the posted IRQ.  We could
166                  * return an error.  However, error handling for this case isn't
167                  * particularly helpful (yet).  The absence of the error does not mean
168                  * the IRQ was posted.  We'll still return 0, meaning "the user didn't
169                  * mess up; we tried." */
170                 return 0;
171         }
172         send_ipi(pcoreid, I_POKE_GUEST);
173         return 0;
174 }
175
176 struct guest_pcore *lookup_guest_pcore(struct proc *p, int guest_pcoreid)
177 {
178         struct guest_pcore **array;
179         struct guest_pcore *ret;
180
181         /* nr_guest_pcores is written once at setup and never changed */
182         if (guest_pcoreid >= p->vmm.nr_guest_pcores)
183                 return 0;
184         /* TODO: (RCU) Synchronizing with __vmm_grow_gpc_array() */
185         do {
186                 array = ACCESS_ONCE(p->vmm.guest_pcores);
187                 ret = array[guest_pcoreid];
188                 rmb();  /* read ret before rereading array pointer */
189         } while (array != ACCESS_ONCE(p->vmm.guest_pcores));
190         return ret;
191 }
192
193 struct guest_pcore *load_guest_pcore(struct proc *p, int guest_pcoreid)
194 {
195         struct guest_pcore *gpc;
196         struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
197
198         gpc = lookup_guest_pcore(p, guest_pcoreid);
199         if (!gpc)
200                 return 0;
201         assert(pcpui->guest_pcoreid == -1);
202         spin_lock(&p->vmm.lock);
203         if (gpc->cpu != -1) {
204                 spin_unlock(&p->vmm.lock);
205                 return 0;
206         }
207         gpc->cpu = core_id();
208         spin_unlock(&p->vmm.lock);
209         /* We've got dibs on the gpc; we don't need to hold the lock any longer. */
210         pcpui->guest_pcoreid = guest_pcoreid;
211         vmx_load_guest_pcore(gpc);
212         /* Load guest's xcr0 */
213         lxcr0(gpc->xcr0);
214
215         /* Manual MSR save/restore */
216         write_kern_gsbase(gpc->msr_kern_gs_base);
217         if (gpc->msr_star != AKAROS_MSR_STAR)
218                 write_msr(MSR_STAR, gpc->msr_star);
219         if (gpc->msr_lstar != AKAROS_MSR_LSTAR)
220                 write_msr(MSR_LSTAR, gpc->msr_lstar);
221         if (gpc->msr_sfmask != AKAROS_MSR_SFMASK)
222                 write_msr(MSR_SFMASK, gpc->msr_sfmask);
223
224         return gpc;
225 }
226
227 void unload_guest_pcore(struct proc *p, int guest_pcoreid)
228 {
229         struct guest_pcore *gpc;
230         struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
231
232         gpc = lookup_guest_pcore(p, guest_pcoreid);
233         assert(gpc);
234         spin_lock(&p->vmm.lock);
235         assert(gpc->cpu != -1);
236         vmx_unload_guest_pcore(gpc);
237         gpc->cpu = -1;
238
239         /* Save guest's xcr0 and restore Akaros's default. */
240         gpc->xcr0 = rxcr0();
241         lxcr0(__proc_global_info.x86_default_xcr0);
242
243         /* We manage these MSRs manually. */
244         gpc->msr_kern_gs_base = read_kern_gsbase();
245         gpc->msr_star = read_msr(MSR_STAR);
246         gpc->msr_lstar = read_msr(MSR_LSTAR);
247         gpc->msr_sfmask = read_msr(MSR_SFMASK);
248
249         write_kern_gsbase((uint64_t)pcpui);
250         if (gpc->msr_star != AKAROS_MSR_STAR)
251                 write_msr(MSR_STAR, AKAROS_MSR_STAR);
252         if (gpc->msr_lstar != AKAROS_MSR_LSTAR)
253                 write_msr(MSR_LSTAR, AKAROS_MSR_LSTAR);
254         if (gpc->msr_sfmask, AKAROS_MSR_SFMASK)
255                 write_msr(MSR_SFMASK, AKAROS_MSR_SFMASK);
256
257         /* As soon as we unlock, this gpc can be started on another core */
258         spin_unlock(&p->vmm.lock);
259         pcpui->guest_pcoreid = -1;
260 }
261
262 /* emulated msr. For now, an msr value and a pointer to a helper that
263  * performs the requested operation.
264  */
265 struct emmsr {
266         uint32_t reg;
267         char *name;
268         bool (*f)(struct emmsr *msr, struct vm_trapframe *vm_tf,
269                   uint32_t opcode);
270         bool written;
271         uint32_t edx, eax;
272 };
273
274 static bool emsr_miscenable(struct emmsr *msr, struct vm_trapframe *vm_tf,
275                             uint32_t opcode);
276 static bool emsr_readonly(struct emmsr *msr, struct vm_trapframe *vm_tf,
277                           uint32_t opcode);
278 static bool emsr_readzero(struct emmsr *msr, struct vm_trapframe *vm_tf,
279                           uint32_t opcode);
280 static bool emsr_fakewrite(struct emmsr *msr, struct vm_trapframe *vm_tf,
281                            uint32_t opcode);
282 static bool emsr_ok(struct emmsr *msr, struct vm_trapframe *vm_tf,
283                     uint32_t opcode);
284 static bool emsr_fake_apicbase(struct emmsr *msr, struct vm_trapframe *vm_tf,
285                                uint32_t opcode);
286
287 struct emmsr emmsrs[] = {
288         {MSR_IA32_MISC_ENABLE, "MSR_IA32_MISC_ENABLE", emsr_miscenable},
289         {MSR_IA32_SYSENTER_CS, "MSR_IA32_SYSENTER_CS", emsr_ok},
290         {MSR_IA32_SYSENTER_EIP, "MSR_IA32_SYSENTER_EIP", emsr_ok},
291         {MSR_IA32_SYSENTER_ESP, "MSR_IA32_SYSENTER_ESP", emsr_ok},
292         {MSR_IA32_UCODE_REV, "MSR_IA32_UCODE_REV", emsr_fakewrite},
293         {MSR_CSTAR, "MSR_CSTAR", emsr_fakewrite},
294         {MSR_IA32_VMX_BASIC_MSR, "MSR_IA32_VMX_BASIC_MSR", emsr_fakewrite},
295         {MSR_IA32_VMX_PINBASED_CTLS_MSR, "MSR_IA32_VMX_PINBASED_CTLS_MSR",
296          emsr_fakewrite},
297         {MSR_IA32_VMX_PROCBASED_CTLS_MSR, "MSR_IA32_VMX_PROCBASED_CTLS_MSR",
298          emsr_fakewrite},
299         {MSR_IA32_VMX_PROCBASED_CTLS2, "MSR_IA32_VMX_PROCBASED_CTLS2",
300          emsr_fakewrite},
301         {MSR_IA32_VMX_EXIT_CTLS_MSR, "MSR_IA32_VMX_EXIT_CTLS_MSR",
302          emsr_fakewrite},
303         {MSR_IA32_VMX_ENTRY_CTLS_MSR, "MSR_IA32_VMX_ENTRY_CTLS_MSR",
304          emsr_fakewrite},
305         {MSR_IA32_ENERGY_PERF_BIAS, "MSR_IA32_ENERGY_PERF_BIAS",
306          emsr_fakewrite},
307         {MSR_LBR_SELECT, "MSR_LBR_SELECT", emsr_ok},
308         {MSR_LBR_TOS, "MSR_LBR_TOS", emsr_ok},
309         {MSR_LBR_NHM_FROM, "MSR_LBR_NHM_FROM", emsr_ok},
310         {MSR_LBR_NHM_TO, "MSR_LBR_NHM_TO", emsr_ok},
311         {MSR_LBR_CORE_FROM, "MSR_LBR_CORE_FROM", emsr_ok},
312         {MSR_LBR_CORE_TO, "MSR_LBR_CORE_TO", emsr_ok},
313
314         // grumble.
315         {MSR_OFFCORE_RSP_0, "MSR_OFFCORE_RSP_0", emsr_ok},
316         {MSR_OFFCORE_RSP_1, "MSR_OFFCORE_RSP_1", emsr_ok},
317         // louder.
318         {MSR_PEBS_LD_LAT_THRESHOLD, "MSR_PEBS_LD_LAT_THRESHOLD", emsr_ok},
319         // aaaaaahhhhhhhhhhhhhhhhhhhhh
320         {MSR_ARCH_PERFMON_EVENTSEL0, "MSR_ARCH_PERFMON_EVENTSEL0", emsr_ok},
321         {MSR_ARCH_PERFMON_EVENTSEL1, "MSR_ARCH_PERFMON_EVENTSEL1", emsr_ok},
322         {MSR_IA32_PERF_CAPABILITIES, "MSR_IA32_PERF_CAPABILITIES", emsr_readzero},
323         // unsafe.
324         {MSR_IA32_APICBASE, "MSR_IA32_APICBASE", emsr_fake_apicbase},
325
326         // mostly harmless.
327         {MSR_TSC_AUX, "MSR_TSC_AUX", emsr_fakewrite},
328         {MSR_RAPL_POWER_UNIT, "MSR_RAPL_POWER_UNIT", emsr_readzero},
329         {MSR_IA32_MCG_CAP, "MSR_IA32_MCG_CAP", emsr_readzero},
330         {MSR_IA32_DEBUGCTLMSR, "MSR_IA32_DEBUGCTLMSR", emsr_fakewrite},
331
332         // TBD
333         {MSR_IA32_TSC_DEADLINE, "MSR_IA32_TSC_DEADLINE", emsr_fakewrite},
334 };
335
336 /* this may be the only register that needs special handling.
337  * If there others then we might want to extend the emmsr struct.
338  */
339 bool emsr_miscenable(struct emmsr *msr, struct vm_trapframe *vm_tf,
340                      uint32_t opcode)
341 {
342         uint64_t val;
343         uint32_t eax, edx;
344
345         if (read_msr_safe(msr->reg, &val))
346                 return FALSE;
347         eax = low32(val);
348         eax |= MSR_IA32_MISC_ENABLE_PEBS_UNAVAIL;
349         edx = high32(val);
350         /* we just let them read the misc msr for now. */
351         if (opcode == VMM_MSR_EMU_READ) {
352                 vm_tf->tf_rax = eax;
353                 vm_tf->tf_rdx = edx;
354                 return TRUE;
355         } else {
356                 /* if they are writing what is already written, that's ok. */
357                 if (((uint32_t) vm_tf->tf_rax == eax)
358                     && ((uint32_t) vm_tf->tf_rdx == edx))
359                         return TRUE;
360         }
361         printk("%s: Wanted to write 0x%x%x, but could not; value was 0x%x%x\n",
362                msr->name, (uint32_t) vm_tf->tf_rdx, (uint32_t) vm_tf->tf_rax,
363                edx, eax);
364         return FALSE;
365 }
366
367 bool emsr_readonly(struct emmsr *msr, struct vm_trapframe *vm_tf,
368                    uint32_t opcode)
369 {
370         uint64_t val;
371
372         if (read_msr_safe(msr->reg, &val))
373                 return FALSE;
374         if (opcode == VMM_MSR_EMU_READ) {
375                 vm_tf->tf_rax = low32(val);
376                 vm_tf->tf_rdx = high32(val);
377                 return TRUE;
378         }
379
380         printk("%s: Tried to write a readonly register\n", msr->name);
381         return FALSE;
382 }
383
384 bool emsr_readzero(struct emmsr *msr, struct vm_trapframe *vm_tf,
385                    uint32_t opcode)
386 {
387         if (opcode == VMM_MSR_EMU_READ) {
388                 vm_tf->tf_rax = 0;
389                 vm_tf->tf_rdx = 0;
390                 return TRUE;
391         }
392
393         printk("%s: Tried to write a readonly register\n", msr->name);
394         return FALSE;
395 }
396
397 /* pretend to write it, but don't write it. */
398 bool emsr_fakewrite(struct emmsr *msr, struct vm_trapframe *vm_tf,
399                     uint32_t opcode)
400 {
401         uint32_t eax, edx;
402         uint64_t val;
403
404         if (!msr->written) {
405                 if (read_msr_safe(msr->reg, &val))
406                         return FALSE;
407                 eax = low32(val);
408                 edx = high32(val);
409         } else {
410                 eax = msr->eax;
411                 edx = msr->edx;
412         }
413         /* we just let them read the misc msr for now. */
414         if (opcode == VMM_MSR_EMU_READ) {
415                 vm_tf->tf_rax = eax;
416                 vm_tf->tf_rdx = edx;
417                 return TRUE;
418         } else {
419                 msr->edx = vm_tf->tf_rdx;
420                 msr->eax = vm_tf->tf_rax;
421                 msr->written = TRUE;
422         }
423         return TRUE;
424 }
425
426 bool emsr_ok(struct emmsr *msr, struct vm_trapframe *vm_tf,
427              uint32_t opcode)
428 {
429         uint64_t val;
430
431         if (opcode == VMM_MSR_EMU_READ) {
432                 if (read_msr_safe(msr->reg, &val))
433                         return FALSE;
434                 vm_tf->tf_rax = low32(val);
435                 vm_tf->tf_rdx = high32(val);
436         } else {
437                 val = (vm_tf->tf_rdx << 32) | (vm_tf->tf_rax & 0xffffffff);
438                 if (write_msr_safe(msr->reg, val))
439                         return FALSE;
440         }
441         return TRUE;
442 }
443
444 /* pretend to write it, but don't write it. */
445 bool emsr_fake_apicbase(struct emmsr *msr, struct vm_trapframe *vm_tf,
446                         uint32_t opcode)
447 {
448         uint32_t eax, edx;
449
450         if (!msr->written) {
451                 /* TODO: tightly coupled to the addr in vmrunkernel.  We want this func
452                  * to return the val that vmrunkernel put into the VMCS. */
453                 eax = 0xfee00d00;
454                 if (vm_tf->tf_guest_pcoreid != 0) {
455                         // Remove BSP bit if not core 0
456                         eax = 0xfee00c00;
457                 }
458                 edx = 0;
459         } else {
460                 edx = msr->edx;
461                 eax = msr->eax;
462         }
463         /* we just let them read the misc msr for now. */
464         if (opcode == VMM_MSR_EMU_READ) {
465                 vm_tf->tf_rax = eax;
466                 vm_tf->tf_rdx = edx;
467                 return TRUE;
468         } else {
469                 /* if they are writing what is already written, that's ok. */
470                 if (((uint32_t) vm_tf->tf_rax == eax)
471                     && ((uint32_t) vm_tf->tf_rdx == edx))
472                         return 0;
473                 msr->edx = vm_tf->tf_rdx;
474                 msr->eax = vm_tf->tf_rax;
475                 msr->written = TRUE;
476         }
477         return TRUE;
478 }
479
480 bool vmm_emulate_msr(struct vm_trapframe *vm_tf, int op)
481 {
482         for (int i = 0; i < ARRAY_SIZE(emmsrs); i++) {
483                 if (emmsrs[i].reg != vm_tf->tf_rcx)
484                         continue;
485                 return emmsrs[i].f(&emmsrs[i], vm_tf, op);
486         }
487         return FALSE;
488 }