vmm: Use a task_thread cache
[akaros.git] / user / vmm / sched.c
1 /* Copyright (c) 2016 Google Inc.
2  * Barret Rhoden <brho@cs.berkeley.edu>
3  * See LICENSE for details.
4  *
5  * 2LS for virtual machines */
6
7 #include <vmm/sched.h>
8 #include <vmm/vmm.h>
9 #include <sys/mman.h>
10 #include <stdlib.h>
11 #include <assert.h>
12 #include <parlib/spinlock.h>
13 #include <parlib/event.h>
14 #include <parlib/ucq.h>
15 #include <parlib/arch/trap.h>
16 #include <parlib/ros_debug.h>
17 #include <parlib/vcore_tick.h>
18 #include <parlib/slab.h>
19
20 int vmm_sched_period_usec = 1000;
21
22 /* For now, we only have one VM managed by the 2LS.  If we ever expand that,
23  * we'll need something analogous to current_uthread, so the 2LS knows which VM
24  * it is working on. */
25 static struct virtual_machine *current_vm;
26
27 static struct spin_pdr_lock queue_lock = SPINPDR_INITIALIZER;
28 /* Runnable queues, broken up by thread type. */
29 static struct vmm_thread_tq rnbl_tasks = TAILQ_HEAD_INITIALIZER(rnbl_tasks);
30 static struct vmm_thread_tq rnbl_guests = TAILQ_HEAD_INITIALIZER(rnbl_guests);
31 /* Counts of *unblocked* threads.  Unblocked = Running + Runnable. */
32 static atomic_t nr_unblk_tasks;
33 static atomic_t nr_unblk_guests;
34 /* Global evq for all syscalls.  Could make this per vcore or whatever. */
35 static struct event_queue *sysc_evq;
36 static struct kmem_cache *task_thread_cache;
37
38 static void vmm_sched_init(void);
39 static void vmm_sched_entry(void);
40 static void vmm_thread_runnable(struct uthread *uth);
41 static void vmm_thread_paused(struct uthread *uth);
42 static void vmm_thread_blockon_sysc(struct uthread *uth, void *sysc);
43 static void vmm_thread_has_blocked(struct uthread *uth, int flags);
44 static void vmm_thread_refl_fault(struct uthread *uth,
45                                   struct user_context *ctx);
46 static void vmm_thread_exited(struct uthread *uth);
47 static struct uthread *vmm_thread_create(void *(*func)(void *), void *arg);
48
49 struct schedule_ops vmm_sched_ops = {
50         .sched_init = vmm_sched_init,
51         .sched_entry = vmm_sched_entry,
52         .thread_runnable = vmm_thread_runnable,
53         .thread_paused = vmm_thread_paused,
54         .thread_blockon_sysc = vmm_thread_blockon_sysc,
55         .thread_has_blocked = vmm_thread_has_blocked,
56         .thread_refl_fault = vmm_thread_refl_fault,
57         .thread_exited = vmm_thread_exited,
58         .thread_create = vmm_thread_create,
59 };
60
61 struct schedule_ops *sched_ops = &vmm_sched_ops;
62
63 /* Helpers */
64 static void vmm_handle_syscall(struct event_msg *ev_msg, unsigned int ev_type,
65                                void *data);
66 static void acct_thread_blocked(struct vmm_thread *vth);
67 static void acct_thread_unblocked(struct vmm_thread *vth);
68 static void enqueue_vmm_thread(struct vmm_thread *vth);
69 static int task_thread_ctor(void *obj, void *priv, int flags);
70 static void task_thread_dtor(void *obj, void *priv);
71 static struct vmm_thread *alloc_vmm_thread(struct virtual_machine *vm,
72                                            int type);
73 static void *__alloc_stack(size_t stacksize);
74 static void __free_stack(void *stacktop, size_t stacksize);
75
76
77 static void restart_thread(struct syscall *sysc)
78 {
79         struct uthread *ut_restartee = (struct uthread*)sysc->u_data;
80
81         /* uthread stuff here: */
82         assert(ut_restartee);
83         assert(ut_restartee->sysc == sysc);     /* set in uthread.c */
84         ut_restartee->sysc = 0; /* so we don't 'reblock' on this later */
85         vmm_thread_runnable(ut_restartee);
86 }
87
88 static void vmm_handle_syscall(struct event_msg *ev_msg, unsigned int ev_type,
89                                void *data)
90 {
91         struct syscall *sysc;
92
93         /* I think we can make this assert now.  If not, check pthread.c. (concern
94          * was having old ev_qs firing and running this handler). */
95         assert(ev_msg);
96         sysc = ev_msg->ev_arg3;
97         assert(sysc);
98         restart_thread(sysc);
99 }
100
101 /* Helper: allocates a UCQ-based event queue suitable for syscalls.  Will
102  * attempt to route the notifs/IPIs to vcoreid */
103 static struct event_queue *setup_sysc_evq(int vcoreid)
104 {
105         struct event_queue *evq;
106         uintptr_t mmap_block;
107
108         mmap_block = (uintptr_t)mmap(0, PGSIZE * 2,
109                                      PROT_WRITE | PROT_READ,
110                                      MAP_POPULATE | MAP_ANONYMOUS, -1, 0);
111         evq = get_eventq_raw();
112         assert(mmap_block && evq);
113         evq->ev_flags = EVENT_IPI | EVENT_INDIR | EVENT_SPAM_INDIR | EVENT_WAKEUP;
114         evq->ev_vcore = vcoreid;
115         evq->ev_mbox->type = EV_MBOX_UCQ;
116         ucq_init_raw(&evq->ev_mbox->ucq, mmap_block, mmap_block + PGSIZE);
117         return evq;
118 }
119
120 static void vmm_sched_init(void)
121 {
122         struct task_thread *thread0;
123
124         /* Note that thread0 doesn't belong to a VM.  We can set this during
125          * vmm_init() if we need to. */
126         thread0 = (struct task_thread*)alloc_vmm_thread(0, VMM_THREAD_TASK);
127         assert(thread0);
128         acct_thread_unblocked((struct vmm_thread*)thread0);
129         thread0->stacksize = USTACK_NUM_PAGES * PGSIZE;
130         thread0->stacktop = (void*)USTACKTOP;
131         /* for lack of a better vcore, might as well send to 0 */
132         sysc_evq = setup_sysc_evq(0);
133         uthread_2ls_init((struct uthread*)thread0, vmm_handle_syscall, NULL);
134         task_thread_cache = kmem_cache_create("task threads",
135                                               sizeof(struct vmm_thread),
136                                               __alignof__(struct vmm_thread), 0,
137                                               task_thread_ctor, task_thread_dtor,
138                                               NULL);
139 }
140
141 /* The scheduling policy is encapsulated in the next few functions (from here
142  * down to sched_entry()). */
143
144 static int desired_nr_vcores(void)
145 {
146         /* Sanity checks on our accounting. */
147         assert(atomic_read(&nr_unblk_guests) >= 0);
148         assert(atomic_read(&nr_unblk_tasks) >= 0);
149         /* Lockless peak.  This is always an estimate.  Some of our tasks busy-wait,
150          * so it's not enough to just give us one vcore for all tasks, yet. */
151         return atomic_read(&nr_unblk_guests) + atomic_read(&nr_unblk_tasks);
152 }
153
154 static struct vmm_thread *__pop_first(struct vmm_thread_tq *tq)
155 {
156         struct vmm_thread *vth;
157
158         vth = TAILQ_FIRST(tq);
159         if (vth)
160                 TAILQ_REMOVE(tq, vth, tq_next);
161         return vth;
162 }
163
164 static struct vmm_thread *pick_a_thread_degraded(void)
165 {
166         struct vmm_thread *vth = 0;
167         static int next_class = VMM_THREAD_GUEST;
168
169         /* We don't have a lot of cores (maybe 0), so we'll alternate which type of
170          * thread we look at first.  Basically, we're RR within a class of threads,
171          * and we'll toggle between those two classes. */
172         spin_pdr_lock(&queue_lock);
173         if (next_class == VMM_THREAD_GUEST) {
174                 if (!vth)
175                         vth = __pop_first(&rnbl_guests);
176                 if (!vth)
177                         vth = __pop_first(&rnbl_tasks);
178                 next_class = VMM_THREAD_TASK;
179         } else {
180                 if (!vth)
181                         vth = __pop_first(&rnbl_tasks);
182                 if (!vth)
183                         vth = __pop_first(&rnbl_guests);
184                 next_class = VMM_THREAD_GUEST;
185         };
186         spin_pdr_unlock(&queue_lock);
187         return vth;
188 }
189
190 /* We have plenty of cores - run whatever we want.  We'll prioritize tasks. */
191 static struct vmm_thread *pick_a_thread_plenty(void)
192 {
193         struct vmm_thread *vth = 0;
194
195         spin_pdr_lock(&queue_lock);
196         if (!vth)
197                 vth = __pop_first(&rnbl_tasks);
198         if (!vth)
199                 vth = __pop_first(&rnbl_guests);
200         spin_pdr_unlock(&queue_lock);
201         return vth;
202 }
203
204 static void yield_current_uth(void)
205 {
206         struct vmm_thread *vth;
207
208         if (!current_uthread)
209                 return;
210         vth = (struct vmm_thread*)stop_current_uthread();
211         enqueue_vmm_thread(vth);
212 }
213
214 /* Helper, tries to get the right number of vcores.  Returns TRUE if we think we
215  * have enough, FALSE otherwise.
216  *
217  * TODO: this doesn't handle a lot of issues, like preemption, how to
218  * run/yield our vcores, dynamic changes in the number of runnables, where
219  * to send events, how to avoid interfering with gpcs, etc. */
220 static bool try_to_get_vcores(void)
221 {
222         int nr_vcores_wanted = desired_nr_vcores();
223         bool have_enough = nr_vcores_wanted <= num_vcores();
224
225         if (have_enough) {
226                 vcore_tick_disable();
227                 return TRUE;
228         }
229         vcore_tick_enable(vmm_sched_period_usec);
230         vcore_request_total(nr_vcores_wanted);
231         return FALSE;
232 }
233
234 static void __attribute__((noreturn)) vmm_sched_entry(void)
235 {
236         struct vmm_thread *vth;
237         bool have_enough;
238
239         have_enough = try_to_get_vcores();
240         if (!have_enough && vcore_tick_poll()) {
241                 /* slightly less than ideal: we grab the queue lock twice */
242                 yield_current_uth();
243         }
244         if (current_uthread)
245                 run_current_uthread();
246         if (have_enough)
247                 vth = pick_a_thread_plenty();
248         else
249                 vth = pick_a_thread_degraded();
250         if (!vth)
251                 vcore_yield_or_restart();
252         run_uthread((struct uthread*)vth);
253 }
254
255 static void vmm_thread_runnable(struct uthread *uth)
256 {
257         /* A thread that was blocked is now runnable.  This counts as becoming
258          * unblocked (running + runnable) */
259         acct_thread_unblocked((struct vmm_thread*)uth);
260         enqueue_vmm_thread((struct vmm_thread*)uth);
261 }
262
263 static void vmm_thread_paused(struct uthread *uth)
264 {
265         /* The thread stopped for some reason, usually a preemption.  We'd like to
266          * just run it whenever we get a chance.  Note that it didn't become
267          * 'blocked' - it's still runnable. */
268         enqueue_vmm_thread((struct vmm_thread*)uth);
269 }
270
271 static void vmm_thread_blockon_sysc(struct uthread *uth, void *syscall)
272 {
273         struct syscall *sysc = (struct syscall*)syscall;
274
275         acct_thread_blocked((struct vmm_thread*)uth);
276         sysc->u_data = uth;
277         if (!register_evq(sysc, sysc_evq)) {
278                 /* Lost the race with the call being done.  The kernel won't send the
279                  * event.  Just restart him. */
280                 restart_thread(sysc);
281         }
282         /* GIANT WARNING: do not touch the thread after this point. */
283 }
284
285 static void vmm_thread_has_blocked(struct uthread *uth, int flags)
286 {
287         /* The thread blocked on something like a mutex.  It's not runnable, so we
288          * don't need to put it on a list, but we do need to account for it not
289          * running.  We'll find out (via thread_runnable) when it starts up again.
290          */
291         acct_thread_blocked((struct vmm_thread*)uth);
292 }
293
294 static void refl_error(struct uthread *uth, unsigned int trap_nr,
295                        unsigned int err, unsigned long aux)
296 {
297         printf("Thread has unhandled fault: %d, err: %d, aux: %p\n",
298                trap_nr, err, aux);
299         /* Note that uthread.c already copied out our ctx into the uth
300          * struct */
301         print_user_context(&uth->u_ctx);
302         printf("Turn on printx to spew unhandled, malignant trap info\n");
303         exit(-1);
304 }
305
306 static bool handle_page_fault(struct uthread *uth, unsigned int err,
307                               unsigned long aux)
308 {
309         if (!(err & PF_VMR_BACKED))
310                 return FALSE;
311         syscall_async(&uth->local_sysc, SYS_populate_va, aux, 1);
312         __block_uthread_on_async_sysc(uth);
313         return TRUE;
314 }
315
316 static void vmm_thread_refl_hw_fault(struct uthread *uth,
317                                      unsigned int trap_nr,
318                                      unsigned int err, unsigned long aux)
319 {
320         switch (trap_nr) {
321         case HW_TRAP_PAGE_FAULT:
322                 if (!handle_page_fault(uth, err, aux))
323                         refl_error(uth, trap_nr, err, aux);
324                 break;
325         default:
326                 refl_error(uth, trap_nr, err, aux);
327         }
328 }
329
330 /* Yield callback for __ctlr_entry */
331 static void __swap_to_gth(struct uthread *uth, void *dummy)
332 {
333         struct ctlr_thread *cth = (struct ctlr_thread*)uth;
334
335         /* We just immediately run our buddy.  The ctlr and the guest are accounted
336          * together ("pass the token" back and forth). */
337         current_uthread = NULL;
338         run_uthread((struct uthread*)cth->buddy);
339         assert(0);
340 }
341
342 /* All ctrl threads start here, each time their guest has a fault.  They can
343  * block and unblock along the way.  Once a ctlr does its final uthread_yield,
344  * the next time it will start again from the top. */
345 static void __ctlr_entry(void)
346 {
347         struct ctlr_thread *cth = (struct ctlr_thread*)current_uthread;
348         struct virtual_machine *vm = gth_to_vm(cth->buddy);
349
350         if (!handle_vmexit(cth->buddy)) {
351                 struct vm_trapframe *vm_tf = gth_to_vmtf(cth->buddy);
352
353                 fprintf(stderr, "vmm: handle_vmexit returned false\n");
354                 fprintf(stderr, "Note: this may be a kernel module, not the kernel\n");
355                 fprintf(stderr, "RSP was %p, ", (void *)vm_tf->tf_rsp);
356                 fprintf(stderr, "RIP was %p:\n", (void *)vm_tf->tf_rip);
357                 /* TODO: properly walk the kernel page tables to map the tf_rip
358                  * to a physical address. For now, however, this hack is good
359                  * enough.
360                  */
361                 hexdump(stderr, (void *)(vm_tf->tf_rip & 0x3fffffff), 16);
362                 showstatus(stderr, cth->buddy);
363                 exit(0);
364         }
365         /* We want to atomically yield and start/reenqueue our buddy.  We do so in
366          * vcore context on the other side of the yield. */
367         uthread_yield(FALSE, __swap_to_gth, 0);
368 }
369
370 static void vmm_thread_refl_vm_fault(struct uthread *uth)
371 {
372         struct guest_thread *gth = (struct guest_thread*)uth;
373         struct ctlr_thread *cth = gth->buddy;
374
375         /* The ctlr starts frm the top every time we get a new fault. */
376         cth->uthread.flags |= UTHREAD_SAVED;
377         init_user_ctx(&cth->uthread.u_ctx, (uintptr_t)&__ctlr_entry,
378                       (uintptr_t)(cth->stacktop));
379         /* We just immediately run our buddy.  The ctlr and the guest are accounted
380          * together ("pass the token" back and forth). */
381         current_uthread = NULL;
382         run_uthread((struct uthread*)cth);
383         assert(0);
384 }
385
386 static void vmm_thread_refl_fault(struct uthread *uth,
387                                   struct user_context *ctx)
388 {
389         switch (ctx->type) {
390         case ROS_HW_CTX:
391                 /* Guests should only ever VM exit */
392                 assert(((struct vmm_thread*)uth)->type != VMM_THREAD_GUEST);
393                 vmm_thread_refl_hw_fault(uth, __arch_refl_get_nr(ctx),
394                                          __arch_refl_get_err(ctx),
395                                          __arch_refl_get_aux(ctx));
396                 break;
397         case ROS_VM_CTX:
398                 vmm_thread_refl_vm_fault(uth);
399                 break;
400         default:
401                 assert(0);
402         }
403 }
404
405 static void task_thread_dtor(void *obj, void *priv)
406 {
407         struct task_thread *tth = (struct task_thread*)obj;
408
409         __free_stack(tth->stacktop, tth->stacksize);
410 }
411
412 static void vmm_thread_exited(struct uthread *uth)
413 {
414         struct vmm_thread *vth = (struct vmm_thread*)uth;
415         struct task_thread *tth = (struct task_thread*)uth;
416
417         /* Catch bugs.  Right now, only tasks threads can exit. */
418         assert(vth->type == VMM_THREAD_TASK);
419
420         acct_thread_blocked((struct vmm_thread*)tth);
421         uthread_cleanup(uth);
422         if (uth->flags & UTHREAD_IS_THREAD0)
423                 return;
424         kmem_cache_free(task_thread_cache, tth);
425 }
426
427 static void destroy_guest_thread(struct guest_thread *gth)
428 {
429         struct ctlr_thread *cth = gth->buddy;
430
431         __free_stack(cth->stacktop, cth->stacksize);
432         uthread_cleanup((struct uthread*)cth);
433         free(cth);
434         uthread_cleanup((struct uthread*)gth);
435         free(gth);
436 }
437
438 static struct guest_thread *create_guest_thread(struct virtual_machine *vm,
439                                                 unsigned int gpcoreid)
440 {
441         struct guest_thread *gth;
442         struct ctlr_thread *cth;
443         /* Guests won't use TLS; they always operate in Ring V.  The controller
444          * might - not because of anything we do, but because of glibc calls. */
445         struct uth_thread_attr gth_attr = {.want_tls = FALSE};
446         struct uth_thread_attr cth_attr = {.want_tls = TRUE};
447
448         gth = (struct guest_thread*)alloc_vmm_thread(vm, VMM_THREAD_GUEST);
449         cth = (struct ctlr_thread*)alloc_vmm_thread(vm, VMM_THREAD_CTLR);
450         if (!gth || !cth) {
451                 free(gth);
452                 free(cth);
453                 return 0;
454         }
455         gth->buddy = cth;
456         cth->buddy = gth;
457         gth->gpc_id = gpcoreid;
458         cth->stacksize = VMM_THR_STACKSIZE;
459         cth->stacktop = __alloc_stack(cth->stacksize);
460         if (!cth->stacktop) {
461                 free(gth);
462                 free(cth);
463                 return 0;
464         }
465         gth->uthread.u_ctx.type = ROS_VM_CTX;
466         gth->uthread.u_ctx.tf.vm_tf.tf_guest_pcoreid = gpcoreid;
467         /* No need to init the ctlr.  It gets re-init'd each time it starts. */
468         uthread_init((struct uthread*)gth, &gth_attr);
469         uthread_init((struct uthread*)cth, &cth_attr);
470         /* TODO: give it a correct FP state.  Our current one is probably fine */
471         restore_fp_state(&gth->uthread.as);
472         gth->uthread.flags |= UTHREAD_FPSAVED;
473         gth->halt_mtx = uth_mutex_alloc();
474         gth->halt_cv = uth_cond_var_alloc();
475         return gth;
476 }
477
478 int vmm_init(struct virtual_machine *vm, int flags)
479 {
480         struct guest_thread **gths;
481
482         if (current_vm)
483                 return -1;
484         current_vm = vm;
485         if (syscall(SYS_vmm_setup, vm->nr_gpcs, vm->gpcis, flags) != vm->nr_gpcs)
486                 return -1;
487         gths = malloc(vm->nr_gpcs * sizeof(struct guest_thread *));
488         if (!gths)
489                 return -1;
490         for (int i = 0; i < vm->nr_gpcs; i++) {
491                 gths[i] = create_guest_thread(vm, i);
492                 if (!gths[i]) {
493                         for (int j = 0; j < i; j++)
494                                 destroy_guest_thread(gths[j]);
495                         free(gths);
496                         return -1;
497                 }
498         }
499         vm->gths = gths;
500         uthread_mcp_init();
501         return 0;
502 }
503
504 void start_guest_thread(struct guest_thread *gth)
505 {
506         acct_thread_unblocked((struct vmm_thread*)gth);
507         enqueue_vmm_thread((struct vmm_thread*)gth);
508 }
509
510 static void __task_thread_run(void)
511 {
512         struct task_thread *tth = (struct task_thread*)current_uthread;
513
514         uth_2ls_thread_exit(tth->func(tth->arg));
515 }
516
517 static int task_thread_ctor(void *obj, void *priv, int flags)
518 {
519         struct vmm_thread *vth = (struct vmm_thread*)obj;
520         struct task_thread *tth = (struct task_thread*)obj;
521
522         memset(vth, 0, sizeof(struct vmm_thread));
523         vth->type = VMM_THREAD_TASK;
524         vth->vm = current_vm;
525         tth->stacksize = VMM_THR_STACKSIZE;
526         tth->stacktop = __alloc_stack(tth->stacksize);
527         if (!tth->stacktop)
528                 return -1;
529         return 0;
530 }
531
532 /* Helper, creates and starts a task thread. */
533 static struct task_thread *__vmm_run_task(struct virtual_machine *vm,
534                                           void *(*func)(void *), void *arg,
535                                           struct uth_thread_attr *tth_attr)
536 {
537         struct task_thread *tth;
538
539         tth = kmem_cache_alloc(task_thread_cache, 0);
540         tth->func = func;
541         tth->arg = arg;
542         init_user_ctx(&tth->uthread.u_ctx, (uintptr_t)&__task_thread_run,
543                       (uintptr_t)(tth->stacktop));
544         uthread_init((struct uthread*)tth, tth_attr);
545         acct_thread_unblocked((struct vmm_thread*)tth);
546         enqueue_vmm_thread((struct vmm_thread*)tth);
547         return tth;
548 }
549
550 struct task_thread *vmm_run_task(struct virtual_machine *vm,
551                                  void *(*func)(void *), void *arg)
552 {
553         struct uth_thread_attr tth_attr = {.want_tls = TRUE, .detached = TRUE};
554
555         return __vmm_run_task(vm, func, arg, &tth_attr);
556 }
557
558 static struct uthread *vmm_thread_create(void *(*func)(void *), void *arg)
559 {
560         struct uth_thread_attr tth_attr = {.want_tls = TRUE, .detached = FALSE};
561         struct task_thread *tth;
562
563         /* It's OK to not have a VM for a generic thread */
564         tth = __vmm_run_task(NULL, func, arg, &tth_attr);
565         /* But just in case, let's poison it */
566         ((struct vmm_thread*)tth)->vm = (void*)0xdeadbeef;
567         return (struct uthread*)tth;
568 }
569
570 /* Helpers for tracking nr_unblk_* threads. */
571 static void acct_thread_blocked(struct vmm_thread *vth)
572 {
573         switch (vth->type) {
574         case VMM_THREAD_GUEST:
575         case VMM_THREAD_CTLR:
576                 atomic_dec(&nr_unblk_guests);
577                 break;
578         case VMM_THREAD_TASK:
579                 atomic_dec(&nr_unblk_tasks);
580                 break;
581         }
582 }
583
584 static void acct_thread_unblocked(struct vmm_thread *vth)
585 {
586         switch (vth->type) {
587         case VMM_THREAD_GUEST:
588         case VMM_THREAD_CTLR:
589                 atomic_inc(&nr_unblk_guests);
590                 break;
591         case VMM_THREAD_TASK:
592                 atomic_inc(&nr_unblk_tasks);
593                 break;
594         }
595 }
596
597 static void enqueue_vmm_thread(struct vmm_thread *vth)
598 {
599         spin_pdr_lock(&queue_lock);
600         switch (vth->type) {
601         case VMM_THREAD_GUEST:
602         case VMM_THREAD_CTLR:
603                 TAILQ_INSERT_TAIL(&rnbl_guests, vth, tq_next);
604                 break;
605         case VMM_THREAD_TASK:
606                 TAILQ_INSERT_TAIL(&rnbl_tasks, vth, tq_next);
607                 break;
608         default:
609                 panic("Bad vmm_thread type %p\n", vth->type);
610         }
611         spin_pdr_unlock(&queue_lock);
612         try_to_get_vcores();
613 }
614
615 static struct vmm_thread *alloc_vmm_thread(struct virtual_machine *vm, int type)
616 {
617         struct vmm_thread *vth;
618         int ret;
619
620         ret = posix_memalign((void**)&vth, __alignof__(struct vmm_thread),
621                              sizeof(struct vmm_thread));
622         if (ret)
623                 return 0;
624         memset(vth, 0, sizeof(struct vmm_thread));
625         vth->type = type;
626         vth->vm = vm;
627         return vth;
628 }
629
630 static void __free_stack(void *stacktop, size_t stacksize)
631 {
632         munmap(stacktop - stacksize, stacksize);
633 }
634
635 static void *__alloc_stack(size_t stacksize)
636 {
637         int force_a_page_fault;
638         void *stacktop;
639         void *stackbot = mmap(0, stacksize, PROT_READ | PROT_WRITE | PROT_EXEC,
640                               MAP_ANONYMOUS, -1, 0);
641
642         if (stackbot == MAP_FAILED)
643                 return 0;
644         stacktop = stackbot + stacksize;
645         /* Want the top of the stack populated, but not the rest of the stack;
646          * that'll grow on demand (up to stacksize, then will clobber memory). */
647         force_a_page_fault = ACCESS_ONCE(*(int*)(stacktop - sizeof(int)));
648         return stacktop;
649 }