net: tcp: Support SACK
[akaros.git] / user / vmm / sched.c
1 /* Copyright (c) 2016 Google Inc.
2  * Barret Rhoden <brho@cs.berkeley.edu>
3  * See LICENSE for details.
4  *
5  * 2LS for virtual machines */
6
7 #include <vmm/sched.h>
8 #include <vmm/vmm.h>
9 #include <sys/mman.h>
10 #include <stdlib.h>
11 #include <assert.h>
12 #include <parlib/spinlock.h>
13 #include <parlib/event.h>
14 #include <parlib/ucq.h>
15 #include <parlib/arch/trap.h>
16 #include <parlib/ros_debug.h>
17 #include <parlib/vcore_tick.h>
18
19 int vmm_sched_period_usec = 1000;
20
21 /* For now, we only have one VM managed by the 2LS.  If we ever expand that,
22  * we'll need something analogous to current_uthread, so the 2LS knows which VM
23  * it is working on. */
24 static struct virtual_machine *current_vm;
25
26 static struct spin_pdr_lock queue_lock = SPINPDR_INITIALIZER;
27 /* Runnable queues, broken up by thread type. */
28 static struct vmm_thread_tq rnbl_tasks = TAILQ_HEAD_INITIALIZER(rnbl_tasks);
29 static struct vmm_thread_tq rnbl_guests = TAILQ_HEAD_INITIALIZER(rnbl_guests);
30 /* Counts of *unblocked* threads.  Unblocked = Running + Runnable. */
31 static atomic_t nr_unblk_tasks;
32 static atomic_t nr_unblk_guests;
33 /* Global evq for all syscalls.  Could make this per vcore or whatever. */
34 static struct event_queue *sysc_evq;
35
36 static void vmm_sched_init(void);
37 static void vmm_sched_entry(void);
38 static void vmm_thread_runnable(struct uthread *uth);
39 static void vmm_thread_paused(struct uthread *uth);
40 static void vmm_thread_blockon_sysc(struct uthread *uth, void *sysc);
41 static void vmm_thread_has_blocked(struct uthread *uth, int flags);
42 static void vmm_thread_refl_fault(struct uthread *uth,
43                                   struct user_context *ctx);
44 static void vmm_thread_exited(struct uthread *uth);
45 static struct uthread *vmm_thread_create(void *(*func)(void *), void *arg);
46
47 struct schedule_ops vmm_sched_ops = {
48         .sched_init = vmm_sched_init,
49         .sched_entry = vmm_sched_entry,
50         .thread_runnable = vmm_thread_runnable,
51         .thread_paused = vmm_thread_paused,
52         .thread_blockon_sysc = vmm_thread_blockon_sysc,
53         .thread_has_blocked = vmm_thread_has_blocked,
54         .thread_refl_fault = vmm_thread_refl_fault,
55         .thread_exited = vmm_thread_exited,
56         .thread_create = vmm_thread_create,
57 };
58
59 struct schedule_ops *sched_ops = &vmm_sched_ops;
60
61 /* Helpers */
62 static void vmm_handle_syscall(struct event_msg *ev_msg, unsigned int ev_type,
63                                void *data);
64 static void acct_thread_blocked(struct vmm_thread *vth);
65 static void acct_thread_unblocked(struct vmm_thread *vth);
66 static void enqueue_vmm_thread(struct vmm_thread *vth);
67 static struct vmm_thread *alloc_vmm_thread(struct virtual_machine *vm,
68                                            int type);
69 static void *__alloc_stack(size_t stacksize);
70 static void __free_stack(void *stacktop, size_t stacksize);
71
72
73 static void restart_thread(struct syscall *sysc)
74 {
75         struct uthread *ut_restartee = (struct uthread*)sysc->u_data;
76
77         /* uthread stuff here: */
78         assert(ut_restartee);
79         assert(ut_restartee->sysc == sysc);     /* set in uthread.c */
80         ut_restartee->sysc = 0; /* so we don't 'reblock' on this later */
81         vmm_thread_runnable(ut_restartee);
82 }
83
84 static void vmm_handle_syscall(struct event_msg *ev_msg, unsigned int ev_type,
85                                void *data)
86 {
87         struct syscall *sysc;
88
89         /* I think we can make this assert now.  If not, check pthread.c. (concern
90          * was having old ev_qs firing and running this handler). */
91         assert(ev_msg);
92         sysc = ev_msg->ev_arg3;
93         assert(sysc);
94         restart_thread(sysc);
95 }
96
97 /* Helper: allocates a UCQ-based event queue suitable for syscalls.  Will
98  * attempt to route the notifs/IPIs to vcoreid */
99 static struct event_queue *setup_sysc_evq(int vcoreid)
100 {
101         struct event_queue *evq;
102         uintptr_t mmap_block;
103
104         mmap_block = (uintptr_t)mmap(0, PGSIZE * 2,
105                                      PROT_WRITE | PROT_READ,
106                                      MAP_POPULATE | MAP_ANONYMOUS, -1, 0);
107         evq = get_eventq_raw();
108         assert(mmap_block && evq);
109         evq->ev_flags = EVENT_IPI | EVENT_INDIR | EVENT_SPAM_INDIR | EVENT_WAKEUP;
110         evq->ev_vcore = vcoreid;
111         evq->ev_mbox->type = EV_MBOX_UCQ;
112         ucq_init_raw(&evq->ev_mbox->ucq, mmap_block, mmap_block + PGSIZE);
113         return evq;
114 }
115
116 static void vmm_sched_init(void)
117 {
118         struct task_thread *thread0;
119
120         /* Note that thread0 doesn't belong to a VM.  We can set this during
121          * vmm_init() if we need to. */
122         thread0 = (struct task_thread*)alloc_vmm_thread(0, VMM_THREAD_TASK);
123         assert(thread0);
124         acct_thread_unblocked((struct vmm_thread*)thread0);
125         thread0->stacksize = USTACK_NUM_PAGES * PGSIZE;
126         thread0->stacktop = (void*)USTACKTOP;
127         /* for lack of a better vcore, might as well send to 0 */
128         sysc_evq = setup_sysc_evq(0);
129         uthread_2ls_init((struct uthread*)thread0, vmm_handle_syscall, NULL);
130 }
131
132 /* The scheduling policy is encapsulated in the next few functions (from here
133  * down to sched_entry()). */
134
135 static int desired_nr_vcores(void)
136 {
137         /* Sanity checks on our accounting. */
138         assert(atomic_read(&nr_unblk_guests) >= 0);
139         assert(atomic_read(&nr_unblk_tasks) >= 0);
140         /* Lockless peak.  This is always an estimate.  Some of our tasks busy-wait,
141          * so it's not enough to just give us one vcore for all tasks, yet. */
142         return atomic_read(&nr_unblk_guests) + atomic_read(&nr_unblk_tasks);
143 }
144
145 static struct vmm_thread *__pop_first(struct vmm_thread_tq *tq)
146 {
147         struct vmm_thread *vth;
148
149         vth = TAILQ_FIRST(tq);
150         if (vth)
151                 TAILQ_REMOVE(tq, vth, tq_next);
152         return vth;
153 }
154
155 static struct vmm_thread *pick_a_thread_degraded(void)
156 {
157         struct vmm_thread *vth = 0;
158         static int next_class = VMM_THREAD_GUEST;
159
160         /* We don't have a lot of cores (maybe 0), so we'll alternate which type of
161          * thread we look at first.  Basically, we're RR within a class of threads,
162          * and we'll toggle between those two classes. */
163         spin_pdr_lock(&queue_lock);
164         if (next_class == VMM_THREAD_GUEST) {
165                 if (!vth)
166                         vth = __pop_first(&rnbl_guests);
167                 if (!vth)
168                         vth = __pop_first(&rnbl_tasks);
169                 next_class = VMM_THREAD_TASK;
170         } else {
171                 if (!vth)
172                         vth = __pop_first(&rnbl_tasks);
173                 if (!vth)
174                         vth = __pop_first(&rnbl_guests);
175                 next_class = VMM_THREAD_GUEST;
176         };
177         spin_pdr_unlock(&queue_lock);
178         return vth;
179 }
180
181 /* We have plenty of cores - run whatever we want.  We'll prioritize tasks. */
182 static struct vmm_thread *pick_a_thread_plenty(void)
183 {
184         struct vmm_thread *vth = 0;
185
186         spin_pdr_lock(&queue_lock);
187         if (!vth)
188                 vth = __pop_first(&rnbl_tasks);
189         if (!vth)
190                 vth = __pop_first(&rnbl_guests);
191         spin_pdr_unlock(&queue_lock);
192         return vth;
193 }
194
195 static void yield_current_uth(void)
196 {
197         struct vmm_thread *vth;
198
199         if (!current_uthread)
200                 return;
201         vth = (struct vmm_thread*)stop_current_uthread();
202         enqueue_vmm_thread(vth);
203 }
204
205 /* Helper, tries to get the right number of vcores.  Returns TRUE if we think we
206  * have enough, FALSE otherwise.
207  *
208  * TODO: this doesn't handle a lot of issues, like preemption, how to
209  * run/yield our vcores, dynamic changes in the number of runnables, where
210  * to send events, how to avoid interfering with gpcs, etc. */
211 static bool try_to_get_vcores(void)
212 {
213         int nr_vcores_wanted = desired_nr_vcores();
214         bool have_enough = nr_vcores_wanted <= num_vcores();
215
216         if (have_enough) {
217                 vcore_tick_disable();
218                 return TRUE;
219         }
220         vcore_tick_enable(vmm_sched_period_usec);
221         vcore_request_total(nr_vcores_wanted);
222         return FALSE;
223 }
224
225 static void __attribute__((noreturn)) vmm_sched_entry(void)
226 {
227         struct vmm_thread *vth;
228         bool have_enough;
229
230         have_enough = try_to_get_vcores();
231         if (!have_enough && vcore_tick_poll()) {
232                 /* slightly less than ideal: we grab the queue lock twice */
233                 yield_current_uth();
234         }
235         if (current_uthread)
236                 run_current_uthread();
237         if (have_enough)
238                 vth = pick_a_thread_plenty();
239         else
240                 vth = pick_a_thread_degraded();
241         if (!vth)
242                 vcore_yield_or_restart();
243         run_uthread((struct uthread*)vth);
244 }
245
246 static void vmm_thread_runnable(struct uthread *uth)
247 {
248         /* A thread that was blocked is now runnable.  This counts as becoming
249          * unblocked (running + runnable) */
250         acct_thread_unblocked((struct vmm_thread*)uth);
251         enqueue_vmm_thread((struct vmm_thread*)uth);
252 }
253
254 static void vmm_thread_paused(struct uthread *uth)
255 {
256         /* The thread stopped for some reason, usually a preemption.  We'd like to
257          * just run it whenever we get a chance.  Note that it didn't become
258          * 'blocked' - it's still runnable. */
259         enqueue_vmm_thread((struct vmm_thread*)uth);
260 }
261
262 static void vmm_thread_blockon_sysc(struct uthread *uth, void *syscall)
263 {
264         struct syscall *sysc = (struct syscall*)syscall;
265
266         acct_thread_blocked((struct vmm_thread*)uth);
267         sysc->u_data = uth;
268         if (!register_evq(sysc, sysc_evq)) {
269                 /* Lost the race with the call being done.  The kernel won't send the
270                  * event.  Just restart him. */
271                 restart_thread(sysc);
272         }
273         /* GIANT WARNING: do not touch the thread after this point. */
274 }
275
276 static void vmm_thread_has_blocked(struct uthread *uth, int flags)
277 {
278         /* The thread blocked on something like a mutex.  It's not runnable, so we
279          * don't need to put it on a list, but we do need to account for it not
280          * running.  We'll find out (via thread_runnable) when it starts up again.
281          */
282         acct_thread_blocked((struct vmm_thread*)uth);
283 }
284
285 static void refl_error(struct uthread *uth, unsigned int trap_nr,
286                        unsigned int err, unsigned long aux)
287 {
288         printf("Thread has unhandled fault: %d, err: %d, aux: %p\n",
289                trap_nr, err, aux);
290         /* Note that uthread.c already copied out our ctx into the uth
291          * struct */
292         print_user_context(&uth->u_ctx);
293         printf("Turn on printx to spew unhandled, malignant trap info\n");
294         exit(-1);
295 }
296
297 static bool handle_page_fault(struct uthread *uth, unsigned int err,
298                               unsigned long aux)
299 {
300         if (!(err & PF_VMR_BACKED))
301                 return FALSE;
302         syscall_async(&uth->local_sysc, SYS_populate_va, aux, 1);
303         __block_uthread_on_async_sysc(uth);
304         return TRUE;
305 }
306
307 static void vmm_thread_refl_hw_fault(struct uthread *uth,
308                                      unsigned int trap_nr,
309                                      unsigned int err, unsigned long aux)
310 {
311         switch (trap_nr) {
312         case HW_TRAP_PAGE_FAULT:
313                 if (!handle_page_fault(uth, err, aux))
314                         refl_error(uth, trap_nr, err, aux);
315                 break;
316         default:
317                 refl_error(uth, trap_nr, err, aux);
318         }
319 }
320
321 /* Yield callback for __ctlr_entry */
322 static void __swap_to_gth(struct uthread *uth, void *dummy)
323 {
324         struct ctlr_thread *cth = (struct ctlr_thread*)uth;
325
326         /* We just immediately run our buddy.  The ctlr and the guest are accounted
327          * together ("pass the token" back and forth). */
328         current_uthread = NULL;
329         run_uthread((struct uthread*)cth->buddy);
330         assert(0);
331 }
332
333 /* All ctrl threads start here, each time their guest has a fault.  They can
334  * block and unblock along the way.  Once a ctlr does its final uthread_yield,
335  * the next time it will start again from the top. */
336 static void __ctlr_entry(void)
337 {
338         struct ctlr_thread *cth = (struct ctlr_thread*)current_uthread;
339         struct virtual_machine *vm = gth_to_vm(cth->buddy);
340
341         if (!handle_vmexit(cth->buddy)) {
342                 struct vm_trapframe *vm_tf = gth_to_vmtf(cth->buddy);
343
344                 fprintf(stderr, "vmm: handle_vmexit returned false\n");
345                 fprintf(stderr, "Note: this may be a kernel module, not the kernel\n");
346                 fprintf(stderr, "RSP was %p, ", (void *)vm_tf->tf_rsp);
347                 fprintf(stderr, "RIP was %p:\n", (void *)vm_tf->tf_rip);
348                 /* TODO: properly walk the kernel page tables to map the tf_rip
349                  * to a physical address. For now, however, this hack is good
350                  * enough.
351                  */
352                 hexdump(stderr, (void *)(vm_tf->tf_rip & 0x3fffffff), 16);
353                 showstatus(stderr, cth->buddy);
354                 exit(0);
355         }
356         /* We want to atomically yield and start/reenqueue our buddy.  We do so in
357          * vcore context on the other side of the yield. */
358         uthread_yield(FALSE, __swap_to_gth, 0);
359 }
360
361 static void vmm_thread_refl_vm_fault(struct uthread *uth)
362 {
363         struct guest_thread *gth = (struct guest_thread*)uth;
364         struct ctlr_thread *cth = gth->buddy;
365
366         /* The ctlr starts frm the top every time we get a new fault. */
367         cth->uthread.flags |= UTHREAD_SAVED;
368         init_user_ctx(&cth->uthread.u_ctx, (uintptr_t)&__ctlr_entry,
369                       (uintptr_t)(cth->stacktop));
370         /* We just immediately run our buddy.  The ctlr and the guest are accounted
371          * together ("pass the token" back and forth). */
372         current_uthread = NULL;
373         run_uthread((struct uthread*)cth);
374         assert(0);
375 }
376
377 static void vmm_thread_refl_fault(struct uthread *uth,
378                                   struct user_context *ctx)
379 {
380         switch (ctx->type) {
381         case ROS_HW_CTX:
382                 /* Guests should only ever VM exit */
383                 assert(((struct vmm_thread*)uth)->type != VMM_THREAD_GUEST);
384                 vmm_thread_refl_hw_fault(uth, __arch_refl_get_nr(ctx),
385                                          __arch_refl_get_err(ctx),
386                                          __arch_refl_get_aux(ctx));
387                 break;
388         case ROS_VM_CTX:
389                 vmm_thread_refl_vm_fault(uth);
390                 break;
391         default:
392                 assert(0);
393         }
394 }
395
396 static void vmm_thread_exited(struct uthread *uth)
397 {
398         struct vmm_thread *vth = (struct vmm_thread*)uth;
399         struct task_thread *tth = (struct task_thread*)uth;
400
401         /* Catch bugs.  Right now, only tasks threads can exit. */
402         assert(vth->type == VMM_THREAD_TASK);
403
404         acct_thread_blocked((struct vmm_thread*)tth);
405         uthread_cleanup(uth);
406         __free_stack(tth->stacktop, tth->stacksize);
407         free(tth);
408 }
409
410 static void destroy_guest_thread(struct guest_thread *gth)
411 {
412         struct ctlr_thread *cth = gth->buddy;
413
414         __free_stack(cth->stacktop, cth->stacksize);
415         uthread_cleanup((struct uthread*)cth);
416         free(cth);
417         uthread_cleanup((struct uthread*)gth);
418         free(gth);
419 }
420
421 static struct guest_thread *create_guest_thread(struct virtual_machine *vm,
422                                                 unsigned int gpcoreid)
423 {
424         struct guest_thread *gth;
425         struct ctlr_thread *cth;
426         /* Guests won't use TLS; they always operate in Ring V.  The controller
427          * might - not because of anything we do, but because of glibc calls. */
428         struct uth_thread_attr gth_attr = {.want_tls = FALSE};
429         struct uth_thread_attr cth_attr = {.want_tls = TRUE};
430
431         gth = (struct guest_thread*)alloc_vmm_thread(vm, VMM_THREAD_GUEST);
432         cth = (struct ctlr_thread*)alloc_vmm_thread(vm, VMM_THREAD_CTLR);
433         if (!gth || !cth) {
434                 free(gth);
435                 free(cth);
436                 return 0;
437         }
438         gth->buddy = cth;
439         cth->buddy = gth;
440         gth->gpc_id = gpcoreid;
441         cth->stacksize = VMM_THR_STACKSIZE;
442         cth->stacktop = __alloc_stack(cth->stacksize);
443         if (!cth->stacktop) {
444                 free(gth);
445                 free(cth);
446                 return 0;
447         }
448         gth->uthread.u_ctx.type = ROS_VM_CTX;
449         gth->uthread.u_ctx.tf.vm_tf.tf_guest_pcoreid = gpcoreid;
450         /* No need to init the ctlr.  It gets re-init'd each time it starts. */
451         uthread_init((struct uthread*)gth, &gth_attr);
452         uthread_init((struct uthread*)cth, &cth_attr);
453         /* TODO: give it a correct FP state.  Our current one is probably fine */
454         restore_fp_state(&gth->uthread.as);
455         gth->uthread.flags |= UTHREAD_FPSAVED;
456         gth->halt_mtx = uth_mutex_alloc();
457         gth->halt_cv = uth_cond_var_alloc();
458         return gth;
459 }
460
461 int vmm_init(struct virtual_machine *vm, int flags)
462 {
463         struct guest_thread **gths;
464
465         if (current_vm)
466                 return -1;
467         current_vm = vm;
468         if (syscall(SYS_vmm_setup, vm->nr_gpcs, vm->gpcis, flags) != vm->nr_gpcs)
469                 return -1;
470         gths = malloc(vm->nr_gpcs * sizeof(struct guest_thread *));
471         if (!gths)
472                 return -1;
473         for (int i = 0; i < vm->nr_gpcs; i++) {
474                 gths[i] = create_guest_thread(vm, i);
475                 if (!gths[i]) {
476                         for (int j = 0; j < i; j++)
477                                 destroy_guest_thread(gths[j]);
478                         free(gths);
479                         return -1;
480                 }
481         }
482         vm->gths = gths;
483         uthread_mcp_init();
484         return 0;
485 }
486
487 void start_guest_thread(struct guest_thread *gth)
488 {
489         acct_thread_unblocked((struct vmm_thread*)gth);
490         enqueue_vmm_thread((struct vmm_thread*)gth);
491 }
492
493 static void __task_thread_run(void)
494 {
495         struct task_thread *tth = (struct task_thread*)current_uthread;
496
497         uth_2ls_thread_exit(tth->func(tth->arg));
498 }
499
500 /* Helper, creates and starts a task thread. */
501 static struct task_thread *__vmm_run_task(struct virtual_machine *vm,
502                                           void *(*func)(void *), void *arg,
503                                           struct uth_thread_attr *tth_attr)
504 {
505         struct task_thread *tth;
506
507         tth = (struct task_thread*)alloc_vmm_thread(vm, VMM_THREAD_TASK);
508         if (!tth)
509                 return 0;
510         tth->stacksize = VMM_THR_STACKSIZE;
511         tth->stacktop = __alloc_stack(tth->stacksize);
512         if (!tth->stacktop) {
513                 free(tth);
514                 return 0;
515         }
516         tth->func = func;
517         tth->arg = arg;
518         init_user_ctx(&tth->uthread.u_ctx, (uintptr_t)&__task_thread_run,
519                       (uintptr_t)(tth->stacktop));
520         uthread_init((struct uthread*)tth, tth_attr);
521         acct_thread_unblocked((struct vmm_thread*)tth);
522         enqueue_vmm_thread((struct vmm_thread*)tth);
523         return tth;
524 }
525
526 struct task_thread *vmm_run_task(struct virtual_machine *vm,
527                                  void *(*func)(void *), void *arg)
528 {
529         struct uth_thread_attr tth_attr = {.want_tls = TRUE, .detached = TRUE};
530
531         return __vmm_run_task(vm, func, arg, &tth_attr);
532 }
533
534 static struct uthread *vmm_thread_create(void *(*func)(void *), void *arg)
535 {
536         struct uth_thread_attr tth_attr = {.want_tls = TRUE, .detached = FALSE};
537         struct task_thread *tth;
538
539         /* It's OK to not have a VM for a generic thread */
540         tth = __vmm_run_task(NULL, func, arg, &tth_attr);
541         /* But just in case, let's poison it */
542         ((struct vmm_thread*)tth)->vm = (void*)0xdeadbeef;
543         return (struct uthread*)tth;
544 }
545
546 /* Helpers for tracking nr_unblk_* threads. */
547 static void acct_thread_blocked(struct vmm_thread *vth)
548 {
549         switch (vth->type) {
550         case VMM_THREAD_GUEST:
551         case VMM_THREAD_CTLR:
552                 atomic_dec(&nr_unblk_guests);
553                 break;
554         case VMM_THREAD_TASK:
555                 atomic_dec(&nr_unblk_tasks);
556                 break;
557         }
558 }
559
560 static void acct_thread_unblocked(struct vmm_thread *vth)
561 {
562         switch (vth->type) {
563         case VMM_THREAD_GUEST:
564         case VMM_THREAD_CTLR:
565                 atomic_inc(&nr_unblk_guests);
566                 break;
567         case VMM_THREAD_TASK:
568                 atomic_inc(&nr_unblk_tasks);
569                 break;
570         }
571 }
572
573 static void enqueue_vmm_thread(struct vmm_thread *vth)
574 {
575         spin_pdr_lock(&queue_lock);
576         switch (vth->type) {
577         case VMM_THREAD_GUEST:
578         case VMM_THREAD_CTLR:
579                 TAILQ_INSERT_TAIL(&rnbl_guests, vth, tq_next);
580                 break;
581         case VMM_THREAD_TASK:
582                 TAILQ_INSERT_TAIL(&rnbl_tasks, vth, tq_next);
583                 break;
584         }
585         spin_pdr_unlock(&queue_lock);
586         try_to_get_vcores();
587 }
588
589 static struct vmm_thread *alloc_vmm_thread(struct virtual_machine *vm, int type)
590 {
591         struct vmm_thread *vth;
592         int ret;
593
594         ret = posix_memalign((void**)&vth, __alignof__(struct vmm_thread),
595                              sizeof(struct vmm_thread));
596         if (ret)
597                 return 0;
598         memset(vth, 0, sizeof(struct vmm_thread));
599         vth->type = type;
600         vth->vm = vm;
601         return vth;
602 }
603
604 static void __free_stack(void *stacktop, size_t stacksize)
605 {
606         munmap(stacktop - stacksize, stacksize);
607 }
608
609 static void *__alloc_stack(size_t stacksize)
610 {
611         int force_a_page_fault;
612         void *stacktop;
613         void *stackbot = mmap(0, stacksize, PROT_READ | PROT_WRITE | PROT_EXEC,
614                               MAP_ANONYMOUS, -1, 0);
615
616         if (stackbot == MAP_FAILED)
617                 return 0;
618         stacktop = stackbot + stacksize;
619         /* Want the top of the stack populated, but not the rest of the stack;
620          * that'll grow on demand (up to stacksize, then will clobber memory). */
621         force_a_page_fault = ACCESS_ONCE(*(int*)(stacktop - sizeof(int)));
622         return stacktop;
623 }