Active messages using dynamically allocated memory
authorBarret Rhoden <brho@cs.berkeley.edu>
Fri, 23 Oct 2009 05:15:03 +0000 (22:15 -0700)
committerBarret Rhoden <brho@cs.berkeley.edu>
Fri, 23 Oct 2009 05:15:03 +0000 (22:15 -0700)
This should avoid the active message deadlock issues.  This uses the
slab allocator to create new messages, so a core doesn't run out of
slots for amsgs.  You could still run out of memory...

Note that the amsg is allocated on one core and freed on the other.
This really wants a magazine layer or something similar per-core, so
these allocations and frees don't contend on a global cache lock.

We can gut some things in a future commit, such as the sync variety of
send_amsg, and any of the TODO (AMDL)s.

Also note there is something seriously wrong, perhaps just with KVM, in
which an interrupt coming in around the time of proc_startcore can make
someone "forget" they are holding the lock.  Or rather other people spin
waiting for a lock that never gets released (though the guilty party
progresses as if it was normal).  (TODO).

12 files changed:
kern/arch/i386/arch.h
kern/arch/i386/smp_boot.c
kern/arch/i386/trap.c
kern/include/smp.h
kern/include/trap.h
kern/src/Makefrag
kern/src/init.c
kern/src/kfs.c
kern/src/manager.c
kern/src/monitor.c
user/apps/roslib/mhello.c
user/apps/roslib/mproctests.c [new file with mode: 0644]

index 3ddb38f..6c8f95c 100644 (file)
@@ -8,6 +8,7 @@
 
 /* Arch Constants */
 #define MAX_NUM_CPUS                           255
+#define HW_CACHE_ALIGN                          64
 
 static __inline void breakpoint(void) __attribute__((always_inline));
 static __inline void invlpg(void *SNT addr) __attribute__((always_inline));
index 29fb9f3..b3d865c 100644 (file)
@@ -257,6 +257,10 @@ uint32_t smp_main(void)
        // set a default logical id for now
        lapic_set_logid(lapic_get_id());
 
+       // TODO: do this somewhere else.  in general, we need to do some per_cpu
+       // info init.  or a per_cpu active_msg init
+       STAILQ_INIT(&per_cpu_info[core_id()].active_msgs);
+
        return my_stack_top; // will be loaded in smp_entry.S
 }
 
index 35a458d..0090f86 100644 (file)
@@ -15,6 +15,7 @@
 #include <monitor.h>
 #include <process.h>
 #include <stdio.h>
+#include <slab.h>
 
 #include <syscall.h>
 
@@ -384,31 +385,32 @@ void sysenter_callwrapper(struct Trapframe *tf)
        proc_startcore(current, tf);
 }
 
+struct kmem_cache *active_msg_cache;
+void active_msg_init(void)
+{
+       active_msg_cache = kmem_cache_create("active_msgs",
+                          sizeof(struct active_message), HW_CACHE_ALIGN, 0, 0, 0);
+}
+
 uint32_t send_active_message(uint32_t dst, amr_t pc,
                              TV(a0t) arg0, TV(a1t) arg1, TV(a2t) arg2)
 {
-       error_t retval = -EBUSY;
+       active_message_t *a_msg;
        assert(pc);
+       // note this will be freed on the destination core
+       a_msg = kmem_cache_alloc(active_msg_cache, 0);
+       a_msg->srcid = core_id();
+       a_msg->pc = pc;
+       a_msg->arg0 = arg0;
+       a_msg->arg1 = arg1;
+       a_msg->arg2 = arg2;
        spin_lock_irqsave(&per_cpu_info[dst].amsg_lock);
-       size_t current_amsg = per_cpu_info[dst].amsg_current;
-       // If there's a PC there, then that means it's an outstanding message
-       FOR_CIRC_BUFFER(current_amsg, NUM_ACTIVE_MESSAGES, i) {
-               if (per_cpu_info[dst].active_msgs[i].pc)
-                       continue;
-               per_cpu_info[dst].active_msgs[i].pc = pc;
-               per_cpu_info[dst].active_msgs[i].arg0 = arg0;
-               per_cpu_info[dst].active_msgs[i].arg1 = arg1;
-               per_cpu_info[dst].active_msgs[i].arg2 = arg2;
-               per_cpu_info[dst].active_msgs[i].srcid = core_id();
-               retval = 0;
-               break;
-       }
+       STAILQ_INSERT_TAIL(&per_cpu_info[dst].active_msgs, a_msg, link);
        spin_unlock_irqsave(&per_cpu_info[dst].amsg_lock);
        // since we touched memory the other core will touch (the lock), we don't
        // need an wmb_f()
-       if (!retval)
-               send_ipi(dst, 0, I_ACTIVE_MSG);
-       return retval;
+       send_ipi(dst, 0, I_ACTIVE_MSG);
+       return 0;
 }
 
 /* Active message handler.  We don't want to block other AMs from coming in, so
@@ -417,35 +419,48 @@ uint32_t send_active_message(uint32_t dst, amr_t pc,
  * we already handled the message (or someone is sending IPIs without loading
  * the active message...)
  * Note that all of this happens from interrupt context, and interrupts are
- * currently disabled for this gate. */
+ * currently disabled for this gate.  Interrupts need to be disabled so that the
+ * self-ipi doesn't preempt the execution of this active message. */
 void __active_message(trapframe_t *tf)
 {
        per_cpu_info_t RO*myinfo = &per_cpu_info[core_id()];
-       active_message_t amsg;
+       active_message_t my_msg, *a_msg;
 
        lapic_send_eoi();
-       while (1) { // will break out when we find an empty amsg
+       while (1) { // will break out when there are no more messages
                /* Get the message */
                spin_lock_irqsave(&myinfo->amsg_lock);
-               if (myinfo->active_msgs[myinfo->amsg_current].pc) {
-                       amsg = myinfo->active_msgs[myinfo->amsg_current];
-                       myinfo->active_msgs[myinfo->amsg_current].pc = 0;
-                       myinfo->amsg_current = (myinfo->amsg_current + 1) %
-                                              NUM_ACTIVE_MESSAGES;
-               } else { // was no PC in the current active message, meaning we do nothing
+               a_msg = STAILQ_FIRST(&myinfo->active_msgs);
+               /* No messages to execute, so break out, etc. */
+               if (!a_msg) {
                        spin_unlock_irqsave(&myinfo->amsg_lock);
                        return;
                }
+               STAILQ_REMOVE_HEAD(&myinfo->active_msgs, link);
+               spin_unlock_irqsave(&myinfo->amsg_lock);
+               // copy in, and then free, in case we don't return
+               my_msg = *a_msg;
+               kmem_cache_free(active_msg_cache, a_msg);
+               assert(my_msg.pc);
                /* In case the function doesn't return (which is common: __startcore,
                 * __death, etc), there is a chance we could lose an amsg.  We can only
                 * have up to two interrupts outstanding, and if we never return, we
                 * never deal with any other amsgs.  This extra IPI hurts performance
                 * but is only necessary if there is another outstanding message in the
                 * buffer, but makes sure we never miss out on an amsg. */
-               if (myinfo->active_msgs[myinfo->amsg_current].pc)
-                       send_ipi(core_id(), 0, I_ACTIVE_MSG);
+               spin_lock_irqsave(&myinfo->amsg_lock);
+               if (!STAILQ_EMPTY(&myinfo->active_msgs))
+                       send_self_ipi(I_ACTIVE_MSG);
                spin_unlock_irqsave(&myinfo->amsg_lock);
+
+               /* TODO: for some reason, this is breaking things (startcore loses the
+                * proc_lock at some point...  it's a null interrupt, so you should just
+                * take it and return.  this happens for any interrupt that comes in
+                * (like from proc_destroy too) when the cores are about to grab the
+                * proc_lock.  Note there is no reason to self ipi #254. */
+               //send_self_ipi(254);
+
                /* Execute the active message */
-               amsg.pc(tf, amsg.srcid, amsg.arg0, amsg.arg1, amsg.arg2);
+               my_msg.pc(tf, my_msg.srcid, my_msg.arg0, my_msg.arg1, my_msg.arg2);
        }
 }
index 0a77e13..6e6fe71 100644 (file)
@@ -11,6 +11,7 @@
 
 #include <arch/smp.h>
 #include <ros/common.h>
+#include <sys/queue.h>
 #include <trap.h>
 #include <atomic.h>
 #include <process.h>
@@ -33,11 +34,8 @@ struct per_cpu_info {
        sharC_env_t sharC_env;
 #endif
 
-#ifdef __i386__
        spinlock_t amsg_lock;
-       unsigned LCKD(&amsg_lock) amsg_current;
-       active_message_t LCKD(&amsg_lock) (RO active_msgs)[NUM_ACTIVE_MESSAGES];
-#endif
+       struct active_msg_list active_msgs;
 };
 
 typedef struct per_cpu_info NTPTV(t) NTPTV(a0t) NTPTV(a1t) NTPTV(a2t) per_cpu_info_t;
index eec4b27..903c37f 100644 (file)
@@ -9,6 +9,7 @@
 #include <arch/arch.h>
 #include <arch/mmu.h>
 #include <arch/trap.h>
+#include <sys/queue.h>
 
 // func ptr for interrupt service routines
 typedef void ( *poly_isr_t)(trapframe_t* tf, TV(t) data);
@@ -49,18 +50,20 @@ extern void sysenter_handler();
  * messages require a unique message.  Also for now, but it might be like that
  * for a while on x86. */
 
+void active_msg_init(void);
 typedef void (*amr_t)(trapframe_t* tf, uint32_t srcid,
                       TV(a0t) a0, TV(a1t) a1, TV(a2t) a2);
 
 struct active_message
 {
+       STAILQ_ENTRY(active_message) link;
        uint32_t srcid;
        amr_t pc;
        TV(a0t) arg0;
        TV(a1t) arg1;
        TV(a2t) arg2;
-       uint32_t pad;
 };
+STAILQ_HEAD(active_msg_list, active_message);
 typedef struct active_message NTPTV(a0t) NTPTV(a1t) NTPTV(a2t) active_message_t;
 
 uint32_t send_active_message(uint32_t dst, amr_t pc,
index 5352eaf..4bf15e7 100644 (file)
@@ -55,6 +55,7 @@ KERN_APPFILES := \
                  $(USER_APPS_ROSLIB_DIR)/spawn \
                  $(USER_APPS_ROSLIB_DIR)/hello \
                  $(USER_APPS_ROSLIB_DIR)/mhello \
+                 $(USER_APPS_ROSLIB_DIR)/mproctests \
                  $(USER_APPS_ROSLIB_DIR)/measurements \
                  $(USER_APPS_PARLIB_DIR)/draw_nanwan_standalone \
                  $(USER_APPS_PARLIB_DIR)/channel_test_client \
index 2c923f4..a58831e 100644 (file)
@@ -69,6 +69,7 @@ void kernel_init(multiboot_info_t *mboot_info)
        kmalloc_init();
 
        idt_init();
+       active_msg_init();
        sysenter_init();
        timer_init();
        
index 8479545..36e0a0c 100644 (file)
@@ -34,6 +34,7 @@ DECL_PROG(roslib_null);
 DECL_PROG(roslib_spawn);
 DECL_PROG(roslib_hello);
 DECL_PROG(roslib_mhello);
+DECL_PROG(roslib_mproctests);
 DECL_PROG(roslib_measurements);
 DECL_PROG(parlib_draw_nanwan_standalone);
 DECL_PROG(parlib_channel_test_client);
@@ -48,6 +49,7 @@ struct kfs_entry kfs[MAX_KFS_FILES] = {
        KFS_ENTRY(roslib_spawn)
        KFS_ENTRY(roslib_hello)
        KFS_ENTRY(roslib_mhello)
+       KFS_ENTRY(roslib_mproctests)
        KFS_ENTRY(roslib_measurements)
        KFS_ENTRY(parlib_draw_nanwan_standalone)
        KFS_ENTRY(parlib_channel_test_client)
index d35dbdf..6cab0fb 100644 (file)
@@ -37,6 +37,7 @@ void manager(void)
        uint32_t corelist[MAX_NUM_CPUS];
        uint32_t num = 3;
 
+       /*
        // This is a bypass of the standard manager structure, for network use
        // If enabled, this spawns parlib_matrix, and allows the execution
        // of a remote binary to function correctly (schedule() call below)
@@ -46,10 +47,11 @@ void manager(void)
                proc_run(envs[0]);
        }
        schedule();
+       */
 
        switch (progress++) {
                case 0:
-                       //p = kfs_proc_create(kfs_lookup_path("roslib_proctests"));
+                       //p = kfs_proc_create(kfs_lookup_path("roslib_mproctests"));
                        p = kfs_proc_create(kfs_lookup_path("roslib_mhello"));
                        // being proper and all:
                        spin_lock_irqsave(&p->proc_lock);
index 8981b05..1c9163d 100644 (file)
@@ -276,6 +276,7 @@ int mon_procinfo(int argc, char *NTS *NT COUNT(argc) argv, trapframe_t *tf)
                printk("\trunnable: show proc_runnablelist\n");
                printk("\tresources: show resources wanted/granted for all procs\n");
                printk("\tpid NUM: show a lot of info for proc NUM\n");
+               printk("\tunlock NUM: unlock the lock for proc NUM (OMG!!!)\n");
                return 1;
        }
        if (!strcmp(argv[1], "idle_cores")) {
@@ -290,6 +291,14 @@ int mon_procinfo(int argc, char *NTS *NT COUNT(argc) argv, trapframe_t *tf)
                        return 1;
                }
                print_proc_info(strtol(argv[2], 0, 0));
+       } else if (!strcmp(argv[1], "unlock")) {
+               if (argc != 3) {
+                       printk("Give me a pid number.\n");
+                       return 1;
+               }
+               struct proc *p;
+               envid2env(strtol(argv[2], 0, 0), &p, 0);
+               spin_unlock_irqsave(&p->proc_lock);
        } else {
                printk("Bad option\n");
                return 1;
index 33e5d73..ac7581a 100644 (file)
@@ -20,23 +20,11 @@ void udelay(uint64_t usec, uint64_t tsc_freq)
        return;
 }
 
-#define TEST_MMAP                                      1
-#define TEST_ONE_CORE                          2
-#define TEST_ASK_FOR_TOO_MANY_CORES    3
-#define TEST_INCREMENTAL_CHANGES       4
-#define TEST_YIELD_OUT_OF_ORDER                5
-#define TEST_YIELD_0_OUT_OF_ORDER      6
-#define TEST_SWITCH_TO_RUNNABLE_S      7
-#define TEST_CRAZY_YIELDS                      8
-#define TEST_CONCURRENT_SYSCALLS       9
-
 int main(int argc, char** argv)
 {
        uint32_t vcoreid;
        error_t retval;
 
-       int test = TEST_INCREMENTAL_CHANGES;
-
        static int first_time = 1; // used by vcore2
 
        if ((vcoreid = newcore())) {
@@ -44,106 +32,7 @@ int main(int argc, char** argv)
        } else { // core 0
                cprintf("Hello from else vcore 0\n");
                cprintf("Multi-Goodbye, world, from PID: %d!\n", sys_getpid());
-               switch (test) {
-                       case TEST_MMAP:
-                               cprintf("Testing MMAP\n");
-                               void* addr;
-                               addr = sys_mmap((void*SNT)USTACKTOP - 20*PGSIZE, 8*PGSIZE, 3,
-                                               MAP_FIXED, 0, 0);
-                               cprintf("got addr = 0x%08x\n", addr);
-                               *(int*)addr = 0xdeadbeef;
-                               *(int*)(addr + 3*PGSIZE) = 0xcafebabe;
-                               // these should work
-                               cprintf("reading addr: 0x%08x\n", *(int*)addr);
-                               cprintf("reading addr+3pg: 0x%08x\n", *(int*)(addr + 3*PGSIZE));
-                               // this should fault
-                               cprintf("Should page fault and die now.\n");
-                               *(int*)(addr - 3*PGSIZE) = 0xdeadbeef;
-                               cprintf("Should not see me!!!!!!!!!!!!!!!!!!\n");
-                               while(1);
-                       case TEST_ONE_CORE:
-                               retval = sys_resource_req(RES_CORES, 1, 0);
-                               cprintf("One core test's core0's retval: %d\n", retval);
-                               cprintf("Check to see it's on a worker core.\n");
-                               while(1);
-                       case TEST_ASK_FOR_TOO_MANY_CORES:
-                               retval = sys_resource_req(RES_CORES, 12, 0);
-                               cprintf("Asked for too many, retval: %d\n", retval);
-                               return 0;
-                       case TEST_INCREMENTAL_CHANGES:
-                               retval = sys_resource_req(RES_CORES, 4, 0);
-                               break;
-                       default:
-                               retval = sys_resource_req(RES_CORES, 7, 0);
-               }
-               cprintf("Should see me if you want to relocate core0's context "
-                       "when moving from RUNNING_S\n");
-       }
-       if ((vcoreid == 2) && first_time) {
-               first_time = 0;
-               switch (test) {
-                       case TEST_INCREMENTAL_CHANGES:
-                               // Testing asking for less than we already have
-                               udelay(1000000, 1995014570); // KVM's freq.  Whatever.
-                               cprintf("Asking for too few:\n");
-                               retval = sys_resource_req(RES_CORES, 2, 0);
-                               cprintf("Should be -EINVAL(7): %d\n", retval);
-                               // Testing getting more while running
-                               cprintf("Asking for more while running:\n");
-                               udelay(1000000, 1995014570);
-                               retval = sys_resource_req(RES_CORES, 7, 0);
-                               cprintf("core2's retval: %d\n", retval);
-                               break;
-                       case TEST_YIELD_OUT_OF_ORDER:
-                               cprintf("Core %d yielding\n", vcoreid);
-                               yield();
-                               break;
-                       case TEST_YIELD_0_OUT_OF_ORDER:
-                               udelay(5000000, 1995014570);
-                               cprintf("Core 0 should have yielded, asking for another\n");
-                               retval = sys_resource_req(RES_CORES, 7, 0);
-               }
-       }
-       if (vcoreid == 0) {
-               switch (test) {
-                       case TEST_YIELD_OUT_OF_ORDER:
-                               udelay(10000000, 1995014570);
-                               cprintf("Core 2 should have yielded, asking for another\n");
-                               retval = sys_resource_req(RES_CORES, 7, 0);
-                               break;
-                       case TEST_YIELD_0_OUT_OF_ORDER:
-                               udelay(5000000, 1995014570);
-                               cprintf("Core %d yielding\n", vcoreid);
-                               yield();
-                               cprintf("Core 0 came back where it left off in RUNNING_M!!!\n");
-                               break;
-               }
-       }
-       /* This assumes the "core0 is the one who gets saved" style */
-       switch (test) {
-               case TEST_SWITCH_TO_RUNNABLE_S:
-                       if (vcoreid) {
-                               yield();
-                       } else {
-                               udelay(5000000, 1995014570);
-                               cprintf("Core %d yielding\n", vcoreid);
-                               yield();
-                               cprintf("Should see me if you are ever scheduled again.\n");
-                       }
-                       while(1);
-               case TEST_CRAZY_YIELDS:
-                       udelay(300000*vcoreid, 1995014570);
-                       sys_resource_req(RES_CORES, 7, 0);
-                       yield();
-                       cprintf("should  never see me, unless you slip into *_S\n");
-                       break;
-               case TEST_CONCURRENT_SYSCALLS:
-                       for (int i = 0; i < 10; i++) {
-                               for (int j = 0; j < 100; j++)
-                                       sys_null();
-                               cprintf("Hello from vcore %d, iteration %d\n", vcoreid, i);
-                       }
-                       break;
+               retval = sys_resource_req(RES_CORES, 7, 0);
        }
        cprintf("Vcore %d Done!\n", vcoreid);
        while (1);
diff --git a/user/apps/roslib/mproctests.c b/user/apps/roslib/mproctests.c
new file mode 100644 (file)
index 0000000..33e5d73
--- /dev/null
@@ -0,0 +1,151 @@
+#include <lib.h>
+#include <ros/mman.h>
+#include <ros/resource.h>
+#include <syswrapper.h>
+#include <stdio.h>
+
+// ghetto udelay, put in a lib somewhere and export the tsc freq
+#include <arch/arch.h>
+void udelay(uint64_t usec, uint64_t tsc_freq)
+{
+       uint64_t start, end, now;
+
+       start = read_tsc();
+    end = start + (tsc_freq * usec) / 1000000;
+       if (end == 0) cprintf("This is terribly wrong \n");
+       do {
+        cpu_relax();
+        now = read_tsc();
+       } while (now < end || (now > start && end < start));
+       return;
+}
+
+#define TEST_MMAP                                      1
+#define TEST_ONE_CORE                          2
+#define TEST_ASK_FOR_TOO_MANY_CORES    3
+#define TEST_INCREMENTAL_CHANGES       4
+#define TEST_YIELD_OUT_OF_ORDER                5
+#define TEST_YIELD_0_OUT_OF_ORDER      6
+#define TEST_SWITCH_TO_RUNNABLE_S      7
+#define TEST_CRAZY_YIELDS                      8
+#define TEST_CONCURRENT_SYSCALLS       9
+
+int main(int argc, char** argv)
+{
+       uint32_t vcoreid;
+       error_t retval;
+
+       int test = TEST_INCREMENTAL_CHANGES;
+
+       static int first_time = 1; // used by vcore2
+
+       if ((vcoreid = newcore())) {
+               cprintf("Hello from vcore %d\n", vcoreid);
+       } else { // core 0
+               cprintf("Hello from else vcore 0\n");
+               cprintf("Multi-Goodbye, world, from PID: %d!\n", sys_getpid());
+               switch (test) {
+                       case TEST_MMAP:
+                               cprintf("Testing MMAP\n");
+                               void* addr;
+                               addr = sys_mmap((void*SNT)USTACKTOP - 20*PGSIZE, 8*PGSIZE, 3,
+                                               MAP_FIXED, 0, 0);
+                               cprintf("got addr = 0x%08x\n", addr);
+                               *(int*)addr = 0xdeadbeef;
+                               *(int*)(addr + 3*PGSIZE) = 0xcafebabe;
+                               // these should work
+                               cprintf("reading addr: 0x%08x\n", *(int*)addr);
+                               cprintf("reading addr+3pg: 0x%08x\n", *(int*)(addr + 3*PGSIZE));
+                               // this should fault
+                               cprintf("Should page fault and die now.\n");
+                               *(int*)(addr - 3*PGSIZE) = 0xdeadbeef;
+                               cprintf("Should not see me!!!!!!!!!!!!!!!!!!\n");
+                               while(1);
+                       case TEST_ONE_CORE:
+                               retval = sys_resource_req(RES_CORES, 1, 0);
+                               cprintf("One core test's core0's retval: %d\n", retval);
+                               cprintf("Check to see it's on a worker core.\n");
+                               while(1);
+                       case TEST_ASK_FOR_TOO_MANY_CORES:
+                               retval = sys_resource_req(RES_CORES, 12, 0);
+                               cprintf("Asked for too many, retval: %d\n", retval);
+                               return 0;
+                       case TEST_INCREMENTAL_CHANGES:
+                               retval = sys_resource_req(RES_CORES, 4, 0);
+                               break;
+                       default:
+                               retval = sys_resource_req(RES_CORES, 7, 0);
+               }
+               cprintf("Should see me if you want to relocate core0's context "
+                       "when moving from RUNNING_S\n");
+       }
+       if ((vcoreid == 2) && first_time) {
+               first_time = 0;
+               switch (test) {
+                       case TEST_INCREMENTAL_CHANGES:
+                               // Testing asking for less than we already have
+                               udelay(1000000, 1995014570); // KVM's freq.  Whatever.
+                               cprintf("Asking for too few:\n");
+                               retval = sys_resource_req(RES_CORES, 2, 0);
+                               cprintf("Should be -EINVAL(7): %d\n", retval);
+                               // Testing getting more while running
+                               cprintf("Asking for more while running:\n");
+                               udelay(1000000, 1995014570);
+                               retval = sys_resource_req(RES_CORES, 7, 0);
+                               cprintf("core2's retval: %d\n", retval);
+                               break;
+                       case TEST_YIELD_OUT_OF_ORDER:
+                               cprintf("Core %d yielding\n", vcoreid);
+                               yield();
+                               break;
+                       case TEST_YIELD_0_OUT_OF_ORDER:
+                               udelay(5000000, 1995014570);
+                               cprintf("Core 0 should have yielded, asking for another\n");
+                               retval = sys_resource_req(RES_CORES, 7, 0);
+               }
+       }
+       if (vcoreid == 0) {
+               switch (test) {
+                       case TEST_YIELD_OUT_OF_ORDER:
+                               udelay(10000000, 1995014570);
+                               cprintf("Core 2 should have yielded, asking for another\n");
+                               retval = sys_resource_req(RES_CORES, 7, 0);
+                               break;
+                       case TEST_YIELD_0_OUT_OF_ORDER:
+                               udelay(5000000, 1995014570);
+                               cprintf("Core %d yielding\n", vcoreid);
+                               yield();
+                               cprintf("Core 0 came back where it left off in RUNNING_M!!!\n");
+                               break;
+               }
+       }
+       /* This assumes the "core0 is the one who gets saved" style */
+       switch (test) {
+               case TEST_SWITCH_TO_RUNNABLE_S:
+                       if (vcoreid) {
+                               yield();
+                       } else {
+                               udelay(5000000, 1995014570);
+                               cprintf("Core %d yielding\n", vcoreid);
+                               yield();
+                               cprintf("Should see me if you are ever scheduled again.\n");
+                       }
+                       while(1);
+               case TEST_CRAZY_YIELDS:
+                       udelay(300000*vcoreid, 1995014570);
+                       sys_resource_req(RES_CORES, 7, 0);
+                       yield();
+                       cprintf("should  never see me, unless you slip into *_S\n");
+                       break;
+               case TEST_CONCURRENT_SYSCALLS:
+                       for (int i = 0; i < 10; i++) {
+                               for (int j = 0; j < 100; j++)
+                                       sys_null();
+                               cprintf("Hello from vcore %d, iteration %d\n", vcoreid, i);
+                       }
+                       break;
+       }
+       cprintf("Vcore %d Done!\n", vcoreid);
+       while (1);
+       return 0;
+}