Added back harts support
authorAndrew Waterman <waterman@parcad.millennium.berkeley.edu>
Sat, 20 Feb 2010 01:25:30 +0000 (17:25 -0800)
committerKevin Klues <klueska@cs.berkeley.edu>
Thu, 3 Nov 2011 00:35:36 +0000 (17:35 -0700)
13 files changed:
.gitignore
kern/arch/i386/ros/hart.h
kern/arch/sparc/ros/hart.h
kern/arch/sparc/ros/syscall.h
kern/include/errno.h [deleted file]
kern/src/process.c
lib/parlib [deleted symlink]
lib/parlib/hart.c [new file with mode: 0644]
lib/parlib/hart.h [new file with mode: 0644]
lib/parlib/parlib.h [new file with mode: 0644]
lib/parlib/syscall.c [new file with mode: 0644]
tools/compilers/gcc-glibc/Makefile
tools/compilers/gcc-glibc/glibc-2.11.1-ros/sysdeps/ros/start.c

index 9563c36..b4c0bde 100644 (file)
@@ -24,6 +24,12 @@ Documentation/doxygen/rosdoc
 sim/
 fs
 tools/.*
 sim/
 fs
 tools/.*
+tools/compilers/gcc-glibc/.*
+tools/compilers/gcc-glibc/binutils-2.18
+tools/compilers/gcc-glibc/gcc-4.3.2
+tools/compilers/gcc-glibc/glibc-2.11.1
+tools/compilers/gcc-glibc/sparc-ros-*
+tools/compilers/gcc-glibc/i386-ros-*
 tools/syscall_server/syscall_server_*
 tools/syscall_server/sandbox/
 tools/syscall_server/apps/*
 tools/syscall_server/syscall_server_*
 tools/syscall_server/sandbox/
 tools/syscall_server/apps/*
index 4ecf4c4..ecc71ce 100644 (file)
@@ -5,6 +5,9 @@
 
 #include <ros/syscall.h>
 
 
 #include <ros/syscall.h>
 
+// this is how we get our thread id on entry.
+register int __hart_self_on_entry asm ("eax");
+
 // The actual hart_self() function is a global symbol that invokes this routine.
 static inline int
 __hart_self()
 // The actual hart_self() function is a global symbol that invokes this routine.
 static inline int
 __hart_self()
index 822ae2e..4af5595 100644 (file)
@@ -7,6 +7,8 @@ double do_fsqrt(double);
 double do_recip(double);
 double do_rsqrt(double);
 
 double do_recip(double);
 double do_rsqrt(double);
 
+#define __hart_self_on_entry (__hart_self())
+
 static inline int
 __hart_self()
 {
 static inline int
 __hart_self()
 {
index 7b75db4..30b2817 100644 (file)
@@ -1,10 +1,10 @@
-#include <errno.h>
-
 #ifndef _ROS_ARCH_SYSCALL_H
 #define _ROS_ARCH_SYSCALL_H
 
 #ifndef ROS_KERNEL
 
 #ifndef _ROS_ARCH_SYSCALL_H
 #define _ROS_ARCH_SYSCALL_H
 
 #ifndef ROS_KERNEL
 
+#include <errno.h>
+
 static inline long __attribute__((always_inline))
 __ros_syscall(long _num, long _a0, long _a1, long _a2, long _a3, long _a4)
 {
 static inline long __attribute__((always_inline))
 __ros_syscall(long _num, long _a0, long _a1, long _a2, long _a3, long _a4)
 {
@@ -17,10 +17,10 @@ __ros_syscall(long _num, long _a0, long _a1, long _a2, long _a3, long _a4)
                     : "r"(num),"0"(a0),"1"(a1),"r"(a2),"r"(a3),"r"(a4));
 
        // move a1, a2 into regular variables so they're volatile across
                     : "r"(num),"0"(a0),"1"(a1),"r"(a2),"r"(a3),"r"(a4));
 
        // move a1, a2 into regular variables so they're volatile across
-       // procedure calls
+       // procedure calls (of which errno is one)
        long ret = a0, err = a1;
        if(err != 0)
        long ret = a0, err = a1;
        if(err != 0)
-               __set_errno(err);
+               errno = err;
 
         return ret;
 }
 
         return ret;
 }
diff --git a/kern/include/errno.h b/kern/include/errno.h
deleted file mode 100644 (file)
index be874f6..0000000
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef _ERRNO_H
-#define _ERRNO_H
-
-#define __set_errno(x) do { } while(0)
-
-#endif
index 386d2d4..50193d3 100644 (file)
@@ -700,10 +700,10 @@ void proc_yield(struct proc *SAFE p)
                        schedule_proc(p);
                        break;
                case (PROC_RUNNING_M):
                        schedule_proc(p);
                        break;
                case (PROC_RUNNING_M):
-                       p->resources[RES_CORES].amt_granted = --(p->num_vcores);
-                       p->resources[RES_CORES].amt_wanted = p->num_vcores;
                        // give up core
                        p->vcoremap[get_vcoreid(p, core_id())] = -1;
                        // give up core
                        p->vcoremap[get_vcoreid(p, core_id())] = -1;
+                       p->resources[RES_CORES].amt_granted = --(p->num_vcores);
+                       p->resources[RES_CORES].amt_wanted = p->num_vcores;
                        // add to idle list
                        put_idle_core(core_id());
                        // last vcore?  then we really want 1, and to yield the gang
                        // add to idle list
                        put_idle_core(core_id());
                        // last vcore?  then we really want 1, and to yield the gang
diff --git a/lib/parlib b/lib/parlib
deleted file mode 120000 (symlink)
index 54bdef8..0000000
+++ /dev/null
@@ -1 +0,0 @@
-../tools/compilers/gcc-glibc/glibc-2.11.1-ros/sysdeps/parlib
\ No newline at end of file
diff --git a/lib/parlib/hart.c b/lib/parlib/hart.c
new file mode 100644 (file)
index 0000000..0f57f46
--- /dev/null
@@ -0,0 +1,250 @@
+#include <stdbool.h>
+#include <errno.h>
+#include <hart.h>
+#include <parlib.h>
+#include <unistd.h>
+#include <stdlib.h>
+
+static size_t _current_harts = 1;
+static hart_lock_t _hart_lock = HART_LOCK_INIT;
+
+extern char** hart_stack_pointers;
+extern void** hart_thread_control_blocks;
+
+static void hart_free_tls(int id)
+{
+       extern void _dl_deallocate_tls (void *tcb, bool dealloc_tcb);
+       if(hart_thread_control_blocks[id])
+       {
+               _dl_deallocate_tls(hart_thread_control_blocks[id],true);
+               hart_thread_control_blocks[id] = 0;
+       }
+}
+
+static int hart_allocate_tls(int id)
+{
+       extern void *_dl_allocate_tls (void *mem);
+       // instead of freeing any old tls that may be present, and then
+       // reallocating it, we could instead just reinitialize it.
+       hart_free_tls(id);
+       if((hart_thread_control_blocks[id] = _dl_allocate_tls(NULL)) == NULL)
+       {
+               errno = ENOMEM;
+               return -1;
+       }
+       return 0;
+}
+
+#define HART_STACK_SIZE (32*1024)
+#define HART_STACK_POINTER_OFFSET (HART_STACK_SIZE-96)
+
+static void hart_free_stack(int id)
+{
+       // don't actually free stacks
+}
+
+static int hart_allocate_stack(int id)
+{
+       if(hart_stack_pointers[id])
+               return 0; // reuse old stack
+
+       if((hart_stack_pointers[id] = malloc(HART_STACK_SIZE)) == NULL)
+       {
+               errno = ENOMEM;
+               return -1;
+       }
+       hart_stack_pointers[id] += HART_STACK_POINTER_OFFSET;
+       return 0;
+}
+
+static int hart_init()
+{
+       static int initialized = 0;
+       if(initialized)
+               return 0;
+
+       hart_stack_pointers = (char**)calloc(hart_max_harts(),sizeof(char*));
+       hart_thread_control_blocks = (void**)calloc(hart_max_harts(),sizeof(void*));
+
+       if(!hart_thread_control_blocks || !hart_stack_pointers)
+       {
+               free(hart_thread_control_blocks);
+               free(hart_stack_pointers);
+               errno = ENOMEM;
+               return -1;
+       }
+
+       initialized = 1;
+       return 0;
+}
+
+int hart_request(size_t k)
+{
+       int ret = -1;
+       size_t i,j;
+
+       if(hart_init() < 0)
+               return -1;
+
+       hart_lock_lock(&_hart_lock);
+
+       if(k < 0 || _current_harts+k > hart_max_harts())
+       {
+               errno = EAGAIN;
+               goto fail;
+       }
+
+       for(i = _current_harts, j = 0; i < _current_harts+k; i++, j++)
+       {
+               if(hart_allocate_stack(i) || hart_allocate_tls(i))
+                       goto fail;
+       }
+
+       if((ret = sys_resource_req(0,_current_harts+k,0)) == 0)
+       {
+               _current_harts += k;
+               goto success;
+       }
+
+fail:
+       for(i = _current_harts; i < _current_harts+k; i++)
+       {
+               hart_free_tls(i);
+               hart_free_stack(i);
+       }
+
+success:
+       hart_lock_unlock(&_hart_lock);
+       return ret;
+}
+
+void hart_yield()
+{
+       int id = hart_self();
+
+       hart_lock_lock(&_hart_lock);
+       _current_harts--;
+       if(_current_harts == 0)
+               exit(0);
+       hart_lock_unlock(&_hart_lock);
+
+       sys_yield();
+}
+
+size_t hart_max_harts()
+{
+       return __procinfo.max_harts < HART_MAX_MAX_HARTS ? __procinfo.max_harts : HART_MAX_MAX_HARTS;
+}
+
+size_t hart_current_harts()
+{
+       return _current_harts;
+}
+
+// MCS locks!!
+void hart_lock_init(hart_lock_t* lock)
+{
+       memset(lock,0,sizeof(hart_lock_t));
+}
+
+static inline hart_lock_qnode_t* hart_qnode_swap(hart_lock_qnode_t** addr, hart_lock_qnode_t* val)
+{
+       return (hart_lock_qnode_t*)hart_swap((int*)addr,(int)val);
+}
+
+void hart_lock_lock(hart_lock_t* lock)
+{
+       hart_lock_qnode_t* qnode = &lock->qnode[hart_self()];
+       qnode->next = 0;
+       hart_lock_qnode_t* predecessor = hart_qnode_swap(&lock->lock,qnode);
+       if(predecessor)
+       {
+               qnode->locked = 1;
+               predecessor->next = qnode;
+               while(qnode->locked);
+       }
+}
+
+void hart_lock_unlock(hart_lock_t* lock)
+{
+       hart_lock_qnode_t* qnode = &lock->qnode[hart_self()];
+       if(qnode->next == 0)
+       {
+               hart_lock_qnode_t* old_tail = hart_qnode_swap(&lock->lock,0);
+               if(old_tail == qnode)
+                       return;
+
+               hart_lock_qnode_t* usurper = hart_qnode_swap(&lock->lock,old_tail);
+               while(qnode->next == 0);
+               if(usurper)
+                       usurper->next = qnode->next;
+               else
+                       qnode->next->locked = 0;
+       }
+       else
+               qnode->next->locked = 0;
+}
+
+// MCS dissemination barrier!
+int hart_barrier_init(hart_barrier_t* b, size_t np)
+{
+       if(np > hart_max_harts())
+               return -1;
+       b->allnodes = (hart_dissem_flags_t*)malloc(np*sizeof(hart_dissem_flags_t));
+       memset(b->allnodes,0,np*sizeof(hart_dissem_flags_t));
+       b->nprocs = np;
+
+       b->logp = (np & (np-1)) != 0;
+       while(np >>= 1)
+               b->logp++;
+
+       size_t i,k;
+       for(i = 0; i < b->nprocs; i++)
+       {
+               b->allnodes[i].parity = 0;
+               b->allnodes[i].sense = 1;
+
+               for(k = 0; k < b->logp; k++)
+               {
+                       size_t j = (i+(1<<k)) % b->nprocs;
+                       b->allnodes[i].partnerflags[0][k] = &b->allnodes[j].myflags[0][k];
+                       b->allnodes[i].partnerflags[1][k] = &b->allnodes[j].myflags[1][k];
+               } 
+       }
+
+       return 0;
+}
+
+void hart_barrier_wait(hart_barrier_t* b, size_t pid)
+{
+       hart_dissem_flags_t* localflags = &b->allnodes[pid];
+       size_t i;
+       for(i = 0; i < b->logp; i++)
+       {
+               *localflags->partnerflags[localflags->parity][i] = localflags->sense;
+               while(localflags->myflags[localflags->parity][i] != localflags->sense);
+       }
+       if(localflags->parity)
+               localflags->sense = 1-localflags->sense;
+       localflags->parity = 1-localflags->parity;
+}
+
+int
+hart_self()
+{
+       // defined in ros/arch/hart.h
+       return __hart_self();
+}
+
+int
+hart_swap(int* addr, int val)
+{
+       return __hart_swap(addr,val);
+}
+
+void
+hart_relax()
+{
+       __hart_relax();
+}
+
diff --git a/lib/parlib/hart.h b/lib/parlib/hart.h
new file mode 100644 (file)
index 0000000..e6dc52c
--- /dev/null
@@ -0,0 +1,70 @@
+#ifndef _HART_H
+#define _HART_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <ros/arch/hart.h>
+#include <string.h>
+
+#define HART_LOG2_MAX_MAX_HARTS 6
+#define HART_MAX_MAX_HARTS (1 << HART_LOG2_MAX_MAX_HARTS)
+#define HART_CL_SIZE 128
+
+typedef struct hart_lock_qnode
+{
+       volatile struct hart_lock_qnode* volatile next;
+       volatile int locked;
+       char pad[HART_CL_SIZE-sizeof(void*)-sizeof(int)];
+} hart_lock_qnode_t;
+
+typedef struct
+{
+       hart_lock_qnode_t* lock;
+       char pad[HART_CL_SIZE-sizeof(hart_lock_qnode_t*)];
+       hart_lock_qnode_t qnode[HART_MAX_MAX_HARTS] __attribute__((aligned(8)));
+} hart_lock_t;
+
+#define HART_LOCK_INIT {0}
+
+typedef struct
+{
+       volatile int myflags[2][HART_LOG2_MAX_MAX_HARTS];
+       volatile int* partnerflags[2][HART_LOG2_MAX_MAX_HARTS];
+       int parity;
+       int sense;
+       char pad[HART_CL_SIZE];
+} hart_dissem_flags_t;
+
+typedef struct
+{
+       size_t nprocs;
+       hart_dissem_flags_t* allnodes;
+       size_t logp;
+} hart_barrier_t;
+
+extern void hart_entry();
+
+int hart_barrier_init(hart_barrier_t* b, size_t nprocs);
+void hart_barrier_wait(hart_barrier_t* b, size_t vcoreid);
+
+void hart_lock_init(hart_lock_t* lock);
+void hart_lock_unlock(hart_lock_t* lock);
+void hart_lock_lock(hart_lock_t* l);
+
+// "int" rather than size_t because of a newlib compiling issue
+int hart_self();
+void hart_relax();
+int hart_swap(int* addr, int val);
+
+int hart_request(size_t k);
+void hart_yield();
+size_t hart_max_harts();
+size_t hart_current_harts();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/lib/parlib/parlib.h b/lib/parlib/parlib.h
new file mode 100644 (file)
index 0000000..6c417cd
--- /dev/null
@@ -0,0 +1,47 @@
+// Main public header file for our user-land support library,
+// whose code lives in the lib directory.
+// This library is roughly our OS's version of a standard C library,
+// and is intended to be linked into all user-mode applications
+// (NOT the kernel or boot loader).
+
+#ifndef ROS_INC_PARLIB_H
+#define ROS_INC_PARLIB_H 1
+
+#ifndef __ASSEMBLER__
+
+#include <ros/common.h>
+#include <ros/memlayout.h>
+#include <ros/syscall.h>
+#include <ros/procinfo.h>
+#include <stdint.h>
+
+enum {
+       PG_RDONLY = 4,
+       PG_RDWR   = 6,
+};
+
+ssize_t     sys_cputs(const uint8_t *s, size_t len);
+uint16_t    sys_cgetc(void);
+ssize_t     sys_serial_write(void* buf, size_t len); 
+ssize_t     sys_serial_read(void* buf, size_t len);
+ssize_t     sys_eth_write(void *COUNT(len) buf, size_t len); 
+ssize_t     sys_eth_read(void *COUNT(len) buf, size_t len);
+ssize_t     sys_run_binary(void* binary_buf, size_t len,
+                           const procinfo_t* pi, size_t num_colors);
+int         sys_getpid(void);
+size_t      sys_getcpuid(void);
+void *      sys_brk(void* addr);
+int     sys_proc_destroy(int pid, int exitcode);
+ssize_t     sys_shared_page_alloc(void *COUNT(PGSIZE) *addr, pid_t p2, 
+                                  int p1_flags, int p2_flags);
+ssize_t     sys_shared_page_free(void *COUNT(PGSIZE) addr, pid_t p2);
+ssize_t     sys_resource_req(int type, size_t amount, uint32_t flags);
+void        sys_reboot();
+void        sys_yield();
+int         gettimeofday(struct timeval* tp, void* tzp);
+void *COUNT(length) sys_mmap(void *SNT addr, size_t length, int prot, int flags,
+                             int fd, size_t offset);
+
+#endif // !ASSEMBLER
+
+#endif // !ROS_INC_PARLIB_H
diff --git a/lib/parlib/syscall.c b/lib/parlib/syscall.c
new file mode 100644 (file)
index 0000000..e396f21
--- /dev/null
@@ -0,0 +1,121 @@
+// System call stubs.
+
+#include <parlib.h>
+
+int sys_proc_destroy(int pid, int exitcode)
+{
+       return ros_syscall(SYS_proc_destroy, pid, exitcode, 0, 0, 0);
+}
+
+void* sys_brk(void* addr)
+{
+       return (void*)ros_syscall(SYS_brk, addr, 0, 0, 0, 0);
+}
+
+int sys_getpid(void)
+{
+        return ros_syscall(SYS_getpid, 0, 0, 0, 0, 0);
+}
+
+size_t sys_getcpuid(void)
+{
+        return ros_syscall(SYS_getcpuid, 0, 0, 0, 0, 0);
+}
+
+ssize_t sys_cputs(const uint8_t *s, size_t len)
+{
+    return ros_syscall(SYS_cputs, s,  len, 0, 0, 0);
+}
+
+uint16_t sys_cgetc(void)
+{
+    return ros_syscall(SYS_cgetc, 0, 0, 0, 0, 0);
+}
+
+ssize_t sys_shared_page_alloc(void** addr, pid_t p2, 
+                              int p1_flags, int p2_flags
+                             ) 
+{
+       return ros_syscall(SYS_shared_page_alloc, addr, 
+                      p2, p1_flags, p2_flags, 0);
+}
+
+ssize_t sys_shared_page_free(void* addr, pid_t p2) 
+{
+       return ros_syscall(SYS_shared_page_free, addr, p2, 0,0,0);
+}
+
+//Write a buffer over the serial port
+ssize_t sys_serial_write(void* buf, size_t len) 
+{
+       return ros_syscall(SYS_serial_write, buf, len, 0, 0, 0);
+}
+
+//Read a buffer over the serial port
+ssize_t sys_serial_read(void* buf, size_t len) 
+{
+       return ros_syscall(SYS_serial_read, buf, len, 0, 0, 0);
+}
+
+//Run a binary loaded at the specificed address with the specified arguments
+ssize_t sys_run_binary(void* binary_buf, size_t len,
+                       const procinfo_t* pi, size_t num_colors) 
+{
+       return ros_syscall(SYS_run_binary, binary_buf, len,
+                                       pi,num_colors,0);
+}
+
+//Write a buffer over ethernet
+ssize_t sys_eth_write(void* buf, size_t len) 
+{
+       if (len == 0)
+               return 0;
+       
+       return ros_syscall(SYS_eth_write, buf, len, 0, 0, 0);
+}
+
+//Read a buffer via ethernet
+ssize_t sys_eth_read(void* buf, size_t len) 
+{
+       if (len == 0)
+               return 0;
+               
+       return ros_syscall(SYS_eth_read, buf, len, 0, 0, 0);
+}
+
+/* Request resources from the kernel.  Flags in ros/resource.h. */
+ssize_t sys_resource_req(int type, size_t amount, uint32_t flags)
+{
+       return ros_syscall(SYS_resource_req, type, amount, flags, 0, 0);
+}
+
+void sys_reboot()
+{
+       ros_syscall(SYS_reboot,0,0,0,0,0);
+}
+
+void sys_yield()
+{
+       ros_syscall(SYS_yield,0,0,0,0,0);
+}
+
+/* We need to do some hackery to pass 6 arguments.  Arg4 pts to the real arg4,
+ * arg5, and arg6.  Keep this in sync with kern/src/syscall.c.
+ * TODO: consider a syscall_multi that can take more args, and keep it in sync
+ * with the kernel.  Maybe wait til we fix sysenter to have 5 or 6 args. */
+void *CT(length) sys_mmap(void *SNT addr, size_t length, int prot, int flags,
+                          int fd, size_t offset)
+{
+       struct args {
+               int _flags;
+               int _fd;
+               size_t _offset;
+       } extra_args;
+       extra_args._flags = flags;
+       extra_args._fd = fd;
+       extra_args._offset = offset;
+       // TODO: deputy bitches about this
+       return (void*CT(length))TC(ros_syscall(SYS_mmap, addr, length,
+                                              prot, &extra_args, 0));
+}
+
index 6d6bae5..7062b82 100644 (file)
@@ -218,11 +218,11 @@ $(BINARY_PREFIX)gcc-stage2-builddir: gcc-$(GCC_VERSION)
 
 .$(BINARY_PREFIX)ros-headers-install: $(ROSDIR)
        $(MAKE) -C $(ROSDIR) symlinks
 
 .$(BINARY_PREFIX)ros-headers-install: $(ROSDIR)
        $(MAKE) -C $(ROSDIR) symlinks
-       mkdir -p $(INSTDIR)/$(ARCH)-ros/sys-include
+       mkdir -p $(INSTDIR)/$(ARCH)-ros/sys-include/arch
        cp -r $(ROSDIR)/kern/include/ros \
           $(INSTDIR)/$(ARCH)-ros/sys-include/
        cp -r $(ROSDIR)/kern/include/ros \
           $(INSTDIR)/$(ARCH)-ros/sys-include/
-       cp -r $(ROSDIR)/kern/include/arch \
-          $(INSTDIR)/$(ARCH)-ros/sys-include/
+       cp -r $(ROSDIR)/kern/include/arch/* \
+          $(INSTDIR)/$(ARCH)-ros/sys-include/arch/
 
 .$(BINARY_PREFIX)gcc-stage1-configure: 
        $(MAKE) $(BINARY_PREFIX)gcc-stage1-builddir
 
 .$(BINARY_PREFIX)gcc-stage1-configure: 
        $(MAKE) $(BINARY_PREFIX)gcc-stage1-builddir
index 4b997b4..f32f4fb 100644 (file)
@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include <ros/syscall.h>
 #include <ros/procinfo.h>
 #include <stdio.h>
 #include <ros/syscall.h>
 #include <ros/procinfo.h>
+#include <unistd.h>
 
 void** __hart_stack_pointers = NULL;
 weak_alias(__hart_stack_pointers,hart_stack_pointers)
 
 void** __hart_stack_pointers = NULL;
 weak_alias(__hart_stack_pointers,hart_stack_pointers)
@@ -24,23 +25,32 @@ __hart_yield(void)
 }
 weak_alias(__hart_yield,hart_yield)
 
 }
 weak_alias(__hart_yield,hart_yield)
 
+#define failmsg(str) write(2,str"\n",sizeof(str"\n")-1)
+
 void
 _start(void)
 {
 void
 _start(void)
 {
-       // threads besides thread 0 must acquire a stack and TCB
-       if(__hart_self() != 0)
+       // threads besides thread 0 must acquire a stack and TCB.
+       // WARNING: no function calls or register spills may occur
+       // before the stack pointer is set!
+       // WARNING2: __hart_self_on_entry must be read before
+       // anything is register-allocated!
+       int id = __hart_self_on_entry;
+       if(id != 0)
        {
        {
-               __hart_set_stack_pointer(__hart_stack_pointers[__hart_self()]);
-               TLS_INIT_TP(__hart_thread_control_blocks[__hart_self()],0);
+               __hart_set_stack_pointer(__hart_stack_pointers[id]);
+               TLS_INIT_TP(__hart_thread_control_blocks[id],0);
                hart_entry();
                hart_yield();
                hart_entry();
                hart_yield();
+               failmsg("why did hart_yield() return?");
+               goto diediedie;
        }
 
        static int init = 0;
        if(init)
        {
        }
 
        static int init = 0;
        if(init)
        {
-               fputs("why is thread 0 re-entering _start, foo?!",stderr);
-               abort();
+               failmsg("why did thread 0 re-enter _start?");
+               goto diediedie;
        }
        init = 1;
 
        }
        init = 1;
 
@@ -68,6 +78,10 @@ _start(void)
 
        __libc_start_main(&main,argc,argv,&__libc_csu_init,&__libc_csu_fini,0,0);
 
 
        __libc_start_main(&main,argc,argv,&__libc_csu_init,&__libc_csu_fini,0,0);
 
+       failmsg("why did main() return?");
+
+diediedie:
+       abort();
        #ifdef ABORT_INSTRUCTION
        ABORT_INSTRUCTION;
        #endif
        #ifdef ABORT_INSTRUCTION
        ABORT_INSTRUCTION;
        #endif