Make arsc and local async calls use the same structure.
authorDavid Zhu <yuzhu@cs.berkeley.edu>
Wed, 26 Jan 2011 23:35:52 +0000 (15:35 -0800)
committerKevin Klues <klueska@cs.berkeley.edu>
Thu, 3 Nov 2011 00:36:01 +0000 (17:36 -0700)
Instead of putting the entire call and the parameters in the ring,
we now use a pointer to an external structure that the caller waits
on.

This allows for consistent waiting interface on both remote and
local calls.

kern/include/ros/ring_syscall.h
kern/include/syscall.h
kern/src/arsc.c
kern/src/syscall.c
tests/arsc_mt.c
tests/arsc_test.c
user/parlib/asynccall.c
user/parlib/include/arc.h

index 336104d..3689a39 100644 (file)
@@ -7,7 +7,7 @@
 #define NUM_SYSCALL_ARGS 6
 /* This will need to change to represent sending pointers to syscalls, not the
  * syscalls themselves */
-
+struct syscall;
 typedef enum {
        RES_free,  // The response has been digested by the user space, can be reallocated
        REQ_alloc, // Space fo request is allocated
@@ -17,23 +17,16 @@ typedef enum {
 
        RES_ready // The response is ready to be picked up
 } syscall_status_t;
-typedef struct syscall_req {
-    syscall_status_t status;
-       void (*cleanup)(void* data);
-       void *data;
-    uint32_t num;
-    uint32_t args[NUM_SYSCALL_ARGS];
-} syscall_req_t;
 
-typedef struct syscall_rsp {
-       syscall_status_t status;
+typedef struct syscall_req {
+    syscall_status_t status; // TODO:rethink this
        void (*cleanup)(void* data);
        void *data;
-       uint32_t retval;
-       uint32_t syserr;
-} syscall_rsp_t;
-
+       struct syscall* sc;
+} syscall_req_t, syscall_rsp_t;
 
+#define RSP_ERRNO(rsp) (rsp->sc->err)
+#define RSP_RESULT(rsp) (rsp->sc->retval)
 
 // Generic Syscall Ring Buffer
 #define SYSCALLRINGSIZE    PGSIZE
index e4e8f81..c24b69b 100644 (file)
@@ -41,6 +41,7 @@ struct sys_table_entry {
 const static struct sys_table_entry syscall_table[];
 /* Syscall invocation */
 void prep_syscalls(struct proc *p, struct syscall *sysc, unsigned int nr_calls);
+void run_local_syscall(struct syscall *sysc);
 intreg_t syscall(struct proc *p, uintreg_t sc_num, uintreg_t a0, uintreg_t a1,
                  uintreg_t a2, uintreg_t a3, uintreg_t a4, uintreg_t a5);
 void set_errno(int errno);
index 594ddf4..af1dff8 100644 (file)
@@ -28,8 +28,9 @@ spinlock_t arsc_proc_lock = SPINLOCK_INITIALIZER;
 
 intreg_t inline syscall_async(struct proc *p, syscall_req_t *call)
 {
-       return syscall(p, call->num, call->args[0], call->args[1],
-                      call->args[2], call->args[3], call->args[4], call->args[5]);
+       struct syscall* sc = call->sc;
+       return syscall(p, sc->num, sc->arg0, sc->arg1,
+                      sc->arg2, sc->arg3, sc->arg4, sc->arg5);
 }
 
 syscall_sring_t* sys_init_arsc(struct proc *p)
@@ -85,7 +86,7 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
 {
        size_t count = 0;
        syscall_back_ring_t* sysbr = &p->syscallbackring;
-       struct per_cpu_info* coreinfo = &per_cpu_info[core_id()];
+       struct per_cpu_info* pcpui = &per_cpu_info[core_id()];
        // looking at a process not initialized to perform arsc. 
        if (sysbr == NULL) 
                return count;
@@ -99,6 +100,7 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
                        // need to switch to the right context, so we can handle the user pointer
                        // that points to a data payload of the syscall
                        lcr3(p->env_cr3);
+                       pcpui->cur_proc = p;
                }
                count++;
                //printk("DEBUG PRE: sring->req_prod: %d, sring->rsp_prod: %d\n",
@@ -111,18 +113,10 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
                // print req
                printd("req no %d, req arg %c\n", req->num, *((char*)req->args[0]));
                
-               /* TODO: when the remote syscall stuff can handle the new async
-                * syscalls, they need to use a real sysc.  This might at least stop it
-                * from crashing. */
-               struct syscall sysc = {0};
-               coreinfo->cur_sysc = &sysc;
+               pcpui->cur_sysc = req->sc;
+               run_local_syscall(req->sc); // TODO: blocking call will block arcs as well.
                
-               rsp.retval = syscall_async(p, req);
-               rsp.syserr = sysc.err;
-               rsp.cleanup = req->cleanup;
-               rsp.data = req->data;
-               // write response into the slot it came from
-               memcpy(req, &rsp, sizeof(syscall_rsp_t));
+               // need to keep the slot in the ring buffer if it is blocked
                (sysbr->rsp_prod_pvt)++;
                req->status = RES_ready;
                RING_PUSH_RESPONSES(sysbr);
index 57f64d9..5b8ec51 100644 (file)
@@ -1427,7 +1427,7 @@ intreg_t syscall(struct proc *p, uintreg_t sc_num, uintreg_t a0, uintreg_t a1,
 }
 
 /* Execute the syscall on the local core */
-static void run_local_syscall(struct syscall *sysc)
+void run_local_syscall(struct syscall *sysc)
 {
        struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
 
index e5fee5e..6e1ba7d 100644 (file)
@@ -8,7 +8,7 @@
 #include <arc.h>
 #include <stdio.h>
 
-#define NUM_THREADS 4 
+#define NUM_THREADS 4
 
 pthread_t t1;
 
@@ -17,26 +17,18 @@ async_desc_t desc1;
 syscall_desc_t* sys_cputs_async(const char *s, size_t len,                                             
                      void (*cleanup_handler)(void*), void* cleanup_data)
 {                                                                                                                     
-    // could just hardcode 4 0's, will eventually wrap this marshaller anyway                                         
-       syscall_desc_t* desc;
-    syscall_req_t syscall = {REQ_alloc, cleanup_handler, cleanup_data,
-                                                       SYS_cputs,{(uint32_t)s, len, [2 ... (NUM_SYSCALL_ARGS-1)] 0} };                          
-    syscall.cleanup = cleanup_handler;                                                                                  
-    syscall.data = cleanup_data;
-    async_syscall(&syscall, &desc);
-       return desc;
+       return arc_call(SYS_cputs, s, len);
 }
 
 void *syscall_thread(void* arg)
-{      
+{
        char testme ='a';
        char buf[20] = {0};
-       sprintf(buf, "%d", pthread_self()->id );
+       sprintf(buf, "%d", (pthread_self()->id % 10) );
        char tid = buf[0];
        syscall_desc_t* sysdesc;
-       syscall_rsp_t sysrsp;
        sysdesc = sys_cputs_async(&tid, 1, NULL, NULL);
-       waiton_syscall(sysdesc, &sysrsp);
+       assert (-1 != waiton_syscall(sysdesc));
 }
 
 int main(int argc, char** argv){
@@ -48,8 +40,9 @@ int main(int argc, char** argv){
        for (int i = 0; i < NUM_THREADS ; i++)
                pthread_create(&my_threads[i], NULL, &syscall_thread, NULL);
        
-       for (int i = 0; i < NUM_THREADS; i++)
+       for (int i = 0; i < NUM_THREADS; i++){
                pthread_join(my_threads[i], NULL);
+       }
 
        printf("multi thread - end\n");
 }
index 40da172..86552cd 100644 (file)
@@ -7,14 +7,15 @@
 syscall_desc_t* sys_cputs_async(const char *s, size_t len,                                             
                      void (*cleanup_handler)(void*), void* cleanup_data)
 {                                                                                                                     
-    // could just hardcode 4 0's, will eventually wrap this marshaller anyway                                         
+    /*// could just hardcode 4 0's, will eventually wrap this marshaller anyway                                         
        syscall_desc_t* desc;
     syscall_req_t syscall = {REQ_alloc, cleanup_handler, cleanup_data,
                                                        SYS_cputs,{(uint32_t)s, len, [2 ... (NUM_SYSCALL_ARGS-1)] 0} };                          
     syscall.cleanup = cleanup_handler;                                                                                  
-    syscall.data = cleanup_data;                                                                                        
+    syscall.data = cleanup_data;
     async_syscall(&syscall, &desc);
-       return desc;
+       */
+       return arc_call(SYS_cputs, s, len);
 }
 
 int main(int argc, char** argv){
@@ -31,7 +32,8 @@ int main(int argc, char** argv){
        sysdesc[1] = sys_cputs_async(&testme, 1, NULL, NULL);
 
        printf ("single thread - call placed \n");
-       waiton_syscall(sysdesc[0], &sysrsp);
-       waiton_syscall(sysdesc[1], &sysrsp);
+       //ignore return value
+       assert(-1 != waiton_syscall(sysdesc[0]));
+       assert(-1 != waiton_syscall(sysdesc[1]));
        printf ("single thread - dummy call \n");       
 }
index 1e57dc1..e6bb8d8 100644 (file)
@@ -33,11 +33,11 @@ void init_arc(struct arsc_channel* ac)
 }
 
 // Wait on all syscalls within this async call.  TODO - timeout or something?
-
 int waiton_group_call(async_desc_t* desc, async_rsp_t* rsp)
 {
        syscall_rsp_t syscall_rsp;
        syscall_desc_t* d;
+       int retval = 0;
        int err = 0;
        if (!desc) {
                errno = EINVAL;
@@ -46,14 +46,14 @@ int waiton_group_call(async_desc_t* desc, async_rsp_t* rsp)
 
        while (!(TAILQ_EMPTY(&desc->syslist))) {
                d = TAILQ_FIRST(&desc->syslist);
-               err = waiton_syscall(d, &syscall_rsp);
+               err = waiton_syscall(d);
                // TODO: processing the retval out of rsp here.  might be specific to
                // the async call.  do we want to accumulate?  return any negative
                // values?  depends what we want from the return value, so we might
                // have to pass in a function that is used to do the processing and
                // pass the answer back out in rsp.
                //rsp->retval += syscall_rsp.retval; // For example
-               rsp->retval = MIN(rsp->retval, syscall_rsp.retval);
+               retval = MIN(retval, err);
                // remove from the list and free the syscall desc
                TAILQ_REMOVE(&desc->syslist, d, next);
                POOL_PUT(&syscall_desc_pool, d);
@@ -112,17 +112,22 @@ int get_all_desc(async_desc_t** a_desc, syscall_desc_t** s_desc)
 }
 
 // This runs one syscall instead of a group. 
-int async_syscall(syscall_req_t* req, syscall_desc_t** desc_ptr2)
+
+// TODO: right now there is one channel (remote), in the future, the caller
+// may specify local which will cause it to give up the core to do the work.
+// creation of additional remote channel also allows the caller to prioritize
+// work, because the default policy for the kernel is to roundrobin between them.
+int async_syscall(arsc_channel_t* chan, syscall_req_t* req, syscall_desc_t** desc_ptr2)
 {
        // Note that this assumes one global frontring (TODO)
        // abort if there is no room for our request.  ring size is currently 64.
        // we could spin til it's free, but that could deadlock if this same thread
        // is supposed to consume the requests it is waiting on later.
        syscall_desc_t* desc = malloc(sizeof (syscall_desc_t));
-       desc->channel = &SYS_CHANNEL;
+       desc->channel = chan;
        syscall_front_ring_t *fr = &(desc->channel->sysfr);
        //TODO: can do it locklessly using CAS, but could change with local async calls
-       mcs_lock_lock(&desc->channel->aclock);
+       mcs_lock_lock(&(chan->aclock));
        if (RING_FULL(fr)) {
                errno = EBUSY;
                return -1;
@@ -146,12 +151,42 @@ int async_syscall(syscall_req_t* req, syscall_desc_t** desc_ptr2)
        *desc_ptr2 = desc;
        return 0;
 }
+// Default convinence wrapper before other method of posting calls are available
+
+syscall_desc_t* arc_call(long int num, ...)
+{
+       va_list vl;
+       va_start(vl,num);
+       struct syscall *p_sysc = malloc(sizeof (struct syscall));
+       syscall_desc_t* desc;
+       if (p_sysc == NULL) {
+               errno = ENOMEM;
+               return 0;
+       }
+       p_sysc->num = num;
+       p_sysc->arg0 = va_arg(vl,long int);
+       p_sysc->arg1 = va_arg(vl,long int);
+       p_sysc->arg2 = va_arg(vl,long int);
+       p_sysc->arg3 = va_arg(vl,long int);
+       p_sysc->arg4 = va_arg(vl,long int);
+       p_sysc->arg5 = va_arg(vl,long int);
+       va_end(vl);
+       syscall_req_t arc = {REQ_alloc,NULL, NULL, p_sysc};
+       async_syscall(&SYS_CHANNEL, &arc, &desc);
+       printf ( "%d pushed at %p \n", desc);
+       return desc;
+}
 
 // consider a timeout too
-int waiton_syscall(syscall_desc_t* desc, syscall_rsp_t* rsp)
+// Wait until arsc returns, caller provides rsp buffer.
+// eventually change this to return ret_val, set errno
+
+// What if someone calls waiton the same desc several times?
+int waiton_syscall(syscall_desc_t* desc)
 {
+       int retval = 0;
        if (desc == NULL || desc->channel == NULL){
-               //errno = EFAIL;
+               errno = EFAIL;
                return -1;
        }
        // Make sure we were given a desc with a non-NULL frontring.  This could
@@ -159,25 +194,27 @@ int waiton_syscall(syscall_desc_t* desc, syscall_rsp_t* rsp)
        syscall_front_ring_t *fr =  &desc->channel->sysfr;
        
        if (!fr){
-               //errno = EFAIL;
+               errno = EFAIL;
                return -1;
        }
-       syscall_rsp_t* rsp_inring = RING_GET_RESPONSE(fr, desc->idx);
+       printf ("waiting %d\n", pthread_self()->id);
+       syscall_rsp_t* rsp = RING_GET_RESPONSE(fr, desc->idx);
 
        // ignoring the ring push response from the kernel side now
-       while (rsp_inring->status != RES_ready)
+       while (rsp->sc->flags != SC_DONE)
                cpu_relax();
-       memcpy(rsp, rsp_inring, sizeof(*rsp));
+       // memcpy(rsp, rsp_inring, sizeof(*rsp));
        
-       atomic_inc((atomic_t*) &(fr->rsp_cons));
     // run a cleanup function for this desc, if available
-    //if (rsp->cleanup)
-    // rsp->cleanup(rsp->data);
-       if (rsp->syserr){
-               //      errno = rsp->syserr;
-               return -1;
+    if (rsp->cleanup)
+       rsp->cleanup(rsp->data);
+       if (RSP_ERRNO(rsp)){
+               errno = RSP_ERRNO(rsp);
+               retval = -1;
        } else 
-               return 0;
+               retval =  RSP_RESULT(rsp); 
+       atomic_inc((atomic_t*) &(fr->rsp_cons));
+       return retval;
 }
 
 
index 552c8ca..6edf27a 100644 (file)
@@ -11,6 +11,7 @@
 #include <pool.h>
 #include <assert.h>
 #include <sys/queue.h>
+#include <ros/syscall.h>
 #include <ros/ring_syscall.h>
 #include <mcs.h>
 
@@ -18,7 +19,6 @@ struct arsc_channel {
        mcs_lock_t aclock;
        syscall_sring_t* ring_page;
        syscall_front_ring_t sysfr;
-       // struct channel_ops* ops;
 }; 
 
 typedef struct arsc_channel arsc_channel_t;
@@ -77,18 +77,24 @@ extern async_desc_pool_t async_desc_pool;
 /* Initialize front and back rings of syscall/event ring */
 void init_arc(struct arsc_channel* ac);
 
-int async_syscall(syscall_req_t* req, syscall_desc_t** desc_ptr2);
+int async_syscall(arsc_channel_t* chan, syscall_req_t* req, syscall_desc_t** desc_ptr2);
 
 /* Generic Async Call */
-int waiton_syscall(syscall_desc_t* desc, syscall_rsp_t* rsp);
+int waiton_syscall(syscall_desc_t* desc);
 
 /* Async group call */
+// not sure how to get results back for these?
+
 int waiton_group_call(async_desc_t* desc, async_rsp_t* rsp);
 
 async_desc_t* get_async_desc(void);
 syscall_desc_t* get_sys_desc(async_desc_t* desc);
 int get_all_desc(async_desc_t** a_desc, syscall_desc_t** s_desc);
 
+// helper function to make arc calls
+
+syscall_desc_t* arc_call(long int num, ...);
+
 #ifdef __cplusplus
   }
 #endif