More addition to arsc infrastructure.
authorDavid Zhu <yuzhu@cs.berkeley.edu>
Thu, 28 Oct 2010 21:34:23 +0000 (14:34 -0700)
committerKevin Klues <klueska@cs.berkeley.edu>
Thu, 3 Nov 2011 00:36:01 +0000 (17:36 -0700)
Specifically, decoupling arsc call page from procdata.
Create an interface that may be used for proc-to-proc call.
Preparing arsc for multi-threaded calls and reordering of request/response.

12 files changed:
kern/arch/i686/atomic.h
kern/include/arsc_server.h
kern/include/ros/procdata.h
kern/include/ros/ring_syscall.h
kern/src/arsc.c
kern/src/mm.c
kern/src/process.c
tests/arsc_test.c
user/parlib/asynccall.c
user/parlib/include/arc.h
user/parlib/include/parlib.h
user/parlib/syscall.c

index a00d927..fd80173 100644 (file)
@@ -5,7 +5,6 @@
 #include <ros/arch/membar.h>
 #include <arch/x86.h>
 #include <arch/arch.h>
-
 typedef void * RACY atomic_t;
 struct spinlock {
        volatile uint32_t RACY rlock;
@@ -14,7 +13,7 @@ struct spinlock {
        uint32_t calling_core;
 #endif
 };
-typedef struct spinlock RACY spinlock_t;
+typedef struct spinlock spinlock_t;
 #define SPINLOCK_INITIALIZER {0}
 
 static inline void atomic_init(atomic_t *number, int32_t val);
index 72c52fa..b8afedd 100644 (file)
@@ -17,7 +17,7 @@
 extern struct proc_list arsc_proc_list;
 extern spinlock_t arsc_proc_lock;
 
-intreg_t sys_init_arsc(struct proc* p);
+syscall_sring_t* sys_init_arsc(struct proc* p);
 intreg_t syscall_async(struct proc* p, syscall_req_t *syscall);
 void arsc_server(trapframe_t *tf);
 
index de898b2..4c61c6f 100644 (file)
 #include <ros/event.h>
 
 typedef struct procdata {
+       /*
        syscall_sring_t                 syscallring;
        char                                    pad1[SYSCALLRINGSIZE - sizeof(syscall_sring_t)];
+       */
+       syscall_sring_t                 *syscallring;
        sysevent_sring_t                syseventring;
        char                                    pad2[SYSEVENTRINGSIZE - sizeof(sysevent_sring_t)];
 #ifdef __i386__
index 4eb6856..336104d 100644 (file)
@@ -7,21 +7,36 @@
 #define NUM_SYSCALL_ARGS 6
 /* This will need to change to represent sending pointers to syscalls, not the
  * syscalls themselves */
+
+typedef enum {
+       RES_free,  // The response has been digested by the user space, can be reallocated
+       REQ_alloc, // Space fo request is allocated
+       REQ_ready, // The request is populated by the caller
+       REQ_processing, // The request is being processed, 
+                                       // or a kernel thread is going to pick up the stack to process this later.
+
+       RES_ready // The response is ready to be picked up
+} syscall_status_t;
 typedef struct syscall_req {
-        uint32_t num;
-        uint32_t flags;
-        uint32_t args[NUM_SYSCALL_ARGS];
+    syscall_status_t status;
+       void (*cleanup)(void* data);
+       void *data;
+    uint32_t num;
+    uint32_t args[NUM_SYSCALL_ARGS];
 } syscall_req_t;
 
 typedef struct syscall_rsp {
-        uint32_t retval;
-               uint32_t syserr;
+       syscall_status_t status;
+       void (*cleanup)(void* data);
+       void *data;
+       uint32_t retval;
+       uint32_t syserr;
 } syscall_rsp_t;
 
+
+
 // Generic Syscall Ring Buffer
 #define SYSCALLRINGSIZE    PGSIZE
-//DEFINE_RING_TYPES_WITH_SIZE(syscall, syscall_req_t, syscall_rsp_t,
-//SYSCALLRINGSIZE);
 DEFINE_RING_TYPES(syscall, syscall_req_t, syscall_rsp_t);
 
 #endif
index 6fed271..0d6a684 100644 (file)
@@ -32,13 +32,30 @@ intreg_t inline syscall_async(struct proc *p, syscall_req_t *call)
                       call->args[2], call->args[3], call->args[4], call->args[5]);
 }
 
-intreg_t sys_init_arsc(struct proc *p)
+syscall_sring_t* sys_init_arsc(struct proc *p)
 {
-       proc_incref(p, 1);              /* we're storing an external ref here */
+       kref_get(&p->kref, 1);          /* we're storing an external ref here */
+       syscall_sring_t* sring;
+       void * va;
+       // TODO: need to pin this page in the future when swapping happens
+       va = do_mmap(p,MMAP_LOWEST_VA, SYSCALLRINGSIZE, PROT_READ|PROT_WRITE, MAP_ANON|MAP_POPULATE, NULL, 0);
+       pte_t *pte = pgdir_walk(p->env_pgdir, (void*)va, 0);
+       assert(pte);
+       sring = (syscall_sring_t*) (ppn2kva(PTE2PPN(*pte)));
+       /*make sure we are able to allocate the shared ring */
+       assert(sring != NULL);
+       p->procdata->syscallring = sring;
+       /* Initialize the generic syscall ring buffer */
+       SHARED_RING_INIT(sring);
+
+       BACK_RING_INIT(&p->syscallbackring,
+                      sring,
+                      SYSCALLRINGSIZE);
+
        spin_lock_irqsave(&arsc_proc_lock);
        TAILQ_INSERT_TAIL(&arsc_proc_list, p, proc_arsc_link);
        spin_unlock_irqsave(&arsc_proc_lock);
-       return ESUCCESS;
+       return (syscall_sring_t*)va;
 }
 
 void arsc_server(struct trapframe *tf)
@@ -48,7 +65,7 @@ void arsc_server(struct trapframe *tf)
        while (1) {
                while (TAILQ_EMPTY(&arsc_proc_list))
                        cpu_relax();
-
+               
                TAILQ_FOREACH(p, &arsc_proc_list, proc_link) {
                        /* Probably want to try to process a dying process's syscalls.  If
                         * not, just move it to an else case */
@@ -72,7 +89,7 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
        // looking at a process not initialized to perform arsc. 
        if (sysbr == NULL) 
                return count;
-
+       
        // max is the most we'll process.  max = 0 means do as many as possible
        // TODO: check for initialization of the ring. 
        while (RING_HAS_UNCONSUMED_REQUESTS(sysbr) && ((!max)||(count < max)) ) {
@@ -90,7 +107,7 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
                // going to explicitly fill in all fields
                syscall_rsp_t rsp;
                // this assumes we get our answer immediately for the syscall.
-               syscall_req_t* req = RING_GET_REQUEST(sysbr, ++(sysbr->req_cons));
+               syscall_req_t* req = RING_GET_REQUEST(sysbr, ++sysbr->req_cons);
                // print req
                printd("req no %d, req arg %c\n", req->num, *((char*)req->args[0]));
                
@@ -102,11 +119,14 @@ static intreg_t process_generic_syscalls(struct proc *p, size_t max)
                
                rsp.retval = syscall_async(p, req);
                rsp.syserr = sysc.err;
+               rsp.cleanup = req->cleanup;
+               rsp.data = req->data;
+               rsp.status = RES_ready;
                // write response into the slot it came from
                memcpy(req, &rsp, sizeof(syscall_rsp_t));
-               // update our counter for what we've produced (assumes we went in order!)
                (sysbr->rsp_prod_pvt)++;
                RING_PUSH_RESPONSES(sysbr);
+
                //printk("DEBUG POST: sring->req_prod: %d, sring->rsp_prod: %d\n",
                //         sysbr->sring->req_prod, sysbr->sring->rsp_prod);
        }
index efb62f9..ba4cd26 100644 (file)
@@ -48,7 +48,6 @@ struct vm_region *create_vmr(struct proc *p, uintptr_t va, size_t len)
        assert(!PGOFF(va));
        assert(!PGOFF(len));
        assert(va + len <= UMAPTOP);
-
        /* Is there room before the first one: */
        vm_i = TAILQ_FIRST(&p->vm_regions);
        /* This works for now, but if all we have is BRK_END ones, we'll start
index 07afab8..292e453 100644 (file)
@@ -304,13 +304,6 @@ error_t proc_alloc(struct proc **pp, struct proc *parent)
        proc_init_procinfo(p);
        /* Initialize the contents of the e->procdata structure */
 
-       /* Initialize the generic syscall ring buffer */
-       SHARED_RING_INIT(&p->procdata->syscallring);
-       /* Initialize the backend of the syscall ring buffer */
-       BACK_RING_INIT(&p->syscallbackring,
-                      &p->procdata->syscallring,
-                      SYSCALLRINGSIZE);
-
        /* Initialize the generic sysevent ring buffer */
        SHARED_RING_INIT(&p->procdata->syseventring);
        /* Initialize the frontend of the sysevent ring buffer */
index bcb771e..7d32b54 100644 (file)
@@ -4,27 +4,32 @@
 #include <arc.h>
 #include <stdio.h>
 
-int sys_cputs_async(const char *s, size_t len, syscall_desc_t* desc,                                              
+syscall_desc_t* sys_cputs_async(const char *s, size_t len,                                             
                      void (*cleanup_handler)(void*), void* cleanup_data)
 {                                                                                                                     
     // could just hardcode 4 0's, will eventually wrap this marshaller anyway                                         
-    syscall_req_t syscall = {SYS_cputs, 0, {(uint32_t)s, len, [2 ... (NUM_SYSCALL_ARGS-1)] 0} };                          
-    desc->cleanup = cleanup_handler;                                                                                  
-    desc->data = cleanup_data;                                                                                        
-    return async_syscall(&syscall, desc);                                                                             
-}     
+       syscall_desc_t* desc;
+    syscall_req_t syscall = {REQ_alloc, cleanup_handler, cleanup_data,
+                                                       SYS_cputs,{(uint32_t)s, len, [2 ... (NUM_SYSCALL_ARGS-1)] 0} };                          
+    syscall.cleanup = cleanup_handler;                                                                                  
+    syscall.data = cleanup_data;                                                                                        
+    async_syscall(&syscall, &desc);
+       return desc;
+}
+
 int main(int argc, char** argv){
        int pid = sys_getpid();
        char testme = 't';
        printf ("single thread - init arsc \n");
-       init_arc();
-       async_desc_t desc1;
-       async_rsp_t rsp1;
+       syscall_desc_t* sysdesc;
        syscall_rsp_t sysrsp;
+       init_arc(&SYS_CHANNEL);
 
-       syscall_desc_t* sysdesc = get_sys_desc (&desc1);
+       printf ("single thread - init complete \n");
        // cprintf_async(&desc1, "Cross-Core call 1, coming from process %08x\n", pid);
-       sys_cputs_async(&testme, 1, sysdesc, NULL, NULL);
-       waiton_syscall(sysdesc, &sysrsp);
+       sysdesc = sys_cputs_async(&testme, 1, NULL, NULL);
+
+       printf ("single thread - call placed \n");
+       waiton_syscall(sysdesc, &sysrsp);       
        printf ("single thread - dummy call \n");       
 }
index 4fdc71e..3392aa6 100644 (file)
@@ -9,28 +9,29 @@
 #include <arch/arch.h>
 #include <sys/param.h>
 
-syscall_front_ring_t syscallfrontring;
-sysevent_back_ring_t syseventbackring;
 syscall_desc_pool_t syscall_desc_pool;
 async_desc_pool_t async_desc_pool;
 async_desc_t* current_async_desc;
 
-// use globals for now
-void init_arc()
+struct arsc_channel global_ac;
+
+void init_arc(struct arsc_channel* ac)
 {
        // Set up the front ring for the general syscall ring
        // and the back ring for the general sysevent ring
-       // TODO: Reorganize these global variables
-       FRONT_RING_INIT(&syscallfrontring, &(__procdata.syscallring), SYSCALLRINGSIZE);
-       BACK_RING_INIT(&syseventbackring, &(__procdata.syseventring), SYSEVENTRINGSIZE);
+       mcs_lock_init(&ac->aclock);
+       ac->ring_page = (syscall_sring_t*)sys_init_arsc();
+
+       FRONT_RING_INIT(&ac->sysfr, ac->ring_page, SYSCALLRINGSIZE);
+       //BACK_RING_INIT(&syseventbackring, &(__procdata.syseventring), SYSEVENTRINGSIZE);
+       //TODO: eventually rethink about desc pools, they are here but no longer necessary
        POOL_INIT(&syscall_desc_pool, MAX_SYSCALLS);
        POOL_INIT(&async_desc_pool, MAX_ASYNCCALLS);
-       sys_init_arsc();
-
-
 }
+
 // Wait on all syscalls within this async call.  TODO - timeout or something?
-int waiton_async_call(async_desc_t* desc, async_rsp_t* rsp)
+
+int waiton_group_call(async_desc_t* desc, async_rsp_t* rsp)
 {
        syscall_rsp_t syscall_rsp;
        syscall_desc_t* d;
@@ -108,27 +109,36 @@ int get_all_desc(async_desc_t** a_desc, syscall_desc_t** s_desc)
 }
 
 // This runs one syscall instead of a group. 
-int async_syscall(syscall_req_t* req, syscall_desc_t* desc)
+int async_syscall(syscall_req_t* req, syscall_desc_t** desc_ptr2)
 {
        // Note that this assumes one global frontring (TODO)
        // abort if there is no room for our request.  ring size is currently 64.
        // we could spin til it's free, but that could deadlock if this same thread
        // is supposed to consume the requests it is waiting on later.
-       if (RING_FULL(&syscallfrontring)) {
+       syscall_desc_t* desc = *desc_ptr2 = malloc(sizeof (syscall_desc_t));
+       desc->channel = &SYS_CHANNEL;
+       syscall_front_ring_t *fr = &(desc->channel->sysfr);
+       mcs_lock_lock(&desc->channel->aclock);
+       if (RING_FULL(fr)) {
                errno = EBUSY;
                return -1;
        }
        // req_prod_pvt comes in as the previously produced item.  need to
        // increment to the next available spot, which is the one we'll work on.
        // at some point, we need to listen for the responses.
-       desc->idx = ++(syscallfrontring.req_prod_pvt);
-       desc->sysfr = &syscallfrontring;
-       syscall_req_t* r = RING_GET_REQUEST(&syscallfrontring, desc->idx);
+       desc->idx = ++(fr->req_prod_pvt);
+       syscall_req_t* r = RING_GET_REQUEST(fr, desc->idx);
+       assert (r->status == RES_free);
+       req->status = REQ_alloc;
+       
        memcpy(r, req, sizeof(syscall_req_t));
+       r->status = REQ_ready;
        // push our updates to syscallfrontring.req_prod_pvt
-       RING_PUSH_REQUESTS(&syscallfrontring);
+       // note: it is ok to push without protection since it is atomic and kernel
+       // won't process any requests until they are marked REQ_ready (also atomic)
+       RING_PUSH_REQUESTS(fr);
        //cprintf("DEBUG: sring->req_prod: %d, sring->rsp_prod: %d\n", 
-       //   syscallfrontring.sring->req_prod, syscallfrontring.sring->rsp_prod);
+       mcs_lock_unlock(&desc->channel->aclock);
        return 0;
 }
 
@@ -137,22 +147,24 @@ int waiton_syscall(syscall_desc_t* desc, syscall_rsp_t* rsp)
 {
        // Make sure we were given a desc with a non-NULL frontring.  This could
        // happen if someone forgot to check the error code on the paired syscall.
-       if (!desc->sysfr){
+       syscall_front_ring_t *fr =  &desc->channel->sysfr;
+       
+       if (!fr){
                errno = EFAIL;
                return -1;
        }
        // this forces us to call wait in the order in which the syscalls are made.
-       if (desc->idx != desc->sysfr->rsp_cons + 1){
+       if (desc->idx != fr->rsp_cons + 1){
                errno = EDEADLOCK;
                return -1;
        }
-       while (!(RING_HAS_UNCONSUMED_RESPONSES(desc->sysfr)))
+       while (!(RING_HAS_UNCONSUMED_RESPONSES(fr)))
                cpu_relax();
-       memcpy(rsp, RING_GET_RESPONSE(desc->sysfr, desc->idx), sizeof(*rsp));
-       desc->sysfr->rsp_cons++;
+       memcpy(rsp, RING_GET_RESPONSE(fr, desc->idx), sizeof(*rsp));
+       fr->rsp_cons++;
     // run a cleanup function for this desc, if available
-    if (desc->cleanup)
-       desc->cleanup(desc->data);
+    if (rsp->cleanup)
+       rsp->cleanup(rsp->data);
        if (rsp->syserr){
                errno = rsp->syserr;
                return -1;
index 772f5ac..552c8ca 100644 (file)
 #include <assert.h>
 #include <sys/queue.h>
 #include <ros/ring_syscall.h>
+#include <mcs.h>
 
+struct arsc_channel {
+       mcs_lock_t aclock;
+       syscall_sring_t* ring_page;
+       syscall_front_ring_t sysfr;
+       // struct channel_ops* ops;
+}; 
+
+typedef struct arsc_channel arsc_channel_t;
+
+#define SYS_CHANNEL (global_ac)
+extern arsc_channel_t global_ac;
 extern syscall_front_ring_t syscallfrontring;
 extern sysevent_back_ring_t syseventbackring;
 
@@ -24,11 +36,8 @@ extern sysevent_back_ring_t syseventbackring;
 typedef struct syscall_desc syscall_desc_t;
 struct syscall_desc {
        TAILQ_ENTRY(syscall_desc) next;
-       syscall_front_ring_t* sysfr;
+       struct arsc_channel* channel;
        uint32_t idx;
-       // cleanup
-       void (*cleanup)(void* data);
-       void* data;
 };
 TAILQ_HEAD(syscall_desc_list, syscall_desc);
 typedef struct syscall_desc_list syscall_desc_list_t;
@@ -66,15 +75,15 @@ extern syscall_desc_pool_t syscall_desc_pool;
 extern async_desc_pool_t async_desc_pool;
 
 /* Initialize front and back rings of syscall/event ring */
-void init_arc();
+void init_arc(struct arsc_channel* ac);
+
+int async_syscall(syscall_req_t* req, syscall_desc_t** desc_ptr2);
 
 /* Generic Async Call */
 int waiton_syscall(syscall_desc_t* desc, syscall_rsp_t* rsp);
 
 /* Async group call */
-int waiton_async_call(async_desc_t* desc, async_rsp_t* rsp);
-
-int async_syscall(syscall_req_t* req, syscall_desc_t* desc);
+int waiton_group_call(async_desc_t* desc, async_rsp_t* rsp);
 
 async_desc_t* get_async_desc(void);
 syscall_desc_t* get_sys_desc(async_desc_t* desc);
index 72b4290..63cd55b 100644 (file)
@@ -48,9 +48,8 @@ int         sys_notify(int pid, unsigned int ev_type, struct event_msg *u_msg);
 int         sys_self_notify(uint32_t vcoreid, unsigned int ev_type,
                             struct event_msg *u_msg);
 int         sys_halt_core(unsigned int usec);
+void*          sys_init_arsc();
 
-/* ARSC */
-int                    sys_init_arsc();
 
 #endif // !ASSEMBLER
 
index 6004fd0..941fc21 100644 (file)
@@ -128,7 +128,8 @@ int sys_halt_core(unsigned int usec)
        return ros_syscall(SYS_halt_core, usec, 0, 0, 0, 0, 0);
 }
 
-int sys_init_arsc()
+void* sys_init_arsc()
 {
-       return ros_syscall(SYS_init_arsc, 0, 0, 0, 0, 0, 0);
+       return (void*)ros_syscall(SYS_init_arsc, 0, 0, 0, 0, 0, 0);
+       return (void*)ros_syscall(SYS_init_arsc, 0, 0, 0, 0, 0);
 }