smp_call_function's wait is split out
authorBarret Rhoden <brho@cs.berkeley.edu>
Mon, 27 Apr 2009 00:03:13 +0000 (17:03 -0700)
committerBarret Rhoden <brho@cs.berkeley.edu>
Mon, 27 Apr 2009 00:03:13 +0000 (17:03 -0700)
waiting on an smp_call can be done via a separate function afterwards,
so that you can do other things before waiting.

need to be very careful with this currently - you can easily deadlock
the system if you try to do an smp_call with a wait while having an
outstanding wait.  you could try to grab the same wrapper as the one you
are currently waiting on, and if the front end is clean, you'll spin
trying to commit to the backend checklist.  need to sort this.

kern/smp.c
kern/smp.h
kern/testing.c

index 15c0093..05e2c23 100644 (file)
@@ -216,10 +216,8 @@ void smp_idle(void)
        asm volatile("1: hlt; pause; jmp 1b;");
 }
 
-// could have the backend checklists static/global too.
-// or at least have the pointers global (save a little RAM)
-
-static void smp_call_function(uint8_t type, uint8_t dest, isr_t handler, bool wait)
+static int smp_call_function(uint8_t type, uint8_t dest, isr_t handler, 
+                              handler_wrapper_t** wait_wrapper)
 {
        extern isr_t interrupt_handlers[];
        int8_t state = 0;
@@ -274,11 +272,19 @@ static void smp_call_function(uint8_t type, uint8_t dest, isr_t handler, bool wa
                vector = (vector + 1) % NUM_HANDLER_WRAPPERS;
        }
 
-       // if we want to wait, we set the mask for our backend too
+       // If we want to wait, we set the mask for our backend too.  Wanting to
+       // wait is expressed by having a non-NULL handler_wrapper_t** passed in.
        // no contention here, since this one is protected by the mask being present
-       // in the frontend
-       if (wait)
+       // in the frontend.  the only way someone would try to commit on this would
+       // be if it holds the front.
+       // though someone could be still waiting on this backend (and be holding the
+       // checklist lock), and we would spin here for a while until they are done.
+       // We could consider erroring out and finding a new front-end.
+       if (wait_wrapper) {
                commit_checklist_wait(wrapper->back_cpu_list, &cpu_mask);
+               // pass out our reference to wrapper, so waiting can be done later.
+               *wait_wrapper = wrapper;
+       }
 
        // now register our handler to run
        register_interrupt_handler(interrupt_handlers, wrapper->vector, handler);
@@ -307,28 +313,33 @@ static void smp_call_function(uint8_t type, uint8_t dest, isr_t handler, bool wa
        // wait long enough to receive our own broadcast (PROBABLY WORKS) TODO
        lapic_wait_to_send();
        disable_irqsave(&state);
-
-       // no longer need to wait to protect the vector (the checklist does that)
-       // if we want to wait, we need to wait on the backend checklist
-       if (wait)
-               waiton_checklist(wrapper->back_cpu_list);
+       return 0;
 }
 
 // I'd rather have these functions take an arbitrary function and arguments...
 // Right now, I build a handler that just calls whatever I want, which is
 // another layer of indirection.
-void smp_call_function_self(isr_t handler, bool wait)
+int smp_call_function_self(isr_t handler, handler_wrapper_t** wait_wrapper)
+{
+       return smp_call_function(1, 0, handler, wait_wrapper);
+}
+
+int smp_call_function_all(isr_t handler, handler_wrapper_t** wait_wrapper)
 {
-       smp_call_function(1, 0, handler, wait);
+       return smp_call_function(2, 0, handler, wait_wrapper);
 }
 
-void smp_call_function_all(isr_t handler, bool wait)
+int smp_call_function_single(uint8_t dest, isr_t handler,
+                             handler_wrapper_t** wait_wrapper)
 {
-       smp_call_function(2, 0, handler, wait);
+       return smp_call_function(4, dest, handler, wait_wrapper);
 }
 
-void smp_call_function_single(uint8_t dest, isr_t handler, bool wait)
+// If you want to wait, pass the address of a pointer up above, then call
+// this to do the actual waiting
+int smp_call_wait(handler_wrapper_t* wrapper)
 {
-       smp_call_function(4, dest, handler, wait);
+       waiton_checklist(wrapper->back_cpu_list);
+       return 0;
 }
 
index d2ed1d6..bf3d379 100644 (file)
@@ -31,8 +31,10 @@ void smp_boot(void);
 void smp_idle(void);
 
 /* SMP utility functions */
-void smp_call_function_self(isr_t handler, bool wait);
-void smp_call_function_all(isr_t handler, bool wait);
-void smp_call_function_single(uint8_t dest, isr_t handler, bool wait);
+int smp_call_function_self(isr_t handler, handler_wrapper_t** wait_wrapper);
+int smp_call_function_all(isr_t handler, handler_wrapper_t** wait_wrapper);
+int smp_call_function_single(uint8_t dest, isr_t handler,
+                             handler_wrapper_t** wait_wrapper);
+int smp_call_wait(handler_wrapper_t* wrapper);
 
 #endif /* !ROS_INC_SMP_H */
index efcecc5..71316a7 100644 (file)
@@ -299,19 +299,22 @@ void test_null_handler(struct Trapframe *tf)
 
 void test_smp_call_functions(void)
 {
+       handler_wrapper_t* waiter;
        uint8_t me = lapic_get_id();
        printk("\nCore %d: SMP Call Self (nowait):\n", me);
        printk("---------------------\n");
        smp_call_function_self(test_hello_world_handler, 0);
        printk("\nCore %d: SMP Call Self (wait):\n", me);
        printk("---------------------\n");
-       smp_call_function_self(test_hello_world_handler, 1);
+       smp_call_function_self(test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
        printk("\nCore %d: SMP Call All (nowait):\n", me);
        printk("---------------------\n");
        smp_call_function_all(test_hello_world_handler, 0);
        printk("\nCore %d: SMP Call All (wait):\n", me);
        printk("---------------------\n");
-       smp_call_function_all(test_hello_world_handler, 1);
+       smp_call_function_all(test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
        printk("\nCore %d: SMP Call All-Else Individually, in order (nowait):\n", me);
        printk("---------------------\n");
        smp_call_function_single(1, test_hello_world_handler, 0);
@@ -323,16 +326,24 @@ void test_smp_call_functions(void)
        smp_call_function_single(7, test_hello_world_handler, 0);
        printk("\nCore %d: SMP Call Self (wait):\n", me);
        printk("---------------------\n");
-       smp_call_function_self(test_hello_world_handler, 1);
+       smp_call_function_self(test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
        printk("\nCore %d: SMP Call All-Else Individually, in order (wait):\n", me);
        printk("---------------------\n");
-       smp_call_function_single(1, test_hello_world_handler, 1);
-       smp_call_function_single(2, test_hello_world_handler, 1);
-       smp_call_function_single(3, test_hello_world_handler, 1);
-       smp_call_function_single(4, test_hello_world_handler, 1);
-       smp_call_function_single(5, test_hello_world_handler, 1);
-       smp_call_function_single(6, test_hello_world_handler, 1);
-       smp_call_function_single(7, test_hello_world_handler, 1);
+       smp_call_function_single(1, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(2, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(3, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(4, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(5, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(6, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
+       smp_call_function_single(7, test_hello_world_handler, &waiter);
+       smp_call_wait(waiter);
        printk("\nTesting to see if any IPI-functions are dropped when not waiting:\n");
        printk("A: %d, B: %d, C: %d (should be 0,0,0)\n", a, b, c);
        smp_call_function_all(test_A_incrementer_handler, 0);
@@ -354,7 +365,8 @@ void test_smp_call_functions(void)
        // wait, so we're sure the others finish before printing.
        // without this, we could (and did) get 19,18,19, since the B_inc
        // handler didn't finish yet
-       smp_call_function_self(test_null_handler, 1);
+       smp_call_function_self(test_null_handler, &waiter);
+       smp_call_wait(waiter);
        printk("A: %d, B: %d, C: %d (should be 19,19,19)\n", a, b, c);
        printk("Done\n");
 }