Added ability to declare local per CPU variables
authorDavide Libenzi <dlibenzi@google.com>
Mon, 16 Nov 2015 22:53:23 +0000 (14:53 -0800)
committerBarret Rhoden <brho@cs.berkeley.edu>
Wed, 16 Dec 2015 21:27:06 +0000 (16:27 -0500)
Added ability to declare local per CPU variables.

Signed-off-by: Davide Libenzi <dlibenzi@google.com>
Signed-off-by: Barret Rhoden <brho@cs.berkeley.edu>
kern/include/percpu.h [new file with mode: 0644]
kern/src/Kbuild
kern/src/init.c
kern/src/percpu.c [new file with mode: 0644]

diff --git a/kern/include/percpu.h b/kern/include/percpu.h
new file mode 100644 (file)
index 0000000..2555bc8
--- /dev/null
@@ -0,0 +1,90 @@
+/* Copyright (c) 2015 Google Inc
+ * Davide Libenzi <dlibenzi@google.com>
+ * See LICENSE for details.
+ *
+ * The per CPU utility macros allow file local declaration of per CPU variables.
+ * When a struct my_struct needs to have a per CPU instance, one would declare
+ * something like:
+ *
+ *   static DEFINE_PERCPU(struct my_struct, my_data);
+ *
+ * The per CPU data can then be accessed with either of those forms:
+ *
+ *   struct my_struct *ptr = PERCPU_VARPTR(my_data);
+ *   PERCPU_VAR(my_data).field = 17;
+ *
+ * When the per CPU data has complex initialization, it is possible to register
+ * functions which will be called immediately after the per CPU data is created:
+ *
+ *   DEFINE_PERCPU_INIT(my_init);
+ *
+ * Then the my_init() function would just:
+ *
+ *   static void my_init(void)
+ *   {
+ *       for (int i = 0; i < num_cores; i++) {
+ *           struct my_struct *ptr = _PERCPU_VARPTR(my_data, i);
+ *
+ *           // Initialize ptr data
+ *       }
+ *   }
+ */
+
+#pragma once
+
+#include <sys/types.h>
+#include <arch/topology.h>
+#include <ros/common.h>
+
+#define PERCPU_SECTION __percpu
+#define PERCPU_SECTION_STR STRINGIFY(PERCPU_SECTION)
+
+#define PASTE_THEM(x, y) x ## y
+#define PASTE(x, y) PASTE_THEM(x, y)
+
+#define PERCPU_VARNAME(var) PASTE(__percpu_, var)
+
+#define PERCPU_START_VAR PASTE(__start_, PERCPU_SECTION)
+#define PERCPU_STOP_VAR PASTE(__stop_, PERCPU_SECTION)
+
+#define PERCPU_SIZE (PERCPU_STOP_VAR - PERCPU_START_VAR)
+#define PERCPU_OFFSET(var) ((char *) &(var) - PERCPU_START_VAR)
+
+#define __PERCPU_VARPTR(var, cpu)                                                                              \
+       ({                                                                                                                                      \
+               typeof(var) *__cv;                                                                                              \
+               if (likely(percpu_base))                                                                                \
+                       __cv = (typeof(var) *) (percpu_base + cpu * PERCPU_SIZE +       \
+                                                                       PERCPU_OFFSET(var));                            \
+               else                                                                                                                    \
+                       __cv = &var;                                                                                            \
+               __cv;                                                                                                                   \
+       })
+#define _PERCPU_VARPTR(var, cpu) __PERCPU_VARPTR(PERCPU_VARNAME(var), cpu)
+#define PERCPU_VARPTR(var) __PERCPU_VARPTR(PERCPU_VARNAME(var), core_id())
+
+#define _PERCPU_VAR(var, cpu) (*__PERCPU_VARPTR(PERCPU_VARNAME(var), cpu))
+#define PERCPU_VAR(var) (*__PERCPU_VARPTR(PERCPU_VARNAME(var), core_id()))
+
+#define DEFINE_PERCPU(type, var)                                               \
+       type PERCPU_VARNAME(var) __attribute__ ((section (PERCPU_SECTION_STR)))
+#define DECLARE_PERCPU(type, var)                                                              \
+       extern type PERCPU_VARNAME(var)                                                         \
+               __attribute__ ((section (PERCPU_SECTION_STR)))
+
+#define PERCPU_INIT_SECTION __percpu_init
+#define PERCPU_INIT_SECTION_STR STRINGIFY(PERCPU_INIT_SECTION)
+
+#define PERCPU_INIT_START_VAR PASTE(__start_, PERCPU_INIT_SECTION)
+#define PERCPU_INIT_STOP_VAR PASTE(__stop_, PERCPU_INIT_SECTION)
+
+#define DEFINE_PERCPU_INIT(func)                                                                               \
+       static void func(void);                                                                                         \
+       void (* const PERCPU_VARNAME(func))(void)                                                       \
+               __attribute__ ((section (PERCPU_INIT_SECTION_STR))) = (func)
+
+extern char __attribute__((weak)) PERCPU_START_VAR[];
+extern char __attribute__((weak)) PERCPU_STOP_VAR[];
+extern char *percpu_base;
+
+void percpu_init(void);
index 8e57062..939a914 100644 (file)
@@ -52,6 +52,7 @@ obj-y                                         += ns/
 obj-y                                          += profiler.o
 obj-y                                          += page_alloc.o
 obj-y                                          += pagemap.o
+obj-y                                          += percpu.o
 obj-y                                          += pmap.o
 obj-y                                          += printf.o
 obj-y                                          += printfmt.o
index 5b2e9d2..a6a0553 100644 (file)
@@ -30,6 +30,7 @@
 #include <mm.h>
 #include <frontend.h>
 #include <ex_table.h>
+#include <percpu.h>
 
 #include <arch/init.h>
 #include <bitmask.h>
@@ -146,6 +147,7 @@ void kernel_init(multiboot_info_t *mboot_info)
        colored_page_alloc_init();      // Allocates colors for agnostic processes
        acpiinit();
        topology_init();
+       percpu_init();
        kthread_init();                                 /* might need to tweak when this happens */
        vmr_init();
        file_init();
diff --git a/kern/src/percpu.c b/kern/src/percpu.c
new file mode 100644 (file)
index 0000000..65806ac
--- /dev/null
@@ -0,0 +1,42 @@
+/* Copyright (c) 2015 Google Inc
+ * Davide Libenzi <dlibenzi@google.com>
+ * See LICENSE for details.
+ */
+
+#include <sys/types.h>
+#include <arch/topology.h>
+#include <kmalloc.h>
+#include <stdio.h>
+#include <assert.h>
+#include <string.h>
+#include <percpu.h>
+
+char *percpu_base;
+
+static void run_init_functions(void)
+{
+       extern char __attribute__((weak)) PERCPU_INIT_START_VAR[];
+       extern char __attribute__((weak)) PERCPU_INIT_STOP_VAR[];
+
+       if (PERCPU_INIT_START_VAR) {
+               void (**pfunc)(void) = (void (**)(void)) PERCPU_INIT_START_VAR;
+               void (**pfunc_top)(void) = (void (**)(void)) PERCPU_INIT_STOP_VAR;
+
+               for (; pfunc < pfunc_top; pfunc++)
+                       (*pfunc)();
+       }
+}
+
+void percpu_init(void)
+{
+       assert(num_cores > 0);
+       percpu_base = kmalloc(num_cores * PERCPU_SIZE, 0);
+       assert(percpu_base);
+
+       if (PERCPU_START_VAR) {
+               for (int i = 0; i < num_cores; i++)
+                       memcpy(percpu_base + i * PERCPU_SIZE, PERCPU_START_VAR,
+                                  PERCPU_SIZE);
+       }
+       run_init_functions();
+}