Rename vcore_init() -> vcore_lib_init()
[akaros.git] / user / parlib / include / vcore.h
1 #ifndef PARLIB_VCORE_H
2 #define PARLIB_VCORE_H
3
4 #include <parlib/arch/vcore.h>
5 #include <parlib/arch/atomic.h>
6 #include <sys/param.h>
7 #include <string.h>
8 #include <parlib/timing.h>
9
10 __BEGIN_DECLS
11
12 /*****************************************************************************/
13 /* TODO: This is a complete hack, but necessary for vcore stuff to work for now
14  * The issue is that exit sometimes calls sys_yield(), and we can't recover from
15  * that properly under our vcore model (we shouldn't though).  We really need to
16  * rethink what sys_yield 'should' do when in multicore mode, or else come up 
17  * with a different syscall entirely. */
18 #include <stdlib.h>
19 #include <unistd.h>
20 #undef exit
21 #define exit(status) _exit(status)
22 /*****************************************************************************/
23
24 #define LOG2_MAX_VCORES 6
25 #define MAX_VCORES (1 << LOG2_MAX_VCORES)
26
27 #define TRANSITION_STACK_PAGES 2
28 #define TRANSITION_STACK_SIZE (TRANSITION_STACK_PAGES*PGSIZE)
29
30 /* Defined by glibc; Must be implemented by a user level threading library */
31 extern void vcore_entry();
32 /* Defined in glibc's start.c */
33 extern __thread bool __vcore_context;
34 extern __thread int __vcoreid;
35 /* Defined in vcore.c */
36 extern __thread struct syscall __vcore_one_sysc;        /* see sys_change_vcore */
37
38 /* Vcore API functions */
39 static inline uint32_t max_vcores(void);
40 static inline uint32_t num_vcores(void);
41 static inline int vcore_id(void);
42 static inline bool in_vcore_context(void);
43 static inline bool in_multi_mode(void);
44 static inline void __enable_notifs(uint32_t vcoreid);
45 static inline void __disable_notifs(uint32_t vcoreid);
46 static inline bool notif_is_enabled(uint32_t vcoreid);
47 static inline bool vcore_is_mapped(uint32_t vcoreid);
48 static inline bool vcore_is_preempted(uint32_t vcoreid);
49 static inline struct preempt_data *vcpd_of(uint32_t vcoreid);
50 static inline bool preempt_is_pending(uint32_t vcoreid);
51 static inline bool __preempt_is_pending(uint32_t vcoreid);
52 static inline void *get_vcpd_tls_desc(uint32_t vcoreid);
53 static inline void set_vcpd_tls_desc(uint32_t vcoreid, void *tls_desc);
54 static inline uint64_t vcore_account_resume_nsec(uint32_t vcoreid);
55 static inline uint64_t vcore_account_total_nsec(uint32_t vcoreid);
56 void vcore_lib_init(void);
57 void vcore_event_init(void);
58 void vcore_change_to_m(void);
59 int vcore_request(long nr_new_vcores);
60 void vcore_yield(bool preempt_pending);
61 void vcore_reenter(void (*entry_func)(void));
62 void enable_notifs(uint32_t vcoreid);
63 void disable_notifs(uint32_t vcoreid);
64 void vcore_idle(void);
65 void ensure_vcore_runs(uint32_t vcoreid);
66 void cpu_relax_vc(uint32_t vcoreid);
67 uint32_t get_vcoreid(void);
68 bool check_vcoreid(const char *str, uint32_t vcoreid);
69
70 /* This works so long as we don't dlopen parlib (which we never do) */
71 #define get_tlsvar_linaddr(_vcoreid, _var)                                     \
72 ({                                                                             \
73         uintptr_t vc_tls_desc = (uintptr_t)get_vcpd_tls_desc(_vcoreid);            \
74         uintptr_t var_off = (uintptr_t)&_var - (uintptr_t)get_tls_desc(vcore_id());\
75         (typeof(_var) *)(vc_tls_desc + var_off);                                   \
76 })
77
78 /* Static inlines */
79 static inline uint32_t max_vcores(void)
80 {
81         return MIN(__procinfo.max_vcores, MAX_VCORES);
82 }
83
84 static inline uint32_t num_vcores(void)
85 {
86         return __procinfo.num_vcores;
87 }
88
89 static inline int vcore_id(void)
90 {
91         return __vcoreid;
92 }
93
94 static inline bool in_vcore_context(void)
95 {
96         return __vcore_context;
97 }
98
99 static inline bool in_multi_mode(void)
100 {
101         return __procinfo.is_mcp;
102 }
103
104 /* Only call this if you know what you are doing. */
105 static inline void __enable_notifs(uint32_t vcoreid)
106 {
107         vcpd_of(vcoreid)->notif_disabled = FALSE;
108 }
109
110 static inline void __disable_notifs(uint32_t vcoreid)
111 {
112         vcpd_of(vcoreid)->notif_disabled = TRUE;
113 }
114
115 static inline bool notif_is_enabled(uint32_t vcoreid)
116 {
117         return !vcpd_of(vcoreid)->notif_disabled;
118 }
119
120 static inline bool vcore_is_mapped(uint32_t vcoreid)
121 {
122         return __procinfo.vcoremap[vcoreid].valid;
123 }
124
125 /* We could also check for VC_K_LOCK, but that's a bit much. */
126 static inline bool vcore_is_preempted(uint32_t vcoreid)
127 {
128         struct preempt_data *vcpd = vcpd_of(vcoreid);
129         return atomic_read(&vcpd->flags) & VC_PREEMPTED;
130 }
131
132 static inline struct preempt_data *vcpd_of(uint32_t vcoreid)
133 {
134         return &__procdata.vcore_preempt_data[vcoreid];
135 }
136
137 /* Uthread's can call this in case they care if a preemption is coming.  If a
138  * preempt is incoming, this will return TRUE, if you are in uthread context.  A
139  * reasonable response for a uthread is to yield, and vcore_entry will deal with
140  * the preempt pending.
141  *
142  * If you call this from vcore context, it will do nothing.  In general, it's
143  * not safe to just yield (or do whatever you plan on doing) from arbitrary
144  * places in vcore context.  So we just lie about PP. */
145 static inline bool preempt_is_pending(uint32_t vcoreid)
146 {
147         if (in_vcore_context())
148                 return FALSE;
149         return __preempt_is_pending(vcoreid);
150 }
151
152 static inline bool __preempt_is_pending(uint32_t vcoreid)
153 {
154         return __procinfo.vcoremap[vcoreid].preempt_pending;
155 }
156
157 /* The kernel interface uses uintptr_t, but we have a lot of older code that
158  * uses void *, hence the casting. */
159 static inline void *get_vcpd_tls_desc(uint32_t vcoreid)
160 {
161         return (void*)__procdata.vcore_preempt_data[vcoreid].vcore_tls_desc;
162 }
163
164 static inline void set_vcpd_tls_desc(uint32_t vcoreid, void *tls_desc)
165 {
166         __procdata.vcore_preempt_data[vcoreid].vcore_tls_desc = (uintptr_t)tls_desc;
167 }
168
169 static inline uint64_t vcore_account_resume_ticks(uint32_t vcoreid)
170 {
171         return __procinfo.vcoremap[vcoreid].resume_ticks;
172 }
173
174 static inline uint64_t vcore_account_resume_nsec(uint32_t vcoreid)
175 {
176         return tsc2nsec(vcore_account_resume_ticks(vcoreid));
177 }
178
179 static inline uint64_t vcore_account_total_ticks(uint32_t vcoreid)
180 {
181         return __procinfo.vcoremap[vcoreid].total_ticks;
182 }
183
184 static inline uint64_t vcore_account_total_nsec(uint32_t vcoreid)
185 {
186         return tsc2nsec(vcore_account_total_ticks(vcoreid));
187 }
188
189 static inline uint64_t vcore_account_uptime_ticks(uint32_t vcoreid)
190 {
191         uint64_t resume = __procinfo.vcoremap[vcoreid].resume_ticks; 
192         uint64_t total = __procinfo.vcoremap[vcoreid].total_ticks; 
193         uint64_t now = read_tsc();
194         return now - resume + total;
195 }
196
197 static inline uint64_t vcore_account_uptime_nsec(uint32_t vcoreid)
198 {
199         return tsc2nsec(vcore_account_uptime_ticks(vcoreid));
200 }
201
202 #ifndef __PIC__
203
204 #define begin_safe_access_tls_vars()
205
206 #define end_safe_access_tls_vars()
207
208 #else
209
210 #include <features.h>
211 #if __GNUC_PREREQ(4,4)
212
213 /* These macro acrobatics trick the compiler into not caching the (linear)
214  * address of TLS variables across loads/stores of the TLS descriptor, in lieu
215  * of a "TLS cmb()". */
216 #define begin_safe_access_tls_vars()                                           \
217 {                                                                              \
218         void __attribute__((noinline, optimize("O0")))                             \
219         safe_access_tls_var_internal() {                                           \
220                 asm("");                                                               \
221
222 #define end_safe_access_tls_vars()                                             \
223         } safe_access_tls_var_internal();                                          \
224 }
225
226 #else
227
228 #define begin_safe_access_tls_vars()                                           \
229         printf("ERROR: For PIC use gcc 4.4 or above for tls support!\n");          \
230         printf("ERROR: As a quick fix, recompile your app with -static...\n");     \
231         exit(2);
232
233 #define end_safe_access_tls_vars()                                             \
234         printf("Will never be called because we abort above!");                    \
235         exit(2);
236
237 #endif //__GNUC_PREREQ
238 #endif // __PIC__
239
240 /* Switches into the TLS 'tls_desc'.  Capable of being called from either
241  * uthread or vcore context.  Pairs with end_access_tls_vars(). */
242 #define begin_access_tls_vars(tls_desc)                                        \
243 {                                                                              \
244         struct uthread *caller;                                                    \
245         uint32_t vcoreid;                                                          \
246         void *temp_tls_desc;                                                       \
247         bool invcore = in_vcore_context();                                         \
248         if (!invcore) {                                                            \
249                 caller = current_uthread;                                              \
250                 /* If you have no current_uthread, you might be called too early in the
251                  * process's lifetime.  Make sure something like uthread_slim_init() has
252                  * been run. */                                                        \
253                 assert(caller);                                                        \
254                 /* We need to disable notifs here (in addition to not migrating), since
255                  * we could get interrupted when we're in the other TLS, and when the
256                  * vcore restarts us, it will put us in our old TLS, not the one we were
257                  * in when we were interrupted.  We need to not migrate, since once we
258                  * know the vcoreid, we depend on being on the same vcore throughout.*/\
259                 caller->flags |= UTHREAD_DONT_MIGRATE;                                 \
260                 /* Not concerned about cross-core memory ordering, so no CPU mbs needed.
261                  * The cmb is to prevent the compiler from issuing the vcore read before
262                  * the DONT_MIGRATE write. */                                          \
263                 cmb();                                                                 \
264                 vcoreid = vcore_id();                                                  \
265                 disable_notifs(vcoreid);                                               \
266         } else { /* vcore context */                                               \
267                 vcoreid = vcore_id();                                                  \
268         }                                                                          \
269         temp_tls_desc = get_tls_desc(vcoreid);                                     \
270         set_tls_desc(tls_desc, vcoreid);                                           \
271         begin_safe_access_tls_vars();
272
273 #define end_access_tls_vars()                                                  \
274         end_safe_access_tls_vars();                                                \
275         set_tls_desc(temp_tls_desc, vcoreid);                                      \
276         if (!invcore) {                                                            \
277                 /* Note we reenable migration before enabling notifs, which is reverse
278                  * from how we disabled notifs.  We must enabling migration before
279                  * enabling notifs.  See 6c7fb12 and 5e4825eb4 for details. */         \
280                 caller->flags &= ~UTHREAD_DONT_MIGRATE;                                \
281                 cmb();  /* turn off DONT_MIGRATE before enabling notifs */             \
282                 if (in_multi_mode())                                                   \
283                         enable_notifs(vcoreid);                                            \
284         }                                                                          \
285 }
286
287 #define safe_set_tls_var(name, val)                                            \
288 ({                                                                             \
289         begin_safe_access_tls_vars();                                              \
290         name = val;                                                                \
291         end_safe_access_tls_vars();                                                \
292 })
293
294 #define safe_get_tls_var(name)                                                 \
295 ({                                                                             \
296         typeof(name) __val;                                                        \
297         begin_safe_access_tls_vars();                                              \
298         __val = name;                                                              \
299         end_safe_access_tls_vars();                                                \
300         __val;                                                                     \
301 })
302
303 #define vcore_set_tls_var(name, val)                                           \
304 ({                                                                             \
305         typeof(val) __val = val;                                                   \
306         begin_access_tls_vars(get_vcpd_tls_desc(vcoreid));                         \
307         name = __val;                                                              \
308         end_access_tls_vars();                                                     \
309 })
310
311 #define vcore_get_tls_var(name)                                                \
312 ({                                                                             \
313         typeof(name) val;                                                          \
314         begin_access_tls_vars(get_vcpd_tls_desc(vcoreid));                         \
315         val = name;                                                                \
316         end_access_tls_vars();                                                     \
317         val;                                                                       \
318 })
319
320 __END_DECLS
321
322 #endif /* PARLIB_VCORE_H */