Remove MAX_VCORES
[akaros.git] / user / parlib / include / parlib / vcore.h
1 #pragma once
2
3 #include <parlib/arch/vcore.h>
4 #include <parlib/arch/atomic.h>
5 #include <sys/param.h>
6 #include <string.h>
7 #include <parlib/timing.h>
8 #include <parlib/common.h>
9
10 __BEGIN_DECLS
11
12 /*****************************************************************************/
13 /* TODO: This is a complete hack, but necessary for vcore stuff to work for now
14  * The issue is that exit sometimes calls sys_yield(), and we can't recover from
15  * that properly under our vcore model (we shouldn't though).  We really need to
16  * rethink what sys_yield 'should' do when in multicore mode, or else come up 
17  * with a different syscall entirely. */
18 #undef exit
19 extern void _exit (int status);
20 extern void exit (int __status) __THROW __attribute__ ((__noreturn__));
21 #define exit(status) _exit(status)
22 /*****************************************************************************/
23
24 #define TRANSITION_STACK_PAGES 2
25 #define TRANSITION_STACK_SIZE (TRANSITION_STACK_PAGES*PGSIZE)
26
27 /* Defined in vcore.c */
28 extern void vcore_entry();
29 extern __thread bool __vcore_context;
30 extern __thread int __vcoreid;
31 extern __thread struct syscall __vcore_one_sysc;        /* see sys_change_vcore */
32
33 /* Vcore API functions */
34 static inline uint32_t max_vcores(void);
35 static inline uint32_t num_vcores(void);
36 static inline int vcore_id(void);
37 static inline bool in_vcore_context(void);
38 static inline bool in_multi_mode(void);
39 static inline void __enable_notifs(uint32_t vcoreid);
40 static inline void __disable_notifs(uint32_t vcoreid);
41 static inline bool notif_is_enabled(uint32_t vcoreid);
42 static inline bool vcore_is_mapped(uint32_t vcoreid);
43 static inline bool vcore_is_preempted(uint32_t vcoreid);
44 static inline struct preempt_data *vcpd_of(uint32_t vcoreid);
45 static inline bool preempt_is_pending(uint32_t vcoreid);
46 static inline bool __preempt_is_pending(uint32_t vcoreid);
47 static inline void *get_vcpd_tls_desc(uint32_t vcoreid);
48 static inline void set_vcpd_tls_desc(uint32_t vcoreid, void *tls_desc);
49 static inline uint64_t vcore_account_resume_nsec(uint32_t vcoreid);
50 static inline uint64_t vcore_account_total_nsec(uint32_t vcoreid);
51 void vcore_lib_init(void);
52 void vcore_change_to_m(void);
53 int vcore_request(long nr_new_vcores);
54 void vcore_yield(bool preempt_pending);
55 void vcore_reenter(void (*entry_func)(void));
56 void enable_notifs(uint32_t vcoreid);
57 void disable_notifs(uint32_t vcoreid);
58 void vcore_idle(void);
59 void ensure_vcore_runs(uint32_t vcoreid);
60 void cpu_relax_vc(uint32_t vcoreid);
61 uint32_t get_vcoreid(void);
62 bool check_vcoreid(const char *str, uint32_t vcoreid);
63 void print_hw_tf(struct hw_trapframe *tf);
64 void print_sw_tf(struct sw_trapframe *sw_tf);
65 void print_user_context(struct user_context *ctx);
66
67 /* This works so long as we don't dlopen parlib (which we never do) */
68 #define get_tlsvar_linaddr(_vcoreid, _var)                                     \
69 ({                                                                             \
70         uintptr_t vc_tls_desc = (uintptr_t)get_vcpd_tls_desc(_vcoreid);            \
71         uintptr_t var_off = (uintptr_t)&_var - (uintptr_t)get_tls_desc();          \
72         (typeof(_var) *)(vc_tls_desc + var_off);                                   \
73 })
74
75 /* Static inlines */
76 static inline uint32_t max_vcores(void)
77 {
78         return MAX(1, __procinfo.max_vcores);
79 }
80
81 static inline uint32_t num_vcores(void)
82 {
83         return __procinfo.num_vcores;
84 }
85
86 static inline int vcore_id(void)
87 {
88         return __vcoreid;
89 }
90
91 static inline bool in_vcore_context(void)
92 {
93         return __vcore_context;
94 }
95
96 static inline bool in_multi_mode(void)
97 {
98         return __procinfo.is_mcp;
99 }
100
101 /* Only call this if you know what you are doing. */
102 static inline void __enable_notifs(uint32_t vcoreid)
103 {
104         vcpd_of(vcoreid)->notif_disabled = FALSE;
105 }
106
107 static inline void __disable_notifs(uint32_t vcoreid)
108 {
109         vcpd_of(vcoreid)->notif_disabled = TRUE;
110 }
111
112 static inline bool notif_is_enabled(uint32_t vcoreid)
113 {
114         return !vcpd_of(vcoreid)->notif_disabled;
115 }
116
117 static inline bool vcore_is_mapped(uint32_t vcoreid)
118 {
119         return __procinfo.vcoremap[vcoreid].valid;
120 }
121
122 /* We could also check for VC_K_LOCK, but that's a bit much. */
123 static inline bool vcore_is_preempted(uint32_t vcoreid)
124 {
125         struct preempt_data *vcpd = vcpd_of(vcoreid);
126         return atomic_read(&vcpd->flags) & VC_PREEMPTED;
127 }
128
129 static inline struct preempt_data *vcpd_of(uint32_t vcoreid)
130 {
131         return &__procdata.vcore_preempt_data[vcoreid];
132 }
133
134 /* Uthread's can call this in case they care if a preemption is coming.  If a
135  * preempt is incoming, this will return TRUE, if you are in uthread context.  A
136  * reasonable response for a uthread is to yield, and vcore_entry will deal with
137  * the preempt pending.
138  *
139  * If you call this from vcore context, it will do nothing.  In general, it's
140  * not safe to just yield (or do whatever you plan on doing) from arbitrary
141  * places in vcore context.  So we just lie about PP. */
142 static inline bool preempt_is_pending(uint32_t vcoreid)
143 {
144         if (in_vcore_context())
145                 return FALSE;
146         return __preempt_is_pending(vcoreid);
147 }
148
149 static inline bool __preempt_is_pending(uint32_t vcoreid)
150 {
151         return __procinfo.vcoremap[vcoreid].preempt_pending;
152 }
153
154 /* The kernel interface uses uintptr_t, but we have a lot of older code that
155  * uses void *, hence the casting. */
156 static inline void *get_vcpd_tls_desc(uint32_t vcoreid)
157 {
158         return (void*)__procdata.vcore_preempt_data[vcoreid].vcore_tls_desc;
159 }
160
161 static inline void set_vcpd_tls_desc(uint32_t vcoreid, void *tls_desc)
162 {
163         __procdata.vcore_preempt_data[vcoreid].vcore_tls_desc = (uintptr_t)tls_desc;
164 }
165
166 static inline uint64_t vcore_account_resume_ticks(uint32_t vcoreid)
167 {
168         return __procinfo.vcoremap[vcoreid].resume_ticks;
169 }
170
171 static inline uint64_t vcore_account_resume_nsec(uint32_t vcoreid)
172 {
173         return tsc2nsec(vcore_account_resume_ticks(vcoreid));
174 }
175
176 static inline uint64_t vcore_account_total_ticks(uint32_t vcoreid)
177 {
178         return __procinfo.vcoremap[vcoreid].total_ticks;
179 }
180
181 static inline uint64_t vcore_account_total_nsec(uint32_t vcoreid)
182 {
183         return tsc2nsec(vcore_account_total_ticks(vcoreid));
184 }
185
186 static inline uint64_t vcore_account_uptime_ticks(uint32_t vcoreid)
187 {
188         uint64_t resume = __procinfo.vcoremap[vcoreid].resume_ticks; 
189         uint64_t total = __procinfo.vcoremap[vcoreid].total_ticks; 
190         uint64_t now = read_tsc();
191         return now - resume + total;
192 }
193
194 static inline uint64_t vcore_account_uptime_nsec(uint32_t vcoreid)
195 {
196         return tsc2nsec(vcore_account_uptime_ticks(vcoreid));
197 }
198
199 #ifndef __PIC__
200
201 #define begin_safe_access_tls_vars()
202
203 #define end_safe_access_tls_vars()
204
205 #else
206
207 #include <features.h>
208
209 /* These macro acrobatics trick the compiler into not caching the (linear)
210  * address of TLS variables across loads/stores of the TLS descriptor, in lieu
211  * of a "TLS cmb()". */
212 #define begin_safe_access_tls_vars()                                           \
213 {                                                                              \
214         void __attribute__((noinline, optimize("O0")))                             \
215         safe_access_tls_var_internal() {                                           \
216                 asm("");                                                               \
217
218 #define end_safe_access_tls_vars()                                             \
219         } safe_access_tls_var_internal();                                          \
220 }
221
222 #endif // __PIC__
223
224 /* Switches into the TLS 'tls_desc'.  Capable of being called from either
225  * uthread or vcore context.  Pairs with end_access_tls_vars(). */
226 #define begin_access_tls_vars(tls_desc)                                        \
227 {                                                                              \
228         struct uthread *caller;                                                    \
229         uint32_t vcoreid;                                                          \
230         void *temp_tls_desc;                                                       \
231         bool invcore = in_vcore_context();                                         \
232         if (!invcore) {                                                            \
233                 caller = current_uthread;                                              \
234                 /* If you have no current_uthread, you might be called too early in the
235                  * process's lifetime.  Make sure something like uthread_slim_init() has
236                  * been run. */                                                        \
237                 assert(caller);                                                        \
238                 /* We need to disable notifs here (in addition to not migrating), since
239                  * we could get interrupted when we're in the other TLS, and when the
240                  * vcore restarts us, it will put us in our old TLS, not the one we were
241                  * in when we were interrupted.  We need to not migrate, since once we
242                  * know the vcoreid, we depend on being on the same vcore throughout.*/\
243                 caller->flags |= UTHREAD_DONT_MIGRATE;                                 \
244                 /* Not concerned about cross-core memory ordering, so no CPU mbs needed.
245                  * The cmb is to prevent the compiler from issuing the vcore read before
246                  * the DONT_MIGRATE write. */                                          \
247                 cmb();                                                                 \
248                 vcoreid = vcore_id();                                                  \
249                 disable_notifs(vcoreid);                                               \
250         } else { /* vcore context */                                               \
251                 vcoreid = vcore_id();                                                  \
252         }                                                                          \
253         temp_tls_desc = get_tls_desc();                                            \
254         set_tls_desc(tls_desc);                                                    \
255         begin_safe_access_tls_vars();
256
257 #define end_access_tls_vars()                                                  \
258         end_safe_access_tls_vars();                                                \
259         set_tls_desc(temp_tls_desc);                                               \
260         if (!invcore) {                                                            \
261                 /* Note we reenable migration before enabling notifs, which is reverse
262                  * from how we disabled notifs.  We must enabling migration before
263                  * enabling notifs.  See 6c7fb12 and 5e4825eb4 for details. */         \
264                 caller->flags &= ~UTHREAD_DONT_MIGRATE;                                \
265                 cmb();  /* turn off DONT_MIGRATE before enabling notifs */             \
266                 enable_notifs(vcoreid);                                                \
267         }                                                                          \
268 }
269
270 #define safe_set_tls_var(name, val)                                            \
271 ({                                                                             \
272         begin_safe_access_tls_vars();                                              \
273         name = val;                                                                \
274         end_safe_access_tls_vars();                                                \
275 })
276
277 #define safe_get_tls_var(name)                                                 \
278 ({                                                                             \
279         typeof(name) __val;                                                        \
280         begin_safe_access_tls_vars();                                              \
281         __val = name;                                                              \
282         end_safe_access_tls_vars();                                                \
283         __val;                                                                     \
284 })
285
286 #define vcore_set_tls_var(name, val)                                           \
287 ({                                                                             \
288         typeof(val) __val = val;                                                   \
289         begin_access_tls_vars(get_vcpd_tls_desc(vcoreid));                         \
290         name = __val;                                                              \
291         end_access_tls_vars();                                                     \
292 })
293
294 #define vcore_get_tls_var(name)                                                \
295 ({                                                                             \
296         typeof(name) val;                                                          \
297         begin_access_tls_vars(get_vcpd_tls_desc(vcoreid));                         \
298         val = name;                                                                \
299         end_access_tls_vars();                                                     \
300         val;                                                                       \
301 })
302
303 __END_DECLS