Ensures IRQs are enabled when proc_destroy()ing
[akaros.git] / kern / arch / riscv / trap.c
1 #include <arch/arch.h>
2 #include <assert.h>
3 #include <arch/trap.h>
4 #include <string.h>
5 #include <process.h>
6 #include <syscall.h>
7 #include <monitor.h>
8 #include <manager.h>
9 #include <stdio.h>
10 #include <smp.h>
11 #include <slab.h>
12 #include <mm.h>
13 #include <umem.h>
14 #include <pmap.h>
15
16 /* These are the stacks the kernel will load when it receives a trap from user
17  * space.  The deal is that they get set right away in entry.S, and can always
18  * be used for finding the top of the stack (from which you should subtract the
19  * sizeof the trapframe.  Note, we need to have a junk value in the array so
20  * that this is NOT part of the BSS.  If it is in the BSS, it will get 0'd in
21  * kernel_init(), which is after these values get set.
22  *
23  * TODO: if these end up becoming contended cache lines, move this to
24  * per_cpu_info. */
25 uintptr_t core_stacktops[MAX_NUM_CPUS] = {0xcafebabe, 0};
26
27 struct kmem_cache *kernel_msg_cache;
28 void kernel_msg_init(void)
29 {
30         kernel_msg_cache = kmem_cache_create("kernel_msgs",
31                            sizeof(struct kernel_message), HW_CACHE_ALIGN, 0, 0, 0);
32 }
33
34 spinlock_t kernel_message_buf_busy[MAX_NUM_CPUS] = {SPINLOCK_INITIALIZER};
35 kernel_message_t kernel_message_buf[MAX_NUM_CPUS];
36
37 /* This is mostly identical to x86's, minus the different send_ipi call. */
38 uint32_t send_kernel_message(uint32_t dst, amr_t pc, long arg0, long arg1,
39                              long arg2, int type)
40 {
41         kernel_message_t *k_msg;
42         assert(pc);
43         // note this will be freed on the destination core
44         k_msg = (kernel_message_t *CT(1))TC(kmem_cache_alloc(kernel_msg_cache, 0));
45         k_msg->srcid = core_id();
46         k_msg->pc = pc;
47         k_msg->arg0 = arg0;
48         k_msg->arg1 = arg1;
49         k_msg->arg2 = arg2;
50         switch (type) {
51                 case KMSG_IMMEDIATE:
52                         spin_lock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
53                         STAILQ_INSERT_TAIL(&per_cpu_info[dst].immed_amsgs, k_msg, link);
54                         spin_unlock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
55                         break;
56                 case KMSG_ROUTINE:
57                         spin_lock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
58                         STAILQ_INSERT_TAIL(&per_cpu_info[dst].routine_amsgs, k_msg, link);
59                         spin_unlock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
60                         break;
61                 default:
62                         panic("Unknown type of kernel message!");
63         }
64         /* if we're sending a routine message locally, we don't want/need an IPI */
65         if ((dst != k_msg->srcid) || (type == KMSG_IMMEDIATE))
66                 send_ipi(dst);
67         return 0;
68 }
69
70 void
71 advance_pc(trapframe_t* state)
72 {
73         state->epc += 4;
74 }
75
76 /* Set stacktop for the current core to be the stack the kernel will start on
77  * when trapping/interrupting from userspace */
78 void set_stack_top(uintptr_t stacktop)
79 {
80         core_stacktops[core_id()] = stacktop;
81 }
82
83 /* Note the assertion assumes we are in the top page of the stack. */
84 uintptr_t get_stack_top(void)
85 {
86         register uintptr_t sp asm ("sp");
87         uintptr_t stacktop = core_stacktops[core_id()];
88         assert(ROUNDUP(sp, PGSIZE) == stacktop);
89         return stacktop;
90 }
91
92 void
93 idt_init(void)
94 {
95 }
96
97 void
98 sysenter_init(void)
99 {
100 }
101
102 /* Helper.  For now, this copies out the TF to pcpui, and sets cur_tf to point
103  * to it. */
104 static void
105 set_current_tf(struct per_cpu_info *pcpui, struct trapframe *tf)
106 {
107         if (irq_is_enabled())
108                 warn("Turn off IRQs until cur_tf is set!");
109         assert(!pcpui->cur_tf);
110         pcpui->actual_tf = *tf;
111         pcpui->cur_tf = &pcpui->actual_tf;
112 }
113
114 static int
115 format_trapframe(trapframe_t *tf, char* buf, int bufsz)
116 {
117         // slightly hackish way to read out the instruction that faulted.
118         // not guaranteed to be right 100% of the time
119         uint32_t insn;
120         if(!(current && !memcpy_from_user(current,&insn,(void*)tf->epc,4)))
121                 insn = -1;
122
123         int len = snprintf(buf,bufsz,"TRAP frame at %p on core %d\n",
124                            tf, core_id());
125         static const char* regnames[] = {
126           "z ", "ra", "v0", "v1", "a0", "a1", "a2", "a3",
127           "a4", "a5", "a6", "a7", "t0", "t1", "t2", "t3",
128           "t4", "t5", "t6", "t7", "s0", "s1", "s2", "s3",
129           "s4", "s5", "s6", "s7", "s8", "fp", "sp", "tp"
130         };
131         
132         tf->gpr[0] = 0;
133         
134         for(int i = 0; i < 32; i+=4)
135         {
136                 for(int j = 0; j < 4; j++)
137                         len += snprintf(buf+len, bufsz-len,
138                                         "%s %016lx%c", regnames[i+j], tf->gpr[i+j], 
139                                         j < 3 ? ' ' : '\n');
140         }
141         len += snprintf(buf+len, bufsz-len,
142                         "sr %016lx pc %016lx va %016lx insn       %08x\n",
143                                         tf->sr, tf->epc, tf->badvaddr, insn);
144
145         buf[bufsz-1] = 0;
146         return len;
147 }
148
149 void
150 print_trapframe(trapframe_t* tf)
151 {
152         char buf[1024];
153         int len = format_trapframe(tf,buf,sizeof(buf));
154         cputbuf(buf,len);
155 }
156
157 /* Helper function.  Returns 0 if the list was empty. */
158 static kernel_message_t *get_next_amsg(struct kernel_msg_list *list_head,
159                                        spinlock_t *list_lock)
160 {
161         kernel_message_t *k_msg;
162         spin_lock_irqsave(list_lock);
163         k_msg = STAILQ_FIRST(list_head);
164         if (k_msg)
165                 STAILQ_REMOVE_HEAD(list_head, link);
166         spin_unlock_irqsave(list_lock);
167         return k_msg;
168 }
169
170 static void exit_halt_loop(trapframe_t* tf)
171 {
172         extern char after_cpu_halt;
173         if ((char*)tf->epc >= (char*)&cpu_halt && (char*)tf->epc < &after_cpu_halt)
174                 tf->epc = tf->gpr[1];
175 }
176
177 /* Mostly the same as x86's implementation.  Keep them in sync.  This assumes
178  * you can send yourself an IPI, and that IPIs can get squashed like on x86. */
179 static void
180 handle_ipi(trapframe_t* tf)
181 {
182
183         if (!in_kernel(tf))
184                 set_current_tf(&per_cpu_info[core_id()], tf);
185         else
186                 exit_halt_loop(tf);
187         
188         clear_ipi();
189
190         per_cpu_info_t *myinfo = &per_cpu_info[core_id()];
191         kernel_message_t msg_cp, *k_msg;
192
193         while (1) { // will break out when there are no more messages
194                 /* Try to get an immediate message.  Exec and free it. */
195                 k_msg = get_next_amsg(&myinfo->immed_amsgs, &myinfo->immed_amsg_lock);
196                 if (k_msg) {
197                         assert(k_msg->pc);
198                         k_msg->pc(tf, k_msg->srcid, k_msg->arg0, k_msg->arg1, k_msg->arg2);
199                         kmem_cache_free(kernel_msg_cache, (void*)k_msg);
200                 } else { // no immediate, might be a routine
201                         if (in_kernel(tf))
202                                 return; // don't execute routine msgs if we were in the kernel
203                         k_msg = get_next_amsg(&myinfo->routine_amsgs,
204                                               &myinfo->routine_amsg_lock);
205                         if (!k_msg) // no routines either
206                                 return;
207                         /* copy in, and then free, in case we don't return */
208                         msg_cp = *k_msg;
209                         kmem_cache_free(kernel_msg_cache, (void*)k_msg);
210                         /* make sure an IPI is pending if we have more work */
211                         /* techincally, we don't need to lock when checking */
212                         if (!STAILQ_EMPTY(&myinfo->routine_amsgs))
213                                 send_ipi(core_id());
214                         /* Execute the kernel message */
215                         assert(msg_cp.pc);
216                         msg_cp.pc(tf, msg_cp.srcid, msg_cp.arg0, msg_cp.arg1, msg_cp.arg2);
217                 }
218         }
219 }
220
221 /* Same as in x86.  Might be diff in the future if there is no way to check for
222  * immediate messages or there is the ability to selectively mask IPI vectors.*/
223 void
224 process_routine_kmsg(struct trapframe *tf)
225 {
226         per_cpu_info_t *myinfo = &per_cpu_info[core_id()];
227         kernel_message_t msg_cp, *k_msg;
228         int8_t irq_state = 0;
229
230         disable_irqsave(&irq_state);
231         /* If we were told what our TF was, use that.  o/w, go with current_tf. */
232         tf = tf ? tf : current_tf;
233         while (1) {
234                 /* normally, we want ints disabled, so we don't have an empty self-ipi
235                  * for every routine message. (imagine a long list of routines).  But we
236                  * do want immediates to run ahead of routines.  This enabling should
237                  * work (might not in some shitty VMs).  Also note we can receive an
238                  * extra self-ipi for routine messages before we turn off irqs again.
239                  * Not a big deal, since we will process it right away. */
240                 if (!STAILQ_EMPTY(&myinfo->immed_amsgs)) {
241                         enable_irq();
242                         cpu_relax();
243                         disable_irq();
244                 }
245                 k_msg = get_next_amsg(&myinfo->routine_amsgs,
246                                       &myinfo->routine_amsg_lock);
247                 if (!k_msg) {
248                         enable_irqsave(&irq_state);
249                         return;
250                 }
251                 /* copy in, and then free, in case we don't return */
252                 msg_cp = *k_msg;
253                 kmem_cache_free(kernel_msg_cache, (void*)k_msg);
254                 /* make sure an IPI is pending if we have more work */
255                 if (!STAILQ_EMPTY(&myinfo->routine_amsgs))
256                         send_ipi(core_id());
257                 /* Execute the kernel message */
258                 assert(msg_cp.pc);
259                 msg_cp.pc(tf, msg_cp.srcid, msg_cp.arg0, msg_cp.arg1, msg_cp.arg2);
260         }
261 }
262
263 /* extremely dangerous and racy: prints out the immed and routine kmsgs for a
264  * specific core (so possibly remotely).  Same as x86. */
265 void print_kmsgs(uint32_t coreid)
266 {
267         struct per_cpu_info *pcpui = &per_cpu_info[coreid];
268         void __print_kmsgs(struct kernel_msg_list *list, char *type)
269         {
270                 char *fn_name;
271                 struct kernel_message *kmsg_i;
272                 STAILQ_FOREACH(kmsg_i, list, link) {
273                         fn_name = get_fn_name((long)kmsg_i->pc);
274                         printk("%s KMSG on %d from %d to run %08p(%s)\n", type,
275                                kmsg_i->dstid, kmsg_i->srcid, kmsg_i->pc, fn_name); 
276                         kfree(fn_name);
277                 }
278         }
279         __print_kmsgs(&pcpui->immed_amsgs, "Immedte");
280         __print_kmsgs(&pcpui->routine_amsgs, "Routine");
281 }
282
283 static void
284 unhandled_trap(trapframe_t* state, const char* name)
285 {
286         static spinlock_t screwup_lock = SPINLOCK_INITIALIZER;
287         spin_lock(&screwup_lock);
288
289         if(in_kernel(state))
290         {
291                 print_trapframe(state);
292                 panic("Unhandled trap in kernel!\nTrap type: %s", name);
293         }
294         else
295         {
296                 char tf_buf[1024];
297                 format_trapframe(state, tf_buf, sizeof(tf_buf));
298
299                 warn("Unhandled trap in user!\nTrap type: %s\n%s", name, tf_buf);
300                 backtrace();
301                 spin_unlock(&screwup_lock);
302
303                 assert(current);
304                 enable_irq();
305                 proc_destroy(current);
306         }
307 }
308
309 static void
310 handle_timer_interrupt(trapframe_t* tf)
311 {
312         if (!in_kernel(tf))
313                 set_current_tf(&per_cpu_info[core_id()], tf);
314         else
315                 exit_halt_loop(tf);
316         
317         timer_interrupt(tf, NULL);
318 }
319
320 static void
321 handle_misaligned_fetch(trapframe_t* state)
322 {
323         unhandled_trap(state, "Misaligned Fetch");
324 }
325
326 static void
327 handle_misaligned_load(trapframe_t* state)
328 {
329         unhandled_trap(state, "Misaligned Load");
330 }
331
332 static void
333 handle_misaligned_store(trapframe_t* state)
334 {
335         unhandled_trap(state, "Misaligned Store");
336 }
337
338 static void
339 handle_fault_fetch(trapframe_t* state)
340 {
341         if(in_kernel(state))
342         {
343                 print_trapframe(state);
344                 panic("Instruction Page Fault in the Kernel at %p!", state->epc);
345         }
346
347         set_current_tf(&per_cpu_info[core_id()], state);
348
349         if(handle_page_fault(current, state->epc, PROT_EXEC))
350                 unhandled_trap(state, "Instruction Page Fault");
351 }
352
353 static void
354 handle_fault_load(trapframe_t* state)
355 {
356         if(in_kernel(state))
357         {
358                 print_trapframe(state);
359                 panic("Load Page Fault in the Kernel at %p!", state->badvaddr);
360         }
361
362         set_current_tf(&per_cpu_info[core_id()], state);
363
364         if(handle_page_fault(current, state->badvaddr, PROT_READ))
365                 unhandled_trap(state, "Load Page Fault");
366 }
367
368 static void
369 handle_fault_store(trapframe_t* state)
370 {
371         if(in_kernel(state))
372         {
373                 print_trapframe(state);
374                 panic("Store Page Fault in the Kernel at %p!", state->badvaddr);
375         }
376         
377         set_current_tf(&per_cpu_info[core_id()], state);
378
379         if(handle_page_fault(current, state->badvaddr, PROT_WRITE))
380                 unhandled_trap(state, "Store Page Fault");
381 }
382
383 static void
384 handle_illegal_instruction(trapframe_t* state)
385 {
386         set_current_tf(&per_cpu_info[core_id()], state);
387
388         if (emulate_fpu(state) == 0)
389         {
390                 advance_pc(per_cpu_info[core_id()].cur_tf);
391                 return;
392         }
393
394         unhandled_trap(state, "Illegal Instruction");
395 }
396
397 static void
398 handle_fp_disabled(trapframe_t* tf)
399 {
400         if(in_kernel(tf))
401                 panic("kernel executed an FP instruction!");
402
403         tf->sr |= SR_EF;
404         env_pop_tf(tf); /* We didn't save our TF, so don't use proc_restartcore */
405 }
406
407 static void
408 handle_syscall(trapframe_t* state)
409 {
410         uintptr_t a0 = state->gpr[4];
411         uintptr_t a1 = state->gpr[5];
412
413         advance_pc(state);
414         set_current_tf(&per_cpu_info[core_id()], state);
415         enable_irq();
416         prep_syscalls(current, (struct syscall*)a0, a1);
417 }
418
419 static void
420 handle_breakpoint(trapframe_t* state)
421 {
422         advance_pc(state);
423         monitor(state);
424 }
425
426 void
427 handle_trap(trapframe_t* tf)
428 {
429         static void (*const trap_handlers[])(trapframe_t*) = {
430           [CAUSE_MISALIGNED_FETCH] = handle_misaligned_fetch,
431           [CAUSE_FAULT_FETCH] = handle_fault_fetch,
432           [CAUSE_ILLEGAL_INSTRUCTION] = handle_illegal_instruction,
433           [CAUSE_PRIVILEGED_INSTRUCTION] = handle_illegal_instruction,
434           [CAUSE_FP_DISABLED] = handle_fp_disabled,
435           [CAUSE_SYSCALL] = handle_syscall,
436           [CAUSE_BREAKPOINT] = handle_breakpoint,
437           [CAUSE_MISALIGNED_LOAD] = handle_misaligned_load,
438           [CAUSE_MISALIGNED_STORE] = handle_misaligned_store,
439           [CAUSE_FAULT_LOAD] = handle_fault_load,
440           [CAUSE_FAULT_STORE] = handle_fault_store,
441         };
442
443         static void (*const irq_handlers[])(trapframe_t*) = {
444           [IRQ_TIMER] = handle_timer_interrupt,
445           [IRQ_IPI] = handle_ipi,
446         };
447         
448         if (tf->cause < 0)
449         {
450                 uint8_t irq = tf->cause;
451                 assert(irq < sizeof(irq_handlers)/sizeof(irq_handlers[0]) &&
452                        irq_handlers[irq]);
453                 irq_handlers[irq](tf);
454         }
455         else
456         {
457                 assert(tf->cause < sizeof(trap_handlers)/sizeof(trap_handlers[0]) &&
458                        trap_handlers[tf->cause]);
459                 trap_handlers[tf->cause](tf);
460         }
461         
462         /* Return to the current process, which should be runnable.  If we're the
463          * kernel, we should just return naturally.  Note that current and tf need
464          * to still be okay (might not be after blocking) */
465         if (in_kernel(tf))
466                 env_pop_tf(tf);
467         else
468                 proc_restartcore();
469 }
470
471 /* We don't have NMIs now. */
472 void send_nmi(uint32_t os_coreid)
473 {
474 }