RISC-V SMP boot works
[akaros.git] / kern / arch / riscv / trap.c
1 #include <arch/arch.h>
2 #include <assert.h>
3 #include <arch/trap.h>
4 #include <string.h>
5 #include <process.h>
6 #include <syscall.h>
7 #include <monitor.h>
8 #include <manager.h>
9 #include <stdio.h>
10 #include <smp.h>
11 #include <slab.h>
12 #include <mm.h>
13 #include <ros/mman.h>
14 #include <umem.h>
15 #include <pmap.h>
16
17 /* These are the stacks the kernel will load when it receives a trap from user
18  * space.  The deal is that they get set right away in entry.S, and can always
19  * be used for finding the top of the stack (from which you should subtract the
20  * sizeof the trapframe.  Note, we need to have a junk value in the array so
21  * that this is NOT part of the BSS.  If it is in the BSS, it will get 0'd in
22  * kernel_init(), which is after these values get set.
23  *
24  * TODO: if these end up becoming contended cache lines, move this to
25  * per_cpu_info. */
26 uintptr_t core_stacktops[MAX_NUM_CPUS] = {0xcafebabe, 0};
27
28 struct kmem_cache *kernel_msg_cache;
29 void kernel_msg_init(void)
30 {
31         kernel_msg_cache = kmem_cache_create("kernel_msgs",
32                            sizeof(struct kernel_message), HW_CACHE_ALIGN, 0, 0, 0);
33 }
34
35 spinlock_t kernel_message_buf_busy[MAX_NUM_CPUS] = {SPINLOCK_INITIALIZER};
36 kernel_message_t kernel_message_buf[MAX_NUM_CPUS];
37
38 /* This is mostly identical to x86's, minus the different send_ipi call. */
39 uint32_t send_kernel_message(uint32_t dst, amr_t pc,
40                              TV(a0t) arg0, TV(a1t) arg1, TV(a2t) arg2, int type)
41 {
42         kernel_message_t *k_msg;
43         assert(pc);
44         // note this will be freed on the destination core
45         k_msg = (kernel_message_t *CT(1))TC(kmem_cache_alloc(kernel_msg_cache, 0));
46         k_msg->srcid = core_id();
47         k_msg->pc = pc;
48         k_msg->arg0 = arg0;
49         k_msg->arg1 = arg1;
50         k_msg->arg2 = arg2;
51         switch (type) {
52                 case KMSG_IMMEDIATE:
53                         spin_lock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
54                         STAILQ_INSERT_TAIL(&per_cpu_info[dst].immed_amsgs, k_msg, link);
55                         spin_unlock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
56                         break;
57                 case KMSG_ROUTINE:
58                         spin_lock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
59                         STAILQ_INSERT_TAIL(&per_cpu_info[dst].routine_amsgs, k_msg, link);
60                         spin_unlock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
61                         break;
62                 default:
63                         panic("Unknown type of kernel message!");
64         }
65         /* if we're sending a routine message locally, we don't want/need an IPI */
66         if ((dst != k_msg->srcid) || (type == KMSG_IMMEDIATE))
67                 send_ipi(dst);
68         return 0;
69 }
70
71 void
72 advance_pc(trapframe_t* state)
73 {
74         state->epc += 4;
75 }
76
77 /* Set stacktop for the current core to be the stack the kernel will start on
78  * when trapping/interrupting from userspace */
79 void set_stack_top(uintptr_t stacktop)
80 {
81         core_stacktops[core_id()] = stacktop;
82 }
83
84 /* Note the assertion assumes we are in the top page of the stack. */
85 uintptr_t get_stack_top(void)
86 {
87         uintptr_t sp, stacktop;
88         stacktop = core_stacktops[core_id()];
89         asm volatile("move %0,$sp" : "=r"(sp));
90         assert(ROUNDUP(sp, PGSIZE) == stacktop);
91         return stacktop;
92 }
93
94 void
95 idt_init(void)
96 {
97 }
98
99 void
100 sysenter_init(void)
101 {
102 }
103
104 /* Helper.  For now, this copies out the TF to pcpui, and sets the tf to use it.
105  * Eventually, we ought to do this in trap_entry.S.  Honestly, do whatever you
106  * want with this.  The **tf is for convenience in x86. */
107 static void set_current_tf(struct per_cpu_info *pcpui, struct trapframe **tf)
108 {
109         pcpui->actual_tf = **tf;
110         pcpui->cur_tf = &pcpui->actual_tf;
111         *tf = &pcpui->actual_tf;
112 }
113
114 static int
115 format_trapframe(trapframe_t *tf, char* buf, int bufsz)
116 {
117         // slightly hackish way to read out the instruction that faulted.
118         // not guaranteed to be right 100% of the time
119         uint32_t insn;
120         if(!(current && !memcpy_from_user(current,&insn,(void*)tf->epc,4)))
121                 insn = -1;
122
123         int len = snprintf(buf,bufsz,"TRAP frame at %p on core %d\n",
124                            tf, core_id());
125         static const char* regnames[] = {
126           "z ", "ra", "v0", "v1", "a0", "a1", "a2", "a3",
127           "a4", "a5", "a6", "a7", "t0", "t1", "t2", "t3",
128           "t4", "t5", "t6", "t7", "s0", "s1", "s2", "s3",
129           "s4", "s5", "s6", "s7", "s8", "fp", "sp", "tp"
130         };
131         
132         tf->gpr[0] = 0;
133         
134         for(int i = 0; i < 32; i+=4)
135         {
136                 for(int j = 0; j < 4; j++)
137                         len += snprintf(buf+len, bufsz-len,
138                                         "%s %016lx%c", regnames[i+j], tf->gpr[i+j], 
139                                         j < 3 ? ' ' : '\n');
140         }
141         len += snprintf(buf+len, bufsz-len,
142                         "sr %016lx pc %016lx va %016lx insn       %008x\n", tf->sr, tf->epc,
143                         tf->badvaddr, (uint32_t)tf->insn);
144
145         return len;
146 }
147
148 void
149 print_trapframe(trapframe_t* tf)
150 {
151         char buf[1024];
152         int len = format_trapframe(tf,buf,sizeof(buf));
153         cputbuf(buf,len);
154 }
155
156 /* Helper function.  Returns 0 if the list was empty. */
157 static kernel_message_t *get_next_amsg(struct kernel_msg_list *list_head,
158                                        spinlock_t *list_lock)
159 {
160         kernel_message_t *k_msg;
161         spin_lock_irqsave(list_lock);
162         k_msg = STAILQ_FIRST(list_head);
163         if (k_msg)
164                 STAILQ_REMOVE_HEAD(list_head, link);
165         spin_unlock_irqsave(list_lock);
166         return k_msg;
167 }
168
169 /* Mostly the same as x86's implementation.  Keep them in sync.  This assumes
170  * you can send yourself an IPI, and that IPIs can get squashed like on x86. */
171 static void
172 handle_ipi(trapframe_t* tf)
173 {
174         struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
175         if (!in_kernel(tf))
176                 set_current_tf(pcpui, &tf);
177         else if((void*)tf->epc == &cpu_halt) // break out of the cpu_halt loop
178                 advance_pc(tf);
179
180         per_cpu_info_t *myinfo = &per_cpu_info[core_id()];
181         kernel_message_t msg_cp, *k_msg;
182
183         while (1) { // will break out when there are no more messages
184                 /* Try to get an immediate message.  Exec and free it. */
185                 k_msg = get_next_amsg(&myinfo->immed_amsgs, &myinfo->immed_amsg_lock);
186                 if (k_msg) {
187                         assert(k_msg->pc);
188                         k_msg->pc(tf, k_msg->srcid, k_msg->arg0, k_msg->arg1, k_msg->arg2);
189                         kmem_cache_free(kernel_msg_cache, (void*)k_msg);
190                 } else { // no immediate, might be a routine
191                         if (in_kernel(tf))
192                                 return; // don't execute routine msgs if we were in the kernel
193                         k_msg = get_next_amsg(&myinfo->routine_amsgs,
194                                               &myinfo->routine_amsg_lock);
195                         if (!k_msg) // no routines either
196                                 return;
197                         /* copy in, and then free, in case we don't return */
198                         msg_cp = *k_msg;
199                         kmem_cache_free(kernel_msg_cache, (void*)k_msg);
200                         /* make sure an IPI is pending if we have more work */
201                         /* techincally, we don't need to lock when checking */
202                         if (!STAILQ_EMPTY(&myinfo->routine_amsgs))
203                                 send_ipi(core_id());
204                         /* Execute the kernel message */
205                         assert(msg_cp.pc);
206                         msg_cp.pc(tf, msg_cp.srcid, msg_cp.arg0, msg_cp.arg1, msg_cp.arg2);
207                 }
208         }
209 }
210
211 /* Same as in x86.  Might be diff in the future if there is no way to check for
212  * immediate messages or there is the ability to selectively mask IPI vectors.*/
213 void process_routine_kmsg(struct trapframe *tf)
214 {
215         per_cpu_info_t *myinfo = &per_cpu_info[core_id()];
216         kernel_message_t msg_cp, *k_msg;
217         int8_t irq_state = 0;
218
219         disable_irqsave(&irq_state);
220         /* If we were told what our TF was, use that.  o/w, go with current_tf. */
221         tf = tf ? tf : current_tf;
222         while (1) {
223                 /* normally, we want ints disabled, so we don't have an empty self-ipi
224                  * for every routine message. (imagine a long list of routines).  But we
225                  * do want immediates to run ahead of routines.  This enabling should
226                  * work (might not in some shitty VMs).  Also note we can receive an
227                  * extra self-ipi for routine messages before we turn off irqs again.
228                  * Not a big deal, since we will process it right away. */
229                 if (!STAILQ_EMPTY(&myinfo->immed_amsgs)) {
230                         enable_irq();
231                         cpu_relax();
232                         disable_irq();
233                 }
234                 k_msg = get_next_amsg(&myinfo->routine_amsgs,
235                                       &myinfo->routine_amsg_lock);
236                 if (!k_msg) {
237                         enable_irqsave(&irq_state);
238                         return;
239                 }
240                 /* copy in, and then free, in case we don't return */
241                 msg_cp = *k_msg;
242                 kmem_cache_free(kernel_msg_cache, (void*)k_msg);
243                 /* make sure an IPI is pending if we have more work */
244                 if (!STAILQ_EMPTY(&myinfo->routine_amsgs))
245                         send_ipi(core_id());
246                 /* Execute the kernel message */
247                 assert(msg_cp.pc);
248                 msg_cp.pc(tf, msg_cp.srcid, msg_cp.arg0, msg_cp.arg1, msg_cp.arg2);
249         }
250 }
251
252 static void
253 unhandled_trap(trapframe_t* state, const char* name)
254 {
255         static spinlock_t screwup_lock = SPINLOCK_INITIALIZER;
256         spin_lock(&screwup_lock);
257
258         if(in_kernel(state))
259         {
260                 print_trapframe(state);
261                 panic("Unhandled trap in kernel!\nTrap type: %s", name);
262         }
263         else
264         {
265                 char tf_buf[1024];
266                 int tf_len = format_trapframe(state, tf_buf, sizeof(tf_buf));
267
268                 warn("Unhandled trap in user!\nTrap type: %s\n%s", name, tf_buf);
269                 backtrace();
270                 spin_unlock(&screwup_lock);
271
272                 assert(current);
273                 proc_incref(current, 1);
274                 proc_destroy(current);
275
276                 panic("I shouldn't have gotten here!");
277         }
278 }
279
280 static void
281 handle_timer_interrupt(trapframe_t* state)
282 {
283         timer_interrupt(state, NULL);
284 }
285
286 static void
287 handle_interrupt(trapframe_t* state)
288 {
289         typedef void (*trap_handler)(trapframe_t*);
290         
291         const static trap_handler trap_handlers[NIRQ] = {
292           [TIMER_IRQ] = handle_timer_interrupt,
293           [IPI_IRQ] = handle_ipi,
294         };
295
296         int interrupts = (state->cause & CAUSE_IP) >> CAUSE_IP_SHIFT;
297
298         for(int i = 0; interrupts; interrupts >>= 1, i++)
299         {
300                 if(interrupts & 1)
301                 {
302                         if(trap_handlers[i])
303                                 trap_handlers[i](state);
304                         else
305                         {
306                                 char name[32];
307                                 snprintf(name, sizeof(name), "Bad Interrupt %d", i);
308                                 unhandled_trap(state, name);
309                         }
310                 }
311         }
312 }
313
314 static void
315 handle_misaligned_fetch(trapframe_t* state)
316 {
317         unhandled_trap(state, "Misaligned Fetch");
318 }
319
320 static void
321 handle_misaligned_load(trapframe_t* state)
322 {
323         unhandled_trap(state, "Misaligned Load");
324 }
325
326 static void
327 handle_misaligned_store(trapframe_t* state)
328 {
329         unhandled_trap(state, "Misaligned Store");
330 }
331
332 static void
333 handle_fault_fetch(trapframe_t* state)
334 {
335         if(in_kernel(state))
336         {
337                 print_trapframe(state);
338                 panic("Load Page Fault in the Kernel at 0x%08x!", state->badvaddr);
339         }
340         
341         if(handle_page_fault(current, state->badvaddr, PROT_READ))
342                 unhandled_trap(state, "Load Page Fault");
343 }
344
345 static void
346 handle_fault_load(trapframe_t* state)
347 {
348         if(in_kernel(state))
349         {
350                 print_trapframe(state);
351                 panic("Load Page Fault in the Kernel at 0x%08x!", state->badvaddr);
352         }
353         
354         if(handle_page_fault(current, state->badvaddr, PROT_READ))
355                 unhandled_trap(state, "Load Page Fault");
356 }
357
358 static void
359 handle_fault_store(trapframe_t* state)
360 {
361         if(in_kernel(state))
362         {
363                 print_trapframe(state);
364                 panic("Store Page Fault in the Kernel at 0x%08x!", state->badvaddr);
365         }
366         
367         if(handle_page_fault(current, state->badvaddr, PROT_WRITE))
368                 unhandled_trap(state, "Store Page Fault");
369 }
370
371 static void
372 handle_illegal_instruction(trapframe_t* state)
373 {
374         unhandled_trap(state, "Illegal Instruction");
375 }
376
377 static void
378 handle_fp_disabled(trapframe_t* state)
379 {
380         if(in_kernel(state))
381                 panic("kernel executed an FP instruction!");
382
383         state->sr |= SR_EF;
384 }
385
386 static void
387 handle_syscall(trapframe_t* state)
388 {
389         struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
390         uintptr_t a0 = state->gpr[4];
391         uintptr_t a1 = state->gpr[5];
392
393         advance_pc(state);
394         enable_irq();
395         struct per_cpu_info* coreinfo = &per_cpu_info[core_id()];
396
397         set_current_tf(pcpui, &state);
398
399         prep_syscalls(current, (struct syscall*)a0, a1);
400
401         proc_restartcore();
402 }
403
404 static void
405 handle_breakpoint(trapframe_t* state)
406 {
407         advance_pc(state);
408         monitor(state);
409 }
410
411 void
412 handle_trap(trapframe_t* tf)
413 {
414         typedef void (*trap_handler)(trapframe_t*);
415         
416         const static trap_handler trap_handlers[NUM_CAUSES] = {
417           [CAUSE_MISALIGNED_FETCH] = handle_misaligned_fetch,
418           [CAUSE_FAULT_FETCH] = handle_fault_fetch,
419           [CAUSE_ILLEGAL_INSTRUCTION] = handle_illegal_instruction,
420           [CAUSE_PRIVILEGED_INSTRUCTION] = handle_illegal_instruction,
421           [CAUSE_FP_DISABLED] = handle_fp_disabled,
422           [CAUSE_INTERRUPT] = handle_interrupt,
423           [CAUSE_SYSCALL] = handle_syscall,
424           [CAUSE_BREAKPOINT] = handle_breakpoint,
425           [CAUSE_MISALIGNED_LOAD] = handle_misaligned_load,
426           [CAUSE_MISALIGNED_STORE] = handle_misaligned_store,
427           [CAUSE_FAULT_LOAD] = handle_fault_load,
428           [CAUSE_FAULT_STORE] = handle_fault_store,
429         };
430         
431         int exccode = (tf->cause & CAUSE_EXCCODE) >> CAUSE_EXCCODE_SHIFT;
432         assert(exccode < NUM_CAUSES && trap_handlers[exccode]);
433         
434         trap_handlers[exccode](tf);
435         
436         env_pop_tf(tf);
437 }