1 /* Copyright (c) 2012 The Regents of the University of California
2 * Barret Rhoden <brho@cs.berkeley.edu>
3 * See LICENSE for details.
5 * Arch-independent trap handling and kernel messaging */
16 static void print_unhandled_trap(struct proc *p, struct user_context *ctx,
17 unsigned int trap_nr, unsigned int err,
20 struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
21 uint32_t vcoreid = pcpui->owning_vcoreid;
22 struct preempt_data *vcpd = &p->procdata->vcore_preempt_data[vcoreid];
24 if (!proc_is_vcctx_ready(p))
25 printk("Unhandled user trap from early SCP\n");
26 else if (vcpd->notif_disabled)
27 printk("Unhandled user trap in vcore context from VC %d\n", vcoreid);
29 printk("err 0x%x (for PFs: User 4, Wr 2, Rd 1), aux %p\n", err, aux);
30 debug_addr_proc(p, get_user_ctx_pc(ctx));
32 backtrace_user_ctx(p, ctx);
35 /* Traps that are considered normal operations. */
36 static bool benign_trap(unsigned int err)
38 return err & PF_VMR_BACKED;
41 static void printx_unhandled_trap(struct proc *p, struct user_context *ctx,
42 unsigned int trap_nr, unsigned int err,
45 if (printx_on && !benign_trap(err))
46 print_unhandled_trap(p, ctx, trap_nr, err, aux);
49 /* Helper, reflects the current context back to the 2LS. Returns 0 on success,
51 int reflect_current_context(void)
53 uint32_t coreid = core_id();
54 struct per_cpu_info *pcpui = &per_cpu_info[coreid];
55 struct proc *p = pcpui->cur_proc;
56 uint32_t vcoreid = pcpui->owning_vcoreid;
57 struct preempt_data *vcpd = &p->procdata->vcore_preempt_data[vcoreid];
59 if (!proc_is_vcctx_ready(p))
61 if (vcpd->notif_disabled)
63 /* the guts of a __notify */
64 vcpd->notif_disabled = TRUE;
65 copy_current_ctx_to(&vcpd->uthread_ctx);
66 memset(pcpui->cur_ctx, 0, sizeof(struct user_context));
67 proc_init_ctx(pcpui->cur_ctx, vcoreid, vcpd->vcore_entry,
68 vcpd->vcore_stack, vcpd->vcore_tls_desc);
72 void reflect_unhandled_trap(unsigned int trap_nr, unsigned int err,
75 uint32_t coreid = core_id();
76 struct per_cpu_info *pcpui = &per_cpu_info[coreid];
77 struct proc *p = pcpui->cur_proc;
80 assert(pcpui->cur_ctx && (pcpui->cur_ctx->type == ROS_HW_CTX));
81 /* need to store trap_nr, err code, and aux into the tf so that it can get
82 * extracted on the other end, and we need to flag the TF in some way so we
83 * can tell it was reflected. for example, on a PF, we need some number (14
84 * on x86), the prot violation (write, read, etc), and the virt addr (aux).
85 * parlib will know how to extract this info. */
86 __arch_reflect_trap_hwtf(&pcpui->cur_ctx->tf.hw_tf, trap_nr, err, aux);
87 printx_unhandled_trap(p, pcpui->cur_ctx, trap_nr, err, aux);
88 if (reflect_current_context()) {
89 print_unhandled_trap(p, pcpui->cur_ctx, trap_nr, err, aux);
94 uintptr_t get_user_ctx_pc(struct user_context *ctx)
98 return get_hwtf_pc(&ctx->tf.hw_tf);
100 return get_swtf_pc(&ctx->tf.sw_tf);
102 return get_vmtf_pc(&ctx->tf.vm_tf);
104 panic("Bad context type %d for ctx %p\n", ctx->type, ctx);
108 uintptr_t get_user_ctx_fp(struct user_context *ctx)
112 return get_hwtf_fp(&ctx->tf.hw_tf);
114 return get_swtf_fp(&ctx->tf.sw_tf);
116 return get_vmtf_fp(&ctx->tf.vm_tf);
118 panic("Bad context type %d for ctx %p\n", ctx->type, ctx);
122 /* Helper, copies the current context to to_ctx. */
123 void copy_current_ctx_to(struct user_context *to_ctx)
125 struct user_context *cur_ctx = current_ctx;
127 /* Be sure to finalize into cur_ctx, not the to_ctx. o/w the arch could get
128 * confused by other calls to finalize. */
129 arch_finalize_ctx(cur_ctx);
133 struct kmem_cache *kernel_msg_cache;
135 void kernel_msg_init(void)
137 kernel_msg_cache = kmem_cache_create("kernel_msgs",
138 sizeof(struct kernel_message), ARCH_CL_SIZE, 0, 0, 0);
141 uint32_t send_kernel_message(uint32_t dst, amr_t pc, long arg0, long arg1,
144 kernel_message_t *k_msg;
146 // note this will be freed on the destination core
147 k_msg = kmem_cache_alloc(kernel_msg_cache, 0);
148 k_msg->srcid = core_id();
156 spin_lock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
157 STAILQ_INSERT_TAIL(&per_cpu_info[dst].immed_amsgs, k_msg, link);
158 spin_unlock_irqsave(&per_cpu_info[dst].immed_amsg_lock);
161 spin_lock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
162 STAILQ_INSERT_TAIL(&per_cpu_info[dst].routine_amsgs, k_msg, link);
163 spin_unlock_irqsave(&per_cpu_info[dst].routine_amsg_lock);
166 panic("Unknown type of kernel message!");
168 /* since we touched memory the other core will touch (the lock), we don't
170 /* if we're sending a routine message locally, we don't want/need an IPI */
171 if ((dst != k_msg->srcid) || (type == KMSG_IMMEDIATE))
172 send_ipi(dst, I_KERNEL_MSG);
176 /* Kernel message IPI/IRQ handler.
178 * This processes immediate messages, and that's it (it used to handle routines
179 * too, if it came in from userspace). Routine messages will get processed when
180 * the kernel has a chance (right before popping to userspace or in smp_idle
183 * Note that all of this happens from interrupt context, and interrupts are
185 void handle_kmsg_ipi(struct hw_trapframe *hw_tf, void *data)
187 struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
188 struct kernel_message *kmsg_i, *temp;
189 /* Avoid locking if the list appears empty (lockless peek is okay) */
190 if (STAILQ_EMPTY(&pcpui->immed_amsgs))
192 /* The lock serves as a cmb to force a re-read of the head of the list */
193 spin_lock_irqsave(&pcpui->immed_amsg_lock);
194 STAILQ_FOREACH_SAFE(kmsg_i, &pcpui->immed_amsgs, link, temp) {
195 pcpui_trace_kmsg(pcpui, (uintptr_t)kmsg_i->pc);
196 kmsg_i->pc(kmsg_i->srcid, kmsg_i->arg0, kmsg_i->arg1, kmsg_i->arg2);
197 STAILQ_REMOVE(&pcpui->immed_amsgs, kmsg_i, kernel_message, link);
198 kmem_cache_free(kernel_msg_cache, (void*)kmsg_i);
200 spin_unlock_irqsave(&pcpui->immed_amsg_lock);
203 bool has_routine_kmsg(void)
205 struct per_cpu_info *pcpui = &per_cpu_info[core_id()];
207 return !STAILQ_EMPTY(&pcpui->routine_amsgs);
210 /* Helper function, gets the next routine KMSG (RKM). Returns 0 if the list was
212 static kernel_message_t *get_next_rkmsg(struct per_cpu_info *pcpui)
214 struct kernel_message *kmsg;
215 /* Avoid locking if the list appears empty (lockless peek is okay) */
216 if (STAILQ_EMPTY(&pcpui->routine_amsgs))
218 /* The lock serves as a cmb to force a re-read of the head of the list.
219 * IRQs are disabled by our caller. */
220 spin_lock(&pcpui->routine_amsg_lock);
221 kmsg = STAILQ_FIRST(&pcpui->routine_amsgs);
223 STAILQ_REMOVE_HEAD(&pcpui->routine_amsgs, link);
224 spin_unlock(&pcpui->routine_amsg_lock);
228 /* Runs routine kernel messages. This might not return. In the past, this
229 * would also run immediate messages, but this is unnecessary. Immediates will
230 * run whenever we reenable IRQs. We could have some sort of ordering or
231 * guarantees between KMSG classes, but that's not particularly useful at this
234 * Note this runs from normal context, with interruptes disabled. However, a
235 * particular RKM could enable interrupts - for instance __launch_kthread() will
236 * restore an old kthread that may have had IRQs on. */
237 void process_routine_kmsg(void)
239 uint32_t pcoreid = core_id();
240 struct per_cpu_info *pcpui = &per_cpu_info[pcoreid];
241 struct kernel_message msg_cp, *kmsg;
243 /* Important that callers have IRQs disabled. When sending cross-core RKMs,
244 * the IPI is used to keep the core from going to sleep - even though RKMs
245 * aren't handled in the kmsg handler. Check smp_idle() for more info. */
246 assert(!irq_is_enabled());
247 while ((kmsg = get_next_rkmsg(pcpui))) {
248 /* Copy in, and then free, in case we don't return */
250 kmem_cache_free(kernel_msg_cache, (void*)kmsg);
251 assert(msg_cp.dstid == pcoreid); /* caught a brutal bug with this */
252 set_rkmsg(pcpui); /* we're now in early RKM ctx */
253 /* The kmsg could block. If it does, we want the kthread code to know
254 * it's not running on behalf of a process, and we're actually spawning
255 * a kernel task. While we do have a syscall that does work in an RKM
256 * (change_to), it's not really the rest of the syscall context. */
257 pcpui->cur_kthread->flags = KTH_KTASK_FLAGS;
258 pcpui_trace_kmsg(pcpui, (uintptr_t)msg_cp.pc);
259 msg_cp.pc(msg_cp.srcid, msg_cp.arg0, msg_cp.arg1, msg_cp.arg2);
260 /* And if we make it back, be sure to restore the default flags. If we
261 * never return, but the kthread exits via some other way (smp_idle()),
262 * then smp_idle() will deal with the flags. The default state includes
264 pcpui->cur_kthread->flags = KTH_DEFAULT_FLAGS;
265 /* If we aren't still in early RKM, it is because the KMSG blocked
266 * (thus leaving early RKM, finishing in default context) and then
267 * returned. This is a 'detached' RKM. Must idle in this scenario,
268 * since we might have migrated or otherwise weren't meant to PRKM
269 * (can't return twice). Also note that this may involve a core
270 * migration, so we need to reread pcpui.*/
272 pcpui = &per_cpu_info[core_id()];
273 if (!in_early_rkmsg_ctx(pcpui))
276 /* Some RKMs might turn on interrupts (perhaps in the future) and then
282 /* extremely dangerous and racy: prints out the immed and routine kmsgs for a
283 * specific core (so possibly remotely) */
284 void print_kmsgs(uint32_t coreid)
286 struct per_cpu_info *pcpui = &per_cpu_info[coreid];
287 void __print_kmsgs(struct kernel_msg_list *list, char *type)
290 struct kernel_message *kmsg_i;
291 STAILQ_FOREACH(kmsg_i, list, link) {
292 fn_name = get_fn_name((long)kmsg_i->pc);
293 printk("%s KMSG on %d from %d to run %p(%s)\n", type,
294 kmsg_i->dstid, kmsg_i->srcid, kmsg_i->pc, fn_name);
298 __print_kmsgs(&pcpui->immed_amsgs, "Immedte");
299 __print_kmsgs(&pcpui->routine_amsgs, "Routine");
302 /* Debugging stuff */
303 void kmsg_queue_stat(void)
305 struct kernel_message *kmsg;
306 bool immed_emp, routine_emp;
307 for (int i = 0; i < num_cores; i++) {
308 spin_lock_irqsave(&per_cpu_info[i].immed_amsg_lock);
309 immed_emp = STAILQ_EMPTY(&per_cpu_info[i].immed_amsgs);
310 spin_unlock_irqsave(&per_cpu_info[i].immed_amsg_lock);
311 spin_lock_irqsave(&per_cpu_info[i].routine_amsg_lock);
312 routine_emp = STAILQ_EMPTY(&per_cpu_info[i].routine_amsgs);
313 spin_unlock_irqsave(&per_cpu_info[i].routine_amsg_lock);
314 printk("Core %d's immed_emp: %d, routine_emp %d\n", i, immed_emp,
317 kmsg = STAILQ_FIRST(&per_cpu_info[i].immed_amsgs);
318 printk("Immed msg on core %d:\n", i);
319 printk("\tsrc: %d\n", kmsg->srcid);
320 printk("\tdst: %d\n", kmsg->dstid);
321 printk("\tpc: %p\n", kmsg->pc);
322 printk("\targ0: %p\n", kmsg->arg0);
323 printk("\targ1: %p\n", kmsg->arg1);
324 printk("\targ2: %p\n", kmsg->arg2);
327 kmsg = STAILQ_FIRST(&per_cpu_info[i].routine_amsgs);
328 printk("Routine msg on core %d:\n", i);
329 printk("\tsrc: %d\n", kmsg->srcid);
330 printk("\tdst: %d\n", kmsg->dstid);
331 printk("\tpc: %p\n", kmsg->pc);
332 printk("\targ0: %p\n", kmsg->arg0);
333 printk("\targ1: %p\n", kmsg->arg1);
334 printk("\targ2: %p\n", kmsg->arg2);
340 void print_kctx_depths(const char *str)
342 uint32_t coreid = core_id();
343 struct per_cpu_info *pcpui = &per_cpu_info[coreid];
347 printk("%s: Core %d, irq depth %d, ktrap depth %d, irqon %d\n", str, coreid,
348 irq_depth(pcpui), ktrap_depth(pcpui), irq_is_enabled());
351 void print_user_ctx(struct user_context *ctx)
355 print_trapframe(&ctx->tf.hw_tf);
358 print_swtrapframe(&ctx->tf.sw_tf);
361 print_vmtrapframe(&ctx->tf.vm_tf);
364 printk("Bad TF %p type %d!\n", ctx, ctx->type);