Add the helper strtoul_from_ubuf()
[akaros.git] / kern / drivers / dev / kprof.c
index 5fe7952..cb777db 100644 (file)
@@ -7,6 +7,8 @@
  * in the LICENSE file.
  */
 
+#include <ros/profiler_records.h>
+#include <arch/time.h>
 #include <vfs.h>
 #include <slab.h>
 #include <kmalloc.h>
 #include <error.h>
 #include <pmap.h>
 #include <smp.h>
+#include <time.h>
 #include <circular_buffer.h>
 #include <umem.h>
 #include <profiler.h>
 #include <kprof.h>
+#include <ros/procinfo.h>
 
 #define KTRACE_BUFFER_SIZE (128 * 1024)
 #define TRACE_PRINTK_BUFFER_SIZE (8 * 1024)
@@ -38,12 +42,12 @@ enum {
 };
 
 struct trace_printk_buffer {
-       int in_use;
+       atomic_t in_use;
        char buffer[TRACE_PRINTK_BUFFER_SIZE];
 };
 
 struct kprof {
-       struct semaphore lock;
+       qlock_t lock;
        struct alarm_waiter *alarms;
        bool mpstat_ipi;
        bool profiling;
@@ -62,13 +66,13 @@ struct dirtab kproftab[] = {
        {"mpstat-raw",  {Kmpstatrawqid},        0,      0600},
 };
 
-extern int booting;
 static struct kprof kprof;
 static bool ktrace_init_done = FALSE;
 static spinlock_t ktrace_lock = SPINLOCK_INITIALIZER_IRQSAVE;
 static struct circular_buffer ktrace_data;
 static char ktrace_buffer[KTRACE_BUFFER_SIZE];
-static int oprof_timer_period = 1000;
+static int kprof_timer_period = 1000;
+static char kprof_control_usage[128];
 
 static size_t mpstat_len(void)
 {
@@ -96,14 +100,15 @@ static void kprof_alarm_handler(struct alarm_waiter *waiter,
        int coreid = core_id();
        struct timer_chain *tchain = &per_cpu_info[coreid].tchain;
 
-       profiler_add_hw_sample(hw_tf);
-       reset_alarm_rel(tchain, waiter, oprof_timer_period);
+       profiler_add_hw_sample(hw_tf, PROF_MKINFO(PROF_DOM_TIMER,
+                                                                                         kprof_timer_period));
+       reset_alarm_rel(tchain, waiter, kprof_timer_period);
 }
 
 static struct chan *kprof_attach(char *spec)
 {
        if (!kprof.alarms)
-               error(ENOMEM, NULL);
+               error(ENOMEM, ERROR_FIXME);
 
        return devattach(devname(), spec);
 }
@@ -116,7 +121,7 @@ static void kprof_enable_timer(int coreid, int on_off)
        if (on_off) {
                /* Per CPU waiters already inited.  Will set/reset each time (1 ms
                 * default). */
-               reset_alarm_rel(tchain, waiter, oprof_timer_period);
+               reset_alarm_rel(tchain, waiter, kprof_timer_period);
        } else {
                /* Since the alarm handler runs and gets reset within IRQ context, then
                 * we should never fail to cancel the alarm if it was already running
@@ -137,43 +142,38 @@ static void kprof_start_profiler(void)
 {
        ERRSTACK(2);
 
-       sem_down(&kprof.lock);
+       qlock(&kprof.lock);
        if (waserror()) {
-               sem_up(&kprof.lock);
+               qunlock(&kprof.lock);
                nexterror();
        }
        if (!kprof.profiling) {
-               profiler_init();
+               profiler_setup();
                if (waserror()) {
+                       kprof.profiling = FALSE;
                        profiler_cleanup();
                        nexterror();
                }
-
                profiler_control_trace(1);
-
-               for (int i = 0; i < num_cores; i++)
-                       kprof_enable_timer(i, 1);
-
                kprof.profiling = TRUE;
-
                kprof_profdata_clear();
+               poperror();
        }
        poperror();
-       poperror();
-       sem_up(&kprof.lock);
+       qunlock(&kprof.lock);
 }
 
 static void kprof_fetch_profiler_data(void)
 {
-       size_t psize = profiler_size();
+       size_t psize = kprof.psize + profiler_size();
+       char *ndata = krealloc(kprof.pdata, psize, MEM_WAIT);
 
-       kprof.pdata = kmalloc(psize, KMALLOC_WAIT);
-       if (!kprof.pdata)
-               error(ENOMEM, NULL);
-       kprof.psize = 0;
+       if (!ndata)
+               error(ENOMEM, ERROR_FIXME);
+       kprof.pdata = ndata;
        while (kprof.psize < psize) {
                size_t csize = profiler_read(kprof.pdata + kprof.psize,
-                                                                        psize - kprof.psize);
+                                            psize - kprof.psize);
 
                if (csize == 0)
                        break;
@@ -185,14 +185,12 @@ static void kprof_stop_profiler(void)
 {
        ERRSTACK(1);
 
-       sem_down(&kprof.lock);
+       qlock(&kprof.lock);
        if (waserror()) {
-               sem_up(&kprof.lock);
+               qunlock(&kprof.lock);
                nexterror();
        }
        if (kprof.profiling) {
-               for (int i = 0; i < num_cores; i++)
-                       kprof_enable_timer(i, 0);
                profiler_control_trace(0);
                kprof_fetch_profiler_data();
                profiler_cleanup();
@@ -200,7 +198,24 @@ static void kprof_stop_profiler(void)
                kprof.profiling = FALSE;
        }
        poperror();
-       sem_up(&kprof.lock);
+       qunlock(&kprof.lock);
+}
+
+static void kprof_flush_profiler(void)
+{
+       ERRSTACK(1);
+
+       qlock(&kprof.lock);
+       if (waserror()) {
+               qunlock(&kprof.lock);
+               nexterror();
+       }
+       if (kprof.profiling) {
+               profiler_trace_data_flush();
+               kprof_fetch_profiler_data();
+       }
+       poperror();
+       qunlock(&kprof.lock);
 }
 
 static void kprof_init(void)
@@ -208,15 +223,17 @@ static void kprof_init(void)
        int i;
        ERRSTACK(1);
 
-       sem_init(&kprof.lock, 1);
+       profiler_init();
+
+       qlock_init(&kprof.lock);
        kprof.profiling = FALSE;
        kprof.pdata = NULL;
        kprof.psize = 0;
 
        kprof.alarms = kzmalloc(sizeof(struct alarm_waiter) * num_cores,
-                                                       KMALLOC_WAIT);
+                               MEM_WAIT);
        if (!kprof.alarms)
-               error(ENOMEM, NULL);
+               error(ENOMEM, ERROR_FIXME);
        if (waserror()) {
                kfree(kprof.alarms);
                kprof.alarms = NULL;
@@ -232,6 +249,11 @@ static void kprof_init(void)
        kproftab[Kmpstatqid].length = mpstat_len();
        kproftab[Kmpstatrawqid].length = mpstatraw_len();
 
+       strlcpy(kprof_control_usage, "clear|start|stop|flush|timer",
+               sizeof(kprof_control_usage));
+       profiler_append_configure_usage(kprof_control_usage,
+                                       sizeof(kprof_control_usage));
+
        poperror();
 }
 
@@ -246,13 +268,13 @@ static void kprof_shutdown(void)
 
 static void kprofclear(void)
 {
-       sem_down(&kprof.lock);
+       qlock(&kprof.lock);
        kprof_profdata_clear();
-       sem_up(&kprof.lock);
+       qunlock(&kprof.lock);
 }
 
 static struct walkqid *kprof_walk(struct chan *c, struct chan *nc, char **name,
-                                                                int nname)
+                                  int nname)
 {
        return devwalk(c, nc, name, nname, kproftab, ARRAY_SIZE(kproftab), devgen);
 }
@@ -264,14 +286,14 @@ static size_t kprof_profdata_size(void)
 
 static long kprof_profdata_read(void *dest, long size, int64_t off)
 {
-       sem_down(&kprof.lock);
+       qlock(&kprof.lock);
        if (kprof.pdata && off < kprof.psize) {
                size = MIN(kprof.psize - off, size);
                memcpy(dest, kprof.pdata + off, size);
        } else {
                size = 0;
        }
-       sem_up(&kprof.lock);
+       qunlock(&kprof.lock);
 
        return size;
 }
@@ -288,7 +310,7 @@ static struct chan *kprof_open(struct chan *c, int omode)
 {
        if (c->qid.type & QTDIR) {
                if (openmode(omode) != O_READ)
-                       error(EPERM, NULL);
+                       error(EPERM, ERROR_FIXME);
        }
        c->mode = openmode(omode);
        c->flag |= COPEN;
@@ -303,7 +325,7 @@ static void kprof_close(struct chan *c)
 static long mpstat_read(void *va, long n, int64_t off)
 {
        size_t bufsz = mpstat_len();
-       char *buf = kmalloc(bufsz, KMALLOC_WAIT);
+       char *buf = kmalloc(bufsz, MEM_WAIT);
        int len = 0;
        struct per_cpu_info *pcpui;
        uint64_t cpu_total;
@@ -341,7 +363,7 @@ static long mpstat_read(void *va, long n, int64_t off)
 static long mpstatraw_read(void *va, long n, int64_t off)
 {
        size_t bufsz = mpstatraw_len();
-       char *buf = kmalloc(bufsz, KMALLOC_WAIT);
+       char *buf = kmalloc(bufsz, MEM_WAIT);
        int len = 0;
        struct per_cpu_info *pcpui;
 
@@ -350,7 +372,7 @@ static long mpstatraw_read(void *va, long n, int64_t off)
 
        /* header line: version, num_cores, tsc freq, state names */
        len += snprintf(buf + len, bufsz - len, "v%03d %5d %16llu", 1, num_cores,
-                       system_timing.tsc_freq);
+                       __proc_global_info.tsc_freq);
        for (int j = 0; j < NR_CPU_STATES; j++)
                len += snprintf(buf + len, bufsz - len, " %6s", cpu_state_names[j]);
        len += snprintf(buf + len, bufsz - len, "\n");
@@ -412,21 +434,6 @@ static void kprof_manage_timer(int coreid, struct cmdbuf *cb)
        }
 }
 
-static void kprof_usage_fail(void)
-{
-       static const char *ctlstring = "clear|start|stop|timer";
-       const char * const *cmds = profiler_configure_cmds();
-       char msgbuf[128];
-
-       strlcpy(msgbuf, ctlstring, sizeof(msgbuf));
-       for (int i = 0; cmds[i]; i++) {
-               strlcat(msgbuf, "|", sizeof(msgbuf));
-               strlcat(msgbuf, cmds[i], sizeof(msgbuf));
-       }
-
-       error(EFAIL, msgbuf);
-}
-
 static long kprof_write(struct chan *c, void *a, long n, int64_t unused)
 {
        ERRSTACK(1);
@@ -439,7 +446,7 @@ static long kprof_write(struct chan *c, void *a, long n, int64_t unused)
        switch ((int) c->qid.path) {
        case Kprofctlqid:
                if (cb->nf < 1)
-                       kprof_usage_fail();
+                       error(EFAIL, kprof_control_usage);
                if (profiler_configure(cb))
                        break;
                if (!strcmp(cb->f[0], "clear")) {
@@ -448,7 +455,7 @@ static long kprof_write(struct chan *c, void *a, long n, int64_t unused)
                        if (cb->nf < 3)
                                error(EFAIL, "timer {{all, N} {on, off}, period USEC}");
                        if (!strcmp(cb->f[1], "period")) {
-                               oprof_timer_period = strtoul(cb->f[2], 0, 10);
+                               kprof_timer_period = strtoul(cb->f[2], 0, 10);
                        } else if (!strcmp(cb->f[1], "all")) {
                                for (int i = 0; i < num_cores; i++)
                                        kprof_manage_timer(i, cb);
@@ -461,14 +468,16 @@ static long kprof_write(struct chan *c, void *a, long n, int64_t unused)
                        }
                } else if (!strcmp(cb->f[0], "start")) {
                        kprof_start_profiler();
+               } else if (!strcmp(cb->f[0], "flush")) {
+                       kprof_flush_profiler();
                } else if (!strcmp(cb->f[0], "stop")) {
                        kprof_stop_profiler();
                } else {
-                       kprof_usage_fail();
+                       error(EFAIL, kprof_control_usage);
                }
                break;
        case Kprofdataqid:
-               profiler_add_trace((uintptr_t) strtoul(a, 0, 0));
+               profiler_add_trace((uintptr_t) strtoul(a, 0, 0), 0);
                break;
        case Kptraceqid:
                if (a && (n > 0)) {
@@ -517,7 +526,7 @@ static long kprof_write(struct chan *c, void *a, long n, int64_t unused)
                }
                break;
        default:
-               error(EBADFD, NULL);
+               error(EBADFD, ERROR_FIXME);
        }
        kfree(cb);
        poperror();
@@ -546,7 +555,7 @@ void kprof_tracedata_write(const char *pretty_buf, size_t len)
        spin_lock_irqsave(&ktrace_lock);
        if (unlikely(!ktrace_init_done)) {
                circular_buffer_init(&ktrace_data, sizeof(ktrace_buffer),
-                                                        ktrace_buffer);
+                                    ktrace_buffer);
                ktrace_init_done = TRUE;
        }
        circular_buffer_write(&ktrace_data, pretty_buf, len);
@@ -557,21 +566,24 @@ static struct trace_printk_buffer *kprof_get_printk_buffer(void)
 {
        static struct trace_printk_buffer boot_tpb;
        static struct trace_printk_buffer *cpu_tpbs;
+       static atomic_t alloc_done;
 
-       if (unlikely(booting))
+       if (unlikely(!num_cores))
                return &boot_tpb;
        if (unlikely(!cpu_tpbs)) {
                /* Poor man per-CPU data structure. I really do no like littering global
                 * data structures with module specific data.
+                * We cannot take the ktrace_lock to protect the kzmalloc() call, as
+                * that might trigger printk()s, and we would reenter here.
+                * Let only one core into the kzmalloc() path, and let the others get
+                * the boot_tpb until finished.
                 */
-               spin_lock_irqsave(&ktrace_lock);
-               if (!cpu_tpbs)
-                       cpu_tpbs = kzmalloc(num_cores * sizeof(struct trace_printk_buffer),
-                                                               0);
-               spin_unlock_irqsave(&ktrace_lock);
+               if (!atomic_cas(&alloc_done, 0, 1))
+                       return &boot_tpb;
+               cpu_tpbs = kzmalloc(num_cores * sizeof(struct trace_printk_buffer), 0);
        }
 
-       return cpu_tpbs + core_id();
+       return cpu_tpbs + core_id_early();
 }
 
 void trace_vprintk(bool btrace, const char *fmt, va_list args)
@@ -610,13 +622,12 @@ void trace_vprintk(bool btrace, const char *fmt, va_list args)
        const char *utop, *uptr;
        char hdr[64];
 
-       if (tpb->in_use)
+       if (!atomic_cas(&tpb->in_use, 0, 1))
                return;
-       tpb->in_use++;
-       if (likely(!booting))
+       if (likely(__proc_global_info.tsc_freq))
                tsc2timespec(read_tsc(), &ts_now);
        snprintf(hdr, sizeof(hdr), "[%lu.%09lu]:cpu%d: ", ts_now.tv_sec,
-                        ts_now.tv_nsec, core_id());
+                ts_now.tv_nsec, core_id_early());
 
        pb.ptr = usrbuf + vsnprintf(usrbuf, usr_bufsz, fmt, args);
        pb.top = usrbuf + usr_bufsz;
@@ -646,7 +657,7 @@ void trace_vprintk(bool btrace, const char *fmt, va_list args)
                uptr = nlptr + 1;
        }
        kprof_tracedata_write(kpbuf, pb.ptr - kpbuf);
-       tpb->in_use--;
+       atomic_set(&tpb->in_use, 0);
 }
 
 void trace_printk(bool btrace, const char *fmt, ...)