bbd75f71d6443be2f06c84145be62b58e0d59c67
[akaros.git] / kern / drivers / dev / mem.c
1 /* Copyright (c) 2016 Google Inc
2  * Barret Rhoden <brho@cs.berkeley.edu>
3  * See LICENSE for details.
4  *
5  * #mem, memory diagnostics (arenas and slabs)
6  */
7
8 #include <ns.h>
9 #include <kmalloc.h>
10 #include <string.h>
11 #include <stdio.h>
12 #include <assert.h>
13 #include <error.h>
14 #include <syscall.h>
15 #include <sys/queue.h>
16
17 struct dev mem_devtab;
18
19 static char *devname(void)
20 {
21         return mem_devtab.name;
22 }
23
24 enum {
25         Qdir,
26         Qarena_stats,
27         Qslab_stats,
28         Qfree,
29         Qkmemstat,
30         Qslab_trace,
31 };
32
33 static struct dirtab mem_dir[] = {
34         {".", {Qdir, 0, QTDIR}, 0, DMDIR | 0555},
35         {"arena_stats", {Qarena_stats, 0, QTFILE}, 0, 0444},
36         {"slab_stats", {Qslab_stats, 0, QTFILE}, 0, 0444},
37         {"free", {Qfree, 0, QTFILE}, 0, 0444},
38         {"kmemstat", {Qkmemstat, 0, QTFILE}, 0, 0444},
39         {"slab_trace", {Qslab_trace, 0, QTFILE}, 0, 0444},
40 };
41
42 /* Protected by the arenas_and_slabs_lock */
43 static struct sized_alloc *slab_trace_data;
44
45 static struct chan *mem_attach(char *spec)
46 {
47         return devattach(devname(), spec);
48 }
49
50 static struct walkqid *mem_walk(struct chan *c, struct chan *nc, char **name,
51                                                                 unsigned int nname)
52 {
53         return devwalk(c, nc, name, nname, mem_dir, ARRAY_SIZE(mem_dir), devgen);
54 }
55
56 static size_t mem_stat(struct chan *c, uint8_t *db, size_t n)
57 {
58         return devstat(c, db, n, mem_dir, ARRAY_SIZE(mem_dir), devgen);
59 }
60
61 /* Prints arena's stats to the sza, adjusting the sza's sofar. */
62 static void fetch_arena_stats(struct arena *arena, struct sized_alloc *sza)
63 {
64         struct btag *bt_i;
65         struct rb_node *rb_i;
66         struct arena *a_i;
67         struct kmem_cache *kc_i;
68
69         size_t nr_allocs = 0;
70         size_t nr_imports = 0;
71         size_t amt_alloc = 0;
72         size_t amt_free = 0;
73         size_t amt_imported = 0;
74         size_t empty_hash_chain = 0;
75         size_t longest_hash_chain = 0;
76
77         sza_printf(sza, "Arena: %s (%p)\n--------------\n", arena->name, arena);
78         sza_printf(sza, "\tquantum: %d, qcache_max: %d\n", arena->quantum,
79                    arena->qcache_max);
80         sza_printf(sza, "\tsource: %s\n",
81                    arena->source ? arena->source->name : "none");
82         spin_lock_irqsave(&arena->lock);
83         for (int i = 0; i < ARENA_NR_FREE_LISTS; i++) {
84                 int j = 0;
85
86                 if (!BSD_LIST_EMPTY(&arena->free_segs[i])) {
87                         sza_printf(sza, "\tList of [2^%d - 2^%d):\n", i, i + 1);
88                         BSD_LIST_FOREACH(bt_i, &arena->free_segs[i], misc_link)
89                                 j++;
90                         sza_printf(sza, "\t\tNr free segs: %d\n", j);
91                 }
92         }
93         for (int i = 0; i < arena->hh.nr_hash_lists; i++) {
94                 int j = 0;
95
96                 if (BSD_LIST_EMPTY(&arena->alloc_hash[i]))
97                         empty_hash_chain++;
98                 BSD_LIST_FOREACH(bt_i, &arena->alloc_hash[i], misc_link)
99                         j++;
100                 longest_hash_chain = MAX(longest_hash_chain, j);
101         }
102         sza_printf(sza, "\tSegments:\n\t--------------\n");
103         for (rb_i = rb_first(&arena->all_segs); rb_i; rb_i = rb_next(rb_i)) {
104                 bt_i = container_of(rb_i, struct btag, all_link);
105                 if (bt_i->status == BTAG_SPAN) {
106                         nr_imports++;
107                         amt_imported += bt_i->size;
108                 }
109                 if (bt_i->status == BTAG_FREE)
110                         amt_free += bt_i->size;
111                 if (bt_i->status == BTAG_ALLOC) {
112                         nr_allocs++;
113                         amt_alloc += bt_i->size;
114                 }
115         }
116         sza_printf(sza, "\tStats:\n\t-----------------\n");
117         sza_printf(sza, "\t\tAmt free: %llu (%p)\n", amt_free, amt_free);
118         sza_printf(sza, "\t\tAmt alloc: %llu (%p), nr allocs %d\n",
119                    amt_alloc, amt_alloc, nr_allocs);
120         sza_printf(sza, "\t\tAmt total segs: %llu, amt alloc segs %llu\n",
121                    arena->amt_total_segs, arena->amt_alloc_segs);
122         sza_printf(sza, "\t\tAmt imported: %llu (%p), nr imports %d\n",
123                    amt_imported, amt_imported, nr_imports);
124         sza_printf(sza, "\t\tNr hash %d, empty hash: %d, longest hash %d\n",
125                    arena->hh.nr_hash_lists, empty_hash_chain,
126                    longest_hash_chain);
127         spin_unlock_irqsave(&arena->lock);
128         sza_printf(sza, "\tImporting Arenas:\n\t-----------------\n");
129         TAILQ_FOREACH(a_i, &arena->__importing_arenas, import_link)
130                 sza_printf(sza, "\t\t%s\n", a_i->name);
131         sza_printf(sza, "\tImporting Slabs:\n\t-----------------\n");
132         TAILQ_FOREACH(kc_i, &arena->__importing_slabs, import_link)
133                 sza_printf(sza, "\t\t%s\n", kc_i->name);
134 }
135
136 static struct sized_alloc *build_arena_stats(void)
137 {
138         struct sized_alloc *sza;
139         size_t alloc_amt = 0;
140         struct arena *a_i;
141
142         qlock(&arenas_and_slabs_lock);
143         /* Rough guess about how many chars per arena we'll need. */
144         TAILQ_FOREACH(a_i, &all_arenas, next)
145                 alloc_amt += 1000;
146         sza = sized_kzmalloc(alloc_amt, MEM_WAIT);
147         TAILQ_FOREACH(a_i, &all_arenas, next)
148                 fetch_arena_stats(a_i, sza);
149         qunlock(&arenas_and_slabs_lock);
150         return sza;
151 }
152
153 /* Prints arena's stats to the sza, updating its sofar. */
154 static void fetch_slab_stats(struct kmem_cache *kc, struct sized_alloc *sza)
155 {
156         struct kmem_slab *s_i;
157         struct kmem_bufctl *bc_i;
158
159         size_t nr_unalloc_objs = 0;
160         size_t empty_hash_chain = 0;
161         size_t longest_hash_chain = 0;
162
163         spin_lock_irqsave(&kc->cache_lock);
164         sza_printf(sza, "\nKmem_cache: %s\n---------------------\n", kc->name);
165         sza_printf(sza, "Source: %s\n", kc->source->name);
166         sza_printf(sza, "Objsize (incl align): %d\n", kc->obj_size);
167         sza_printf(sza, "Align: %d\n", kc->align);
168         TAILQ_FOREACH(s_i, &kc->empty_slab_list, link) {
169                 assert(!s_i->num_busy_obj);
170                 nr_unalloc_objs += s_i->num_total_obj;
171         }
172         TAILQ_FOREACH(s_i, &kc->partial_slab_list, link)
173                 nr_unalloc_objs += s_i->num_total_obj - s_i->num_busy_obj;
174         sza_printf(sza, "Nr unallocated in slab layer: %lu\n", nr_unalloc_objs);
175         sza_printf(sza, "Nr allocated from slab layer: %d\n", kc->nr_cur_alloc);
176         for (int i = 0; i < kc->hh.nr_hash_lists; i++) {
177                 int j = 0;
178
179                 if (BSD_LIST_EMPTY(&kc->alloc_hash[i]))
180                         empty_hash_chain++;
181                 BSD_LIST_FOREACH(bc_i, &kc->alloc_hash[i], link)
182                         j++;
183                 longest_hash_chain = MAX(longest_hash_chain, j);
184         }
185         sza_printf(sza, "Nr hash %d, empty hash: %d, longest hash %d, loadlim %d\n",
186                    kc->hh.nr_hash_lists, empty_hash_chain,
187                    longest_hash_chain, kc->hh.load_limit);
188         spin_unlock_irqsave(&kc->cache_lock);
189         spin_lock_irqsave(&kc->depot.lock);
190         sza_printf(sza, "Depot magsize: %d\n", kc->depot.magsize);
191         sza_printf(sza, "Nr empty mags: %d\n", kc->depot.nr_empty);
192         sza_printf(sza, "Nr non-empty mags: %d\n", kc->depot.nr_not_empty);
193         spin_unlock_irqsave(&kc->depot.lock);
194 }
195
196 static struct sized_alloc *build_slab_stats(void)
197 {
198         struct sized_alloc *sza;
199         size_t alloc_amt = 0;
200         struct kmem_cache *kc_i;
201
202         qlock(&arenas_and_slabs_lock);
203         TAILQ_FOREACH(kc_i, &all_kmem_caches, all_kmc_link)
204                 alloc_amt += 500;
205         sza = sized_kzmalloc(alloc_amt, MEM_WAIT);
206         TAILQ_FOREACH(kc_i, &all_kmem_caches, all_kmc_link)
207                 fetch_slab_stats(kc_i, sza);
208         qunlock(&arenas_and_slabs_lock);
209         return sza;
210 }
211
212 static struct sized_alloc *build_free(void)
213 {
214         struct arena *a_i;
215         struct sized_alloc *sza;
216         size_t amt_total = 0;
217         size_t amt_alloc = 0;
218
219         sza = sized_kzmalloc(500, MEM_WAIT);
220         qlock(&arenas_and_slabs_lock);
221         TAILQ_FOREACH(a_i, &all_arenas, next) {
222                 if (!a_i->is_base)
223                         continue;
224                 amt_total += a_i->amt_total_segs;
225                 amt_alloc += a_i->amt_alloc_segs;
226         }
227         qunlock(&arenas_and_slabs_lock);
228         sza_printf(sza, "Total Memory : %15llu\n", amt_total);
229         sza_printf(sza, "Used Memory  : %15llu\n", amt_alloc);
230         sza_printf(sza, "Free Memory  : %15llu\n", amt_total - amt_alloc);
231         return sza;
232 }
233
234 #define KMEMSTAT_NAME                   30
235 #define KMEMSTAT_OBJSIZE                8
236 #define KMEMSTAT_TOTAL                  15
237 #define KMEMSTAT_ALLOCED                15
238 #define KMEMSTAT_NR_ALLOCS              12
239 #define KMEMSTAT_LINE_LN (8 + KMEMSTAT_NAME + KMEMSTAT_OBJSIZE + KMEMSTAT_TOTAL\
240                           + KMEMSTAT_ALLOCED + KMEMSTAT_NR_ALLOCS)
241
242 const char kmemstat_fmt[]     = "%-*s: %c :%*llu:%*llu:%*llu:%*llu\n";
243 const char kmemstat_hdr_fmt[] = "%-*s:Typ:%*s:%*s:%*s:%*s\n";
244
245 static void fetch_arena_line(struct arena *arena, struct sized_alloc *sza,
246                              int indent)
247 {
248         for (int i = 0; i < indent; i++)
249                 sza_printf(sza, "    ");
250         sza_printf(sza, kmemstat_fmt,
251                    KMEMSTAT_NAME - indent * 4, arena->name,
252                    'A',
253                    KMEMSTAT_OBJSIZE, arena->quantum,
254                    KMEMSTAT_TOTAL, arena->amt_total_segs,
255                    KMEMSTAT_ALLOCED, arena->amt_alloc_segs,
256                    KMEMSTAT_NR_ALLOCS, arena->nr_allocs_ever);
257 }
258
259 static void fetch_slab_line(struct kmem_cache *kc, struct sized_alloc *sza,
260                             int indent)
261 {
262         struct kmem_pcpu_cache *pcc;
263         struct kmem_slab *s_i;
264         size_t nr_unalloc_objs = 0;
265         size_t nr_allocs_ever = 0;
266
267         spin_lock_irqsave(&kc->cache_lock);
268         TAILQ_FOREACH(s_i, &kc->empty_slab_list, link)
269                 nr_unalloc_objs += s_i->num_total_obj;
270         TAILQ_FOREACH(s_i, &kc->partial_slab_list, link)
271                 nr_unalloc_objs += s_i->num_total_obj - s_i->num_busy_obj;
272         nr_allocs_ever = kc->nr_direct_allocs_ever;
273         spin_unlock_irqsave(&kc->cache_lock);
274         /* Lockless peak at the pcpu state */
275         for (int i = 0; i < kmc_nr_pcpu_caches(); i++) {
276                 pcc = &kc->pcpu_caches[i];
277                 nr_allocs_ever += pcc->nr_allocs_ever;
278         }
279
280         for (int i = 0; i < indent; i++)
281                 sza_printf(sza, "    ");
282         sza_printf(sza, kmemstat_fmt,
283                    KMEMSTAT_NAME - indent * 4, kc->name,
284                    'S',
285                    KMEMSTAT_OBJSIZE, kc->obj_size,
286                    KMEMSTAT_TOTAL, kc->obj_size * (nr_unalloc_objs +
287                                                    kc->nr_cur_alloc),
288                    KMEMSTAT_ALLOCED, kc->obj_size * kc->nr_cur_alloc,
289                    KMEMSTAT_NR_ALLOCS, nr_allocs_ever);
290 }
291
292 static void fetch_arena_and_kids(struct arena *arena, struct sized_alloc *sza,
293                                  int indent)
294 {
295         struct arena *a_i;
296         struct kmem_cache *kc_i;
297
298         fetch_arena_line(arena, sza, indent);
299         TAILQ_FOREACH(a_i, &arena->__importing_arenas, import_link)
300                 fetch_arena_and_kids(a_i, sza, indent + 1);
301         TAILQ_FOREACH(kc_i, &arena->__importing_slabs, import_link)
302                 fetch_slab_line(kc_i, sza, indent + 1);
303 }
304
305 static struct sized_alloc *build_kmemstat(void)
306 {
307         struct arena *a_i;
308         struct kmem_cache *kc_i;
309         struct sized_alloc *sza;
310         size_t alloc_amt = 100;
311
312         qlock(&arenas_and_slabs_lock);
313         TAILQ_FOREACH(a_i, &all_arenas, next)
314                 alloc_amt += 100;
315         TAILQ_FOREACH(kc_i, &all_kmem_caches, all_kmc_link)
316                 alloc_amt += 100;
317         sza = sized_kzmalloc(alloc_amt, MEM_WAIT);
318         sza_printf(sza, kmemstat_hdr_fmt,
319                    KMEMSTAT_NAME, "Arena/Slab Name",
320                    KMEMSTAT_OBJSIZE, "Objsize",
321                    KMEMSTAT_TOTAL, "Total Amt",
322                    KMEMSTAT_ALLOCED, "Alloc Amt",
323                    KMEMSTAT_NR_ALLOCS, "Allocs Ever");
324         for (int i = 0; i < KMEMSTAT_LINE_LN; i++)
325                 sza_printf(sza, "-");
326         sza_printf(sza, "\n");
327         TAILQ_FOREACH(a_i, &all_arenas, next) {
328                 if (a_i->source)
329                         continue;
330                 fetch_arena_and_kids(a_i, sza, 0);
331         }
332         qunlock(&arenas_and_slabs_lock);
333         return sza;
334 }
335
336 static struct chan *mem_open(struct chan *c, int omode)
337 {
338         if (c->qid.type & QTDIR) {
339                 if (openmode(omode) != O_READ)
340                         error(EPERM, "Tried opening directory not read-only");
341         }
342         switch (c->qid.path) {
343         case Qarena_stats:
344                 c->synth_buf = build_arena_stats();
345                 break;
346         case Qslab_stats:
347                 c->synth_buf = build_slab_stats();
348                 break;
349         case Qfree:
350                 c->synth_buf = build_free();
351                 break;
352         case Qkmemstat:
353                 c->synth_buf = build_kmemstat();
354                 break;
355         case Qslab_trace:
356                 /* slab_trace is built on write, not on open. */
357                 assert(!c->synth_buf);
358                 break;
359         }
360         c->mode = openmode(omode);
361         c->flag |= COPEN;
362         c->offset = 0;
363         return c;
364 }
365
366 static void mem_close(struct chan *c)
367 {
368         if (!(c->flag & COPEN))
369                 return;
370         switch (c->qid.path) {
371         case Qarena_stats:
372         case Qslab_stats:
373         case Qfree:
374         case Qkmemstat:
375         case Qslab_trace:
376                 kfree(c->synth_buf);
377                 c->synth_buf = NULL;
378                 break;
379         }
380 }
381
382 static size_t slab_trace_read(struct chan *c, void *ubuf, size_t n,
383                               off64_t offset)
384 {
385         size_t ret = 0;
386
387         qlock(&arenas_and_slabs_lock);
388         if (slab_trace_data)
389                 ret = readstr(offset, ubuf, n, slab_trace_data->buf);
390         qunlock(&arenas_and_slabs_lock);
391         return ret;
392 }
393
394 static size_t mem_read(struct chan *c, void *ubuf, size_t n, off64_t offset)
395 {
396         struct sized_alloc *sza;
397
398         switch (c->qid.path) {
399         case Qdir:
400                 return devdirread(c, ubuf, n, mem_dir, ARRAY_SIZE(mem_dir),
401                                                   devgen);
402         case Qarena_stats:
403         case Qslab_stats:
404         case Qfree:
405         case Qkmemstat:
406                 sza = c->synth_buf;
407                 return readstr(offset, ubuf, n, sza->buf);
408         case Qslab_trace:
409                 return slab_trace_read(c, ubuf, n, offset);
410         default:
411                 panic("Bad Qid %p!", c->qid.path);
412         }
413         return -1;
414 }
415
416 /* start, then stop, then print, then read to get the trace */
417 #define SLAB_TRACE_USAGE "start|stop|print|reset SLAB_NAME"
418
419 static void slab_trace_cmd(struct chan *c, struct cmdbuf *cb)
420 {
421         ERRSTACK(1);
422         struct sized_alloc *sza, *old_sza;
423         struct kmem_cache *kc;
424
425         if (cb->nf < 2)
426                 error(EFAIL, SLAB_TRACE_USAGE);
427
428         qlock(&arenas_and_slabs_lock);
429         if (waserror()) {
430                 qunlock(&arenas_and_slabs_lock);
431                 nexterror();
432         }
433         TAILQ_FOREACH(kc, &all_kmem_caches, all_kmc_link)
434                 if (!strcmp(kc->name, cb->f[1]))
435                         break;
436         if (!kc)
437                 error(ENOENT, "No such slab %s", cb->f[1]);
438         /* Note that the only time we have a real sza is when printing.
439          * Otherwise, it's NULL.  We still want this to be the chan's sza, since
440          * the reader should get nothing back until they ask for a print. */
441         sza = NULL;
442         if (!strcmp(cb->f[0], "start")) {
443                 if (kmem_trace_start(kc))
444                         error(EFAIL, "Unable to trace slab %s", kc->name);
445         } else if (!strcmp(cb->f[0], "stop")) {
446                 kmem_trace_stop(kc);
447         } else if (!strcmp(cb->f[0], "print")) {
448                 sza = kmem_trace_print(kc);
449         } else if (!strcmp(cb->f[0], "reset")) {
450                 kmem_trace_reset(kc);
451         } else {
452                 error(EFAIL, SLAB_TRACE_USAGE);
453         }
454         old_sza = slab_trace_data;
455         slab_trace_data = sza;
456         qunlock(&arenas_and_slabs_lock);
457         poperror();
458         kfree(old_sza);
459 }
460
461 static size_t mem_write(struct chan *c, void *ubuf, size_t n, off64_t unused)
462 {
463         ERRSTACK(1);
464         struct cmdbuf *cb = parsecmd(ubuf, n);
465
466         if (waserror()) {
467                 kfree(cb);
468                 nexterror();
469         }
470         switch (c->qid.path) {
471         case Qslab_trace:
472                 slab_trace_cmd(c, cb);
473                 break;
474         default:
475                 error(EFAIL, "Unable to write to %s", devname());
476         }
477         kfree(cb);
478         poperror();
479         return n;
480 }
481
482 struct dev mem_devtab __devtab = {
483         .name = "mem",
484         .reset = devreset,
485         .init = devinit,
486         .shutdown = devshutdown,
487         .attach = mem_attach,
488         .walk = mem_walk,
489         .stat = mem_stat,
490         .open = mem_open,
491         .create = devcreate,
492         .close = mem_close,
493         .read = mem_read,
494         .bread = devbread,
495         .write = mem_write,
496         .bwrite = devbwrite,
497         .remove = devremove,
498         .wstat = devwstat,
499         .power = devpower,
500         .chaninfo = devchaninfo,
501 };