First, second, and third pass at apic code. Crude
[akaros.git] / tests / vmm / vmrunkernel.c
1 #include <stdio.h> 
2 #include <pthread.h>
3 #include <sys/types.h>
4 #include <sys/stat.h>
5 #include <fcntl.h>
6 #include <parlib/arch/arch.h>
7 #include <parlib/ros_debug.h>
8 #include <unistd.h>
9 #include <errno.h>
10 #include <dirent.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include <ros/syscall.h>
14 #include <sys/mman.h>
15 #include <vmm/coreboot_tables.h>
16 #include <vmm/vmm.h>
17 #include <acpi/acpi.h>
18 #include <ros/arch/mmu.h>
19 #include <ros/vmx.h>
20 #include <parlib/uthread.h>
21 #include <vmm/virtio.h>
22 #include <vmm/virtio_mmio.h>
23 #include <vmm/virtio_ids.h>
24 #include <vmm/virtio_config.h>
25
26 /* Kind of sad what a total clusterf the pc world is. By 1999, you could just scan the hardware 
27  * and work it out. But 2005, that was no longer possible. How sad. 
28  * so we have to fake acpi to make it all work. !@#$!@#$#.
29  * This will be copied to memory at 0xe0000, so the kernel can find it.
30  */
31 /* assume they're all 256 bytes long just to make it easy. Just have pointers that point to aligned things. */
32
33 struct acpi_table_rsdp rsdp = {
34         .signature = "RSD PTR ",
35         .oem_id = "AKAROS",
36         .revision = 2,
37         .length = 36,
38 };
39
40 struct acpi_table_xsdt xsdt = {
41         .header = {
42                 .signature= "XSDT",
43                 // This is so stupid. Incredibly stupid.
44                 .revision = 0,
45                 .oem_id = "AKAROS",
46                 .oem_table_id = "ALPHABET",
47                 .oem_revision = 0,
48                 .asl_compiler_id = "RON ",
49                 .asl_compiler_revision = 0,
50         },
51 };
52 struct acpi_table_fadt fadt = {
53         .header = {
54                 .signature= "FADT",
55                 // This is so stupid. Incredibly stupid.
56                 .revision = 0,
57                 .oem_id = "AKAROS",
58                 .oem_table_id = "ALPHABET",
59                 .oem_revision = 0,
60                 .asl_compiler_id = "RON ",
61                 .asl_compiler_revision = 0,
62         },
63 };
64
65 /* This has to be dropped into memory, then the other crap just follows it.
66  */
67 struct acpi_table_madt madt = {
68         .header = {
69                 .signature = "APIC",
70                 .revision = 0,
71                 .oem_id = "AKAROS",
72                 .oem_table_id = "ALPHABET",
73                 .oem_revision = 0,
74                 .asl_compiler_id = "RON ",
75                 .asl_compiler_revision = 0,
76         },
77         
78         .address = 0xfee00000ULL,
79 };
80
81 struct acpi_madt_local_apic Apic0 = {.header = {.type = ACPI_MADT_TYPE_LOCAL_APIC, .length = sizeof(struct acpi_madt_local_apic)},
82                                      .processor_id = 0, .id = 0};
83 struct acpi_madt_io_apic Apic1 = {.header = {.type = ACPI_MADT_TYPE_IO_APIC, .length = sizeof(struct acpi_madt_io_apic)},
84                                   .id = 1, .address = 0xfec00000, .global_irq_base = 0};
85
86 /* this test will run the "kernel" in the negative address space. We hope. */
87 void *low1m;
88 uint8_t low4k[4096];
89 unsigned long long stack[1024];
90 volatile int shared = 0;
91 volatile int quit = 0;
92 int mcp = 1;
93
94 /* total hack. If the vm runs away we want to get control again. */
95 unsigned int maxresume = (unsigned int) -1;
96
97 #define MiB 0x100000u
98 #define GiB (1u<<30)
99 #define GKERNBASE (16*MiB)
100 #define KERNSIZE (128*MiB+GKERNBASE)
101 uint8_t _kernel[KERNSIZE];
102
103 unsigned long long *p512, *p1, *p2m;
104
105 void **my_retvals;
106 int nr_threads = 3;
107 int debug = 0;
108 int resumeprompt = 0;
109 /* unlike Linux, this shared struct is for both host and guest. */
110 //      struct virtqueue *constoguest = 
111 //              vring_new_virtqueue(0, 512, 8192, 0, inpages, NULL, NULL, "test");
112 uint64_t virtio_mmio_base = 0x100000000;
113
114 void *consout(void *arg)
115 {
116         char *line, *consline, *outline;
117         static struct scatterlist out[] = { {NULL, sizeof(outline)}, };
118         static struct scatterlist in[] = { {NULL, sizeof(line)}, };
119         static struct scatterlist iov[32];
120         struct virtio_threadarg *a = arg;
121         static unsigned int inlen, outlen, conslen;
122         struct virtqueue *v = a->arg->virtio;
123         fprintf(stderr, "talk thread ..\n");
124         uint16_t head, gaveit = 0, gotitback = 0;
125         uint32_t vv;
126         int i;
127         int num;
128         if (debug) {
129                 printf("----------------------- TT a %p\n", a);
130                 printf("talk thread ttargs %x v %x\n", a, v);
131         }
132         
133         for(num = 0;;num++) {
134                 /* host: use any buffers we should have been sent. */
135                 head = wait_for_vq_desc(v, iov, &outlen, &inlen);
136                 if (debug)
137                         printf("CCC: vq desc head %d, gaveit %d gotitback %d\n", head, gaveit, gotitback);
138                 for(i = 0; debug && i < outlen + inlen; i++)
139                         printf("CCC: v[%d/%d] v %p len %d\n", i, outlen + inlen, iov[i].v, iov[i].length);
140                 /* host: if we got an output buffer, just output it. */
141                 for(i = 0; i < outlen; i++) {
142                         num++;
143                         int j;
144                         for (j = 0; j < iov[i].length; j++)
145                                 printf("%c", ((char *)iov[i].v)[j]);
146                 }
147                 
148                 if (debug)
149                         printf("CCC: outlen is %d; inlen is %d\n", outlen, inlen);
150                 /* host: fill in the writeable buffers. */
151                 /* why we're getting these I don't know. */
152                 for (i = outlen; i < outlen + inlen; i++) {
153                         if (debug) fprintf(stderr, "CCC: send back empty writeable");
154                         iov[i].length = 0;
155                 }
156                 if (debug) printf("CCC: call add_used\n");
157                 /* host: now ack that we used them all. */
158                 add_used(v, head, outlen+inlen);
159                 if (debug) printf("CCC: DONE call add_used\n");
160         }
161         fprintf(stderr, "All done\n");
162         return NULL;
163 }
164
165 void *consin(void *arg)
166 {
167         struct virtio_threadarg *a = arg;
168         char *line, *outline;
169         static char consline[128];
170         static struct scatterlist iov[32];
171         static struct scatterlist out[] = { {NULL, sizeof(outline)}, };
172         static struct scatterlist in[] = { {NULL, sizeof(line)}, };
173
174         static unsigned int inlen, outlen, conslen;
175         struct virtqueue *v = a->arg->virtio;
176         fprintf(stderr, "consin thread ..\n");
177         uint16_t head, gaveit = 0, gotitback = 0;
178         uint32_t vv;
179         int i;
180         int num;
181         
182         if (debug) printf("Spin on console being read, print num queues, halt\n");
183
184         for(num = 0;! quit;num++) {
185                 int debug = 1;
186                 /* host: use any buffers we should have been sent. */
187                 head = wait_for_vq_desc(v, iov, &outlen, &inlen);
188                 if (debug)
189                         printf("vq desc head %d, gaveit %d gotitback %d\n", head, gaveit, gotitback);
190                 for(i = 0; debug && i < outlen + inlen; i++)
191                         printf("v[%d/%d] v %p len %d\n", i, outlen + inlen, iov[i].v, iov[i].length);
192                 if (debug)
193                         printf("outlen is %d; inlen is %d\n", outlen, inlen);
194                 /* host: fill in the writeable buffers. */
195                 for (i = outlen; i < outlen + inlen; i++) {
196                         /* host: read a line. */
197                         memset(consline, 0, 128);
198                         if (fgets(consline, 4096-256, stdin) == NULL) {
199                                 exit(0);
200                         } 
201                         if (debug) printf("GOT A LINE:%s:\n", consline);
202                         if (strlen(consline) < 3 && consline[0] == 'q' ) {
203                                 quit = 1;
204                                 break;
205                         }
206
207                         memmove(iov[i].v, consline, strlen(consline)+ 1);
208                         iov[i].length = strlen(consline) + 1;
209                 }
210                 if (debug) printf("call add_used\n");
211                 /* host: now ack that we used them all. */
212                 add_used(v, head, outlen+inlen);
213                 if (debug) printf("DONE call add_used\n");
214         }
215         fprintf(stderr, "All done\n");
216         return NULL;
217 }
218
219 static struct vqdev vqdev= {
220 name: "console",
221 dev: VIRTIO_ID_CONSOLE,
222 device_features: 0, /* Can't do it: linux console device does not support it. VIRTIO_F_VERSION_1*/
223 numvqs: 2,
224 vqs: {
225                 {name: "consin", maxqnum: 64, f: &consin, arg: (void *)0},
226                 {name: "consout", maxqnum: 64, f: consout, arg: (void *)0},
227         }
228 };
229
230 void lowmem() {
231         __asm__ __volatile__ (".section .lowmem, \"aw\"\n\tlow: \n\t.=0x1000\n\t.align 0x100000\n\t.previous\n");
232 }
233
234 static uint8_t acpi_tb_checksum(uint8_t *buffer, uint32_t length)
235 {
236         uint8_t sum = 0;
237         uint8_t *end = buffer + length;
238         printf("tbchecksum %p for %d", buffer, length);
239         while (buffer < end) {
240                 if (end - buffer < 2)
241                         printf("%02x\n", sum);
242                 sum = (uint8_t)(sum + *(buffer++));
243         }
244         printf(" is %02x\n", sum);
245         return (sum);
246 }
247
248 static void gencsum(uint8_t *target, void *data, int len)
249 {
250         uint8_t csum;
251         // blast target to zero so it does not get counted (it might be in the struct we checksum) 
252         // And, yes, it is, goodness.
253         printf("gencsum %p target %p source %d bytes\n", target, data, len);
254         *target = 0;
255         csum  = acpi_tb_checksum((uint8_t *)data, len);
256         *target = ~csum + 1;
257         printf("Cmoputed is %02x\n", *target);
258 }
259
260 int main(int argc, char **argv)
261 {
262         void *a = (void *)0xe0000;
263         struct acpi_table_rsdp *r;
264         struct acpi_table_fadt *f;
265         struct acpi_table_madt *m;
266         struct acpi_table_xsdt *x;
267         uint64_t virtiobase = 0x100000000ULL;
268         // lowmem is a bump allocated pointer to 2M at the "physbase" of memory 
269         void *lowmem = (void *) 0x1000000;
270         struct vmctl vmctl;
271         int amt;
272         int vmmflags = 0; // Disabled probably forever. VMM_VMCALL_PRINTF;
273         uint64_t entry = 0x1200000, kerneladdress = 0x1200000;
274         int nr_gpcs = 1;
275         int fd = open("#c/vmctl", O_RDWR), ret;
276         void * xp;
277         int kfd = -1;
278         static char cmd[512];
279         int i;
280         uint8_t csum;
281         void *coreboot_tables = (void *) 0x1165000;
282 printf("%p %p %p %p\n", PGSIZE, PGSHIFT, PML1_SHIFT, PML1_PTE_REACH);
283
284         // mmap is not working for us at present.
285         if ((uint64_t)_kernel > GKERNBASE) {
286                 printf("kernel array @%p is above , GKERNBASE@%p sucks\n", _kernel, GKERNBASE);
287                 exit(1);
288         }
289         memset(_kernel, 0, sizeof(_kernel));
290         memset(lowmem, 0xff, 2*1048576);
291         memset(low4k, 0xff, 4096);
292         // avoid at all costs, requires too much instruction emulation.
293         //low4k[0x40e] = 0;
294         //low4k[0x40f] = 0xe0;
295
296         if (fd < 0) {
297                 perror("#cons/sysctl");
298                 exit(1);
299         }
300         argc--,argv++;
301         // switches ...
302         // Sorry, I don't much like the gnu opt parsing code.
303         while (1) {
304                 if (*argv[0] != '-')
305                         break;
306                 switch(argv[0][1]) {
307                 case 'd':
308                         debug++;
309                         break;
310                 case 'v':
311                         vmmflags |= VMM_VMCALL_PRINTF;
312                         break;
313                 case 'm':
314                         argc--,argv++;
315                         maxresume = strtoull(argv[0], 0, 0);
316                         break;
317                 default:
318                         printf("BMAFR\n");
319                         break;
320                 }
321                 argc--,argv++;
322         }
323         if (argc < 1) {
324                 fprintf(stderr, "Usage: %s vmimage [-n (no vmcall printf)] [coreboot_tables [loadaddress [entrypoint]]]\n", argv[0]);
325                 exit(1);
326         }
327         if (argc > 1)
328                 coreboot_tables = (void *) strtoull(argv[1], 0, 0);
329         if (argc > 2)
330                 kerneladdress = strtoull(argv[2], 0, 0);
331         if (argc > 3)
332                 entry = strtoull(argv[3], 0, 0);
333         kfd = open(argv[0], O_RDONLY);
334         if (kfd < 0) {
335                 perror(argv[0]);
336                 exit(1);
337         }
338         // read in the kernel.
339         xp = (void *)kerneladdress;
340         for(;;) {
341                 amt = read(kfd, xp, 1048576);
342                 if (amt < 0) {
343                         perror("read");
344                         exit(1);
345                 }
346                 if (amt == 0) {
347                         break;
348                 }
349                 xp += amt;
350         }
351         fprintf(stderr, "Read in %d bytes\n", xp-kerneladdress);
352         close(kfd);
353
354         // The low 1m so we can fill in bullshit like ACPI. */
355         low1m = mmap((int*)4096, MiB-4096, PROT_READ | PROT_WRITE,
356                          MAP_ANONYMOUS, -1, 0);
357         if (low1m != (void *)4096) {
358                 perror("Unable to mmap low 1m");
359                 exit(1);
360         }
361         memset(low1m, 0xff, MiB-4096);
362         r = a;
363         fprintf(stderr, "install rsdp to %p\n", r);
364         *r = rsdp;
365         a += sizeof(*r);
366         memmove(&r->xsdt_physical_address, &a, sizeof(a));
367         gencsum(&r->checksum, r, ACPI_RSDP_CHECKSUM_LENGTH);
368         if ((csum = acpi_tb_checksum((uint8_t *) r, ACPI_RSDP_CHECKSUM_LENGTH)) != 0) {
369                 printf("RSDP has bad checksum; summed to %x\n", csum);
370                 exit(1);
371         }
372
373         /* Check extended checksum if table version >= 2 */
374         gencsum(&r->extended_checksum, r, ACPI_RSDP_XCHECKSUM_LENGTH);
375         if ((rsdp.revision >= 2) &&
376             (acpi_tb_checksum((uint8_t *) r, ACPI_RSDP_XCHECKSUM_LENGTH) != 0)) {
377                 printf("RSDP has bad checksum v2\n");
378                 exit(1);
379         }
380
381         /* just leave a bunch of space for the xsdt. */
382         /* we need to zero the area since it has pointers. */
383         x = a;
384         a += sizeof(*x) + 8*sizeof(void *);
385         memset(x, 0, a - (void *)x);
386         fprintf(stderr, "install xsdt to %p\n", x);
387         *x = xsdt;
388         x->table_offset_entry[0] = 0;
389         x->table_offset_entry[1] = 0;
390         x->header.length = a - (void *)x;
391
392         f = a;
393         fprintf(stderr, "install fadt to %p\n", f);
394         *f = fadt;
395         x->table_offset_entry[2] = (uint64_t) f;
396         a += sizeof(*f);
397         f->header.length = a - (void *)f;
398         gencsum(&f->header.checksum, f, f->header.length);
399         if (acpi_tb_checksum((uint8_t *)f, f->header.length) != 0) {
400                 printf("ffadt has bad checksum v2\n");
401                 exit(1);
402         }
403
404         m = a;
405         *m = madt;
406         x->table_offset_entry[3] = (uint64_t) m;
407         a += sizeof(*m);
408         fprintf(stderr, "install madt to %p\n", m);
409         memmove(a, &Apic0, sizeof(Apic0));
410         a += sizeof(Apic0);
411         memmove(a, &Apic1, sizeof(Apic1));
412         a += sizeof(Apic1);
413         m->header.length = a - (void *)m;
414         gencsum(&m->header.checksum, m, m->header.length);
415         if (acpi_tb_checksum((uint8_t *) m, m->header.length) != 0) {
416                 printf("madt has bad checksum v2\n");
417                 exit(1);
418         }
419         fprintf(stderr, "allchecksums ok\n");
420
421         gencsum(&x->header.checksum, x, x->header.length);
422         if ((csum = acpi_tb_checksum((uint8_t *) x, x->header.length)) != 0) {
423                 printf("XSDT has bad checksum; summed to %x\n", csum);
424                 exit(1);
425         }
426
427         hexdump(stdout, r, a-(void *)r);
428
429         if (ros_syscall(SYS_setup_vmm, nr_gpcs, vmmflags, 0, 0, 0, 0) != nr_gpcs) {
430                 perror("Guest pcore setup failed");
431                 exit(1);
432         }
433
434         fprintf(stderr, "Run with %d cores and vmmflags 0x%x\n", nr_gpcs, vmmflags);
435         mcp = 1;
436         if (mcp) {
437                 my_retvals = malloc(sizeof(void*) * nr_threads);
438                 if (!my_retvals)
439                         perror("Init threads/malloc");
440
441                 pthread_can_vcore_request(FALSE);       /* 2LS won't manage vcores */
442                 pthread_need_tls(FALSE);
443                 pthread_mcp_init();                                     /* gives us one vcore */
444                 vcore_request(nr_threads - 1);          /* ghetto incremental interface */
445                 for (int i = 0; i < nr_threads; i++) {
446                         xp = __procinfo.vcoremap;
447                         printf("%p\n", __procinfo.vcoremap);
448                         printf("Vcore %d mapped to pcore %d\n", i,
449                                 __procinfo.vcoremap[i].pcoreid);
450                 }
451         }
452
453         ret = syscall(33, 1);
454         if (ret < 0) {
455                 perror("vm setup");
456                 exit(1);
457         }
458         ret = posix_memalign((void **)&p512, 4096, 3*4096);
459         printf("memalign is %p\n", p512);
460         if (ret) {
461                 perror("ptp alloc");
462                 exit(1);
463         }
464         p1 = &p512[512];
465         p2m = &p512[1024];
466         uint64_t kernbase = 0; //0xffffffff80000000;
467         uint64_t highkernbase = 0xffffffff80000000;
468         p512[PML4(kernbase)] = (unsigned long long)p1 | 7;
469         p1[PML3(kernbase)] = /*0x87; */(unsigned long long)p2m | 7;
470         p512[PML4(highkernbase)] = (unsigned long long)p1 | 7;
471         p1[PML3(highkernbase)] = /*0x87; */(unsigned long long)p2m | 7;
472 #define _2MiB (0x200000)
473
474         for (i = 0; i < 512; i++) {
475                 p2m[PML2(kernbase + i * _2MiB)] = 0x87 | i * _2MiB;
476         }
477
478         kernbase >>= (0+12);
479         kernbase <<= (0 + 12);
480         uint8_t *kernel = (void *)GKERNBASE;
481         //write_coreboot_table(coreboot_tables, ((void *)VIRTIOBASE) /*kernel*/, KERNSIZE + 1048576);
482         hexdump(stdout, coreboot_tables, 512);
483         printf("kernbase for pml4 is 0x%llx and entry is %llx\n", kernbase, entry);
484         printf("p512 %p p512[0] is 0x%lx p1 %p p1[0] is 0x%x\n", p512, p512[0], p1, p1[0]);
485         vmctl.command = REG_RSP_RIP_CR3;
486         vmctl.cr3 = (uint64_t) p512;
487         vmctl.regs.tf_rip = entry;
488         vmctl.regs.tf_rsp = (uint64_t) &stack[1024];
489         if (mcp) {
490                 /* set up virtio bits, which depend on threads being enabled. */
491                 register_virtio_mmio(&vqdev, virtio_mmio_base);
492         }
493         printf("threads started\n");
494         printf("Writing command :%s:\n", cmd);
495
496         ret = write(fd, &vmctl, sizeof(vmctl));
497         if (ret != sizeof(vmctl)) {
498                 perror(cmd);
499         }
500         while (1) {
501                 void showstatus(FILE *f, struct vmctl *v);
502                 int c;
503                 uint8_t byte;
504                 vmctl.command = REG_RIP;
505                 if (maxresume-- == 0) {
506                         debug = 1;
507                         resumeprompt = 1;
508                 }
509                 if (debug) {
510                         printf("RIP %p, shutdown 0x%x\n", vmctl.regs.tf_rip, vmctl.shutdown);
511                         showstatus(stdout, &vmctl);
512                 }
513                 if (resumeprompt) {
514                         printf("RESUME?\n");
515                         c = getchar();
516                         if (c == 'q')
517                                 break;
518                 }
519                 if (vmctl.shutdown == SHUTDOWN_EPT_VIOLATION) {
520                         uint64_t gpa, *regp, val;
521                         uint8_t regx;
522                         int store, size;
523                         int advance;
524                         if (decode(&vmctl, &gpa, &regx, &regp, &store, &size, &advance)) {
525                                 printf("RIP %p, shutdown 0x%x\n", vmctl.regs.tf_rip, vmctl.shutdown);
526                                 showstatus(stdout, &vmctl);
527                                 quit = 1;
528                                 break;
529                         }
530                         if (debug) printf("%p %p %p %p %p %p\n", gpa, regx, regp, store, size, advance);
531                         if ((gpa & ~0xfffULL) == virtiobase) {
532                                 if (debug) printf("DO SOME VIRTIO\n");
533                                 // Lucky for us the various virtio ops are well-defined.
534                                 virtio_mmio(&vmctl, gpa, regx, regp, store);
535                         } else if ((gpa & 0xfee00000) == 0xfee00000) {
536                                 // until we fix our include mess, just put the proto here.
537                                 int apic(struct vmctl *v, uint64_t gpa, int destreg, uint64_t *regp, int store);
538                                 apic(&vmctl, gpa, regx, regp, store);
539                         } else if ((gpa & 0xfec00000) == 0xfec00000) {
540                                 // until we fix our include mess, just put the proto here.
541                                 int ioapic(struct vmctl *v, uint64_t gpa, int destreg, uint64_t *regp, int store);
542                                 ioapic(&vmctl, gpa, regx, regp, store);
543                         } else if (gpa < 4096) {
544                                 uint64_t val = 0;
545                                 memmove(&val, &low4k[gpa], size);
546                                 hexdump(stdout, &low4k[gpa], size);
547                                 printf("Low 1m, code %p read @ %p, size %d, val %p\n", vmctl.regs.tf_rip, gpa, size, val);
548                                 memmove(regp, &low4k[gpa], size);
549                                 hexdump(stdout, regp, size);
550                         } else {
551                                 printf("EPT violation: can't handle %p\n", gpa);
552                                 printf("RIP %p, shutdown 0x%x\n", vmctl.regs.tf_rip, vmctl.shutdown);
553                                 showstatus(stdout, &vmctl);
554                                 quit = 1;
555                                 break;
556                         }
557                         vmctl.regs.tf_rip += advance;
558                         if (debug) printf("Advance rip by %d bytes to %p\n", advance, vmctl.regs.tf_rip);
559                         vmctl.shutdown = 0;
560                         vmctl.gpa = 0;
561                         vmctl.command = REG_ALL;
562                 } else if (vmctl.shutdown == SHUTDOWN_UNHANDLED_EXIT_REASON) {
563                         switch(vmctl.ret_code){
564                         case  EXIT_REASON_VMCALL:
565                                 byte = vmctl.regs.tf_rdi;
566                                 printf("%c", byte);
567                                 if (byte == '\n') printf("%c", 'V');
568                                 vmctl.regs.tf_rip += 3;
569                                 break;
570                         case EXIT_REASON_EXTERNAL_INTERRUPT:
571                                 //debug = 1;
572                                 fprintf(stderr, "XINT 0x%x 0x%x\n", vmctl.intrinfo1, vmctl.intrinfo2);
573                                 vmctl.interrupt = 0x80000302; // b0d;
574                                 // That sent an NMI and we got it.
575
576                                 vmctl.interrupt = 0x80000320; // b0d;
577                                 // This fails on entry
578                                 
579                                 vmctl.interrupt = 0x80000306; // b0d;
580                                 // This succeedd in sending a UD.
581
582                                 vmctl.interrupt = 0x8000030f; // b0d;
583                                 
584                                 vmctl.command = RESUME;
585                                 break;
586                         case EXIT_REASON_IO_INSTRUCTION:
587                                 printf("IO @ %p\n", vmctl.regs.tf_rip);
588                                 io(&vmctl);
589                                 vmctl.shutdown = 0;
590                                 vmctl.gpa = 0;
591                                 vmctl.command = REG_ALL;
592                                 break;
593                         case EXIT_REASON_HLT:
594                                 printf("\n================== Guest halted. RIP. =======================\n");
595                                 quit = 1;
596                                 break;
597                         default:
598                                 fprintf(stderr, "Don't know how to handle exit %d\n", vmctl.ret_code);
599                                 quit = 1;
600                                 break;
601                         }
602                 }
603                 if (debug) printf("at bottom of switch, quit is %d\n", quit);
604                 if (quit)
605                         break;
606                 if (debug) printf("NOW DO A RESUME\n");
607                 ret = write(fd, &vmctl, sizeof(vmctl));
608                 if (ret != sizeof(vmctl)) {
609                         perror(cmd);
610                 }
611         }
612
613         /* later. 
614         for (int i = 0; i < nr_threads-1; i++) {
615                 int ret;
616                 if (pthread_join(my_threads[i], &my_retvals[i]))
617                         perror("pth_join failed");
618                 printf("%d %d\n", i, ret);
619         }
620  */
621
622         return 0;
623 }