kern/arch/x86/vmx_mmu.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * This module enables machines with Intel VT-x extensions to run virtual
   5  * machines without emulation or binary translation.
   6  *
   7  * MMU support
   8  *
   9  * Copyright (C) 2006 Qumranet, Inc.
  10  *
  11  * Authors:
  12  *   Yaniv Kamay  <yaniv@qumranet.com>
  13  *   Avi Kivity   <avi@qumranet.com>
  14  *
  15  */
  16 #define DEBUG
  17 #include <kmalloc.h>
  18 #include <string.h>
  19 #include <stdio.h>
  20 #include <assert.h>
  21 #include <error.h>
  22 #include <pmap.h>
  23 #include <sys/queue.h>
  24 #include <smp.h>
  25 #include <kref.h>
  26 #include <atomic.h>
  27 #include <alarm.h>
  28 #include <event.h>
  29 #include <umem.h>
  30 #include <devalarm.h>
  31 #include <arch/types.h>
  32 #include <arch/vm.h>
  33 #include <arch/emulate.h>
  34 #include <arch/vmdebug.h>
  35 #include <arch/msr-index.h>
  36
  37 #define pgprintk(x...) do { } while (0)
  38
  39 #define ASSERT(x)                                                       \
  40         if (!(x)) {                                                     \
  41                 printd( "assertion failed %s:%d: %s\n", \
  42                        __FILE__, __LINE__, #x);                         \
  43         }
  44
  45 #define PT64_ENT_PER_PAGE 512
  46 #define PT32_ENT_PER_PAGE 1024
  47
  48 #define PT_WRITABLE_SHIFT 1
  49
  50 #define PT_PRESENT_MASK (1ULL << 0)
  51 #define PT_WRITABLE_MASK (1ULL << PT_WRITABLE_SHIFT)
  52 #define PT_USER_MASK (1ULL << 2)
  53 #define PT_PWT_MASK (1ULL << 3)
  54 #define PT_PCD_MASK (1ULL << 4)
  55 #define PT_ACCESSED_MASK (1ULL << 5)
  56 #define PT_DIRTY_MASK (1ULL << 6)
  57 #define PT_PAGE_SIZE_MASK (1ULL << 7)
  58 #define PT_PAT_MASK (1ULL << 7)
  59 #define PT_GLOBAL_MASK (1ULL << 8)
  60 #define PT64_NX_MASK (1ULL << 63)
  61
  62 #define PT_PAT_SHIFT 7
  63 #define PT_DIR_PAT_SHIFT 12
  64 #define PT_DIR_PAT_MASK (1ULL << PT_DIR_PAT_SHIFT)
  65
  66 #define PT32_DIR_PSE36_SIZE 4
  67 #define PT32_DIR_PSE36_SHIFT 13
  68 #define PT32_DIR_PSE36_MASK (((1ULL << PT32_DIR_PSE36_SIZE) - 1) << PT32_DIR_PSE36_SHIFT)
  69
  70 #define PT32_PTE_COPY_MASK \
  71         (PT_PRESENT_MASK | PT_PWT_MASK | PT_PCD_MASK | \
  72         PT_ACCESSED_MASK | PT_DIRTY_MASK | PT_PAT_MASK | \
  73         PT_GLOBAL_MASK )
  74
  75 #define PT32_NON_PTE_COPY_MASK \
  76         (PT_PRESENT_MASK | PT_PWT_MASK | PT_PCD_MASK | \
  77         PT_ACCESSED_MASK | PT_DIRTY_MASK)
  78
  79 #define PT64_PTE_COPY_MASK \
  80         (PT64_NX_MASK | PT32_PTE_COPY_MASK)
  81
  82 #define PT64_NON_PTE_COPY_MASK \
  83         (PT64_NX_MASK | PT32_NON_PTE_COPY_MASK)
  84
  85 #define PT_FIRST_AVAIL_BITS_SHIFT 9
  86 #define PT64_SECOND_AVAIL_BITS_SHIFT 52
  87
  88 #define PT_SHADOW_PS_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  89 #define PT_SHADOW_IO_MARK (1ULL << PT_FIRST_AVAIL_BITS_SHIFT)
  90
  91 #define PT_SHADOW_WRITABLE_SHIFT (PT_FIRST_AVAIL_BITS_SHIFT + 1)
  92 #define PT_SHADOW_WRITABLE_MASK (1ULL << PT_SHADOW_WRITABLE_SHIFT)
  93
  94 #define PT_SHADOW_USER_SHIFT (PT_SHADOW_WRITABLE_SHIFT + 1)
  95 #define PT_SHADOW_USER_MASK (1ULL << (PT_SHADOW_USER_SHIFT))
  96
  97 #define PT_SHADOW_BITS_OFFSET (PT_SHADOW_WRITABLE_SHIFT - PT_WRITABLE_SHIFT)
  98
  99 #define VALID_PAGE(x) ((x) != INVALID_PAGE)
 100
 101 #define PT64_LEVEL_BITS 9
 102
 103 #define PT64_LEVEL_SHIFT(level) \
 104                 ( PAGE_SHIFT + (level - 1) * PT64_LEVEL_BITS )
 105
 106 #define PT64_LEVEL_MASK(level) \
 107                 (((1ULL << PT64_LEVEL_BITS) - 1) << PT64_LEVEL_SHIFT(level))
 108
 109 #define PT64_INDEX(address, level)\
 110         (((address) >> PT64_LEVEL_SHIFT(level)) & ((1 << PT64_LEVEL_BITS) - 1))
 111
 112 #define PT32_LEVEL_BITS 10
 113
 114 #define PT32_LEVEL_SHIFT(level) \
 115                 ( PAGE_SHIFT + (level - 1) * PT32_LEVEL_BITS )
 116
 117 #define PT32_LEVEL_MASK(level) \
 118                 (((1ULL << PT32_LEVEL_BITS) - 1) << PT32_LEVEL_SHIFT(level))
 119
 120 #define PT32_INDEX(address, level)\
 121         (((address) >> PT32_LEVEL_SHIFT(level)) & ((1 << PT32_LEVEL_BITS) - 1))
 122
 123 #define PT64_BASE_ADDR_MASK (((1ULL << 52) - 1) & PAGE_MASK)
 124 #define PT64_DIR_BASE_ADDR_MASK \
 125         (PT64_BASE_ADDR_MASK & ~((1ULL << (PAGE_SHIFT + PT64_LEVEL_BITS)) - 1))
 126
 127 #define PT32_BASE_ADDR_MASK PAGE_MASK
 128 #define PT32_DIR_BASE_ADDR_MASK \
 129         (PAGE_MASK & ~((1ULL << (PAGE_SHIFT + PT32_LEVEL_BITS)) - 1))
 130
 131 #define PFERR_PRESENT_MASK (1U << 0)
 132 #define PFERR_WRITE_MASK (1U << 1)
 133 #define PFERR_USER_MASK (1U << 2)
 134
 135 #define PT64_ROOT_LEVEL 4
 136 #define PT32_ROOT_LEVEL 2
 137 #define PT32E_ROOT_LEVEL 3
 138
 139 #define PT_DIRECTORY_LEVEL 2
 140 #define PT_PAGE_TABLE_LEVEL 1
 141
 142 static int is_write_protection(void)
 143 {
 144         print_func_entry();
 145         print_func_exit();
 146         return guest_cr0() & CR0_WP_MASK;
 147 }
 148
 149 static int is_cpuid_PSE36(void)
 150 {
 151         print_func_entry();
 152         print_func_exit();
 153         return 1;
 154 }
 155
 156 static int is_present_pte(unsigned long pte)
 157 {
 158         //print_func_entry();
 159         //print_func_exit();
 160         return pte & PT_PRESENT_MASK;
 161 }
 162
 163 static int is_writeble_pte(unsigned long pte)
 164 {
 165         //print_func_entry();
 166         //print_func_exit();
 167         return pte & PT_WRITABLE_MASK;
 168 }
 169
 170 static int is_io_pte(unsigned long pte)
 171 {
 172         //print_func_entry();
 173         //print_func_exit();
 174         return pte & PT_SHADOW_IO_MARK;
 175 }
 176
 177 static void litevm_mmu_free_page(struct litevm_vcpu *vcpu, hpa_t page_hpa)
 178 {
 179         print_func_entry();
 180         struct litevm_mmu_page *page_head = page_header(page_hpa);
 181
 182         LIST_REMOVE(page_head, link);
 183         //list_del(&page_head->link);
 184         page_head->page_hpa = page_hpa;
 185         //list_add(&page_head->link, &vcpu->free_pages);
 186         LIST_INSERT_HEAD(&vcpu->link, page_head, link);
 187         print_func_exit();
 188 }
 189
 190 static int is_empty_shadow_page(hpa_t page_hpa)
 191 {
 192         print_func_entry();
 193         uint32_t *pos;
 194         uint32_t *end;
 195         for (pos = KADDR(page_hpa), end = pos + PAGE_SIZE / sizeof(uint32_t);
 196                  pos != end; pos++)
 197                 if (*pos != 0) {
 198                         print_func_exit();
 199                         return 0;
 200                 }
 201         print_func_exit();
 202         return 1;
 203 }
 204
 205 static hpa_t litevm_mmu_alloc_page(struct litevm_vcpu *vcpu,
 206                                                                    uint64_t * parent_pte)
 207 {
 208         print_func_entry();
 209         struct litevm_mmu_page *page;
 210
 211         if (LIST_EMPTY(&vcpu->link)) {
 212                 print_func_exit();
 213                 return INVALID_PAGE;
 214         }
 215
 216         page = LIST_FIRST(&vcpu->link);
 217         LIST_REMOVE(page, link);
 218         LIST_INSERT_HEAD(&vcpu->litevm->link, page, link);
 219         ASSERT(is_empty_shadow_page(page->page_hpa));
 220         page->slot_bitmap = 0;
 221         page->global = 1;
 222         page->parent_pte = parent_pte;
 223         print_func_exit();
 224         return page->page_hpa;
 225 }
 226
 227 static void page_header_update_slot(struct litevm *litevm, void *pte, gpa_t gpa)
 228 {
 229         print_func_entry();
 230         int slot = memslot_id(litevm, gfn_to_memslot(litevm, gpa >> PAGE_SHIFT));
 231         struct litevm_mmu_page *page_head = page_header(PADDR(pte));
 232
 233         SET_BITMASK_BIT_ATOMIC((uint8_t *) & page_head->slot_bitmap, slot);
 234         print_func_exit();
 235 }
 236
 237 hpa_t safe_gpa_to_hpa(struct litevm_vcpu *vcpu, gpa_t gpa)
 238 {
 239         print_func_entry();
 240         hpa_t hpa = gpa_to_hpa(vcpu, gpa);
 241
 242         print_func_exit();
 243         return is_error_hpa(hpa) ? bad_page_address | (gpa & ~PAGE_MASK) : hpa;
 244 }
 245
 246 hpa_t gpa_to_hpa(struct litevm_vcpu * vcpu, gpa_t gpa)
 247 {
 248         print_func_entry();
 249         struct litevm_memory_slot *slot;
 250         struct page *page;
 251
 252         ASSERT((gpa & HPA_ERR_MASK) == 0);
 253         slot = gfn_to_memslot(vcpu->litevm, gpa >> PAGE_SHIFT);
 254         printk("GFN %016lx memslot %p\n", gpa>>PAGE_SHIFT, slot);
 255         if (!slot) {
 256                 printk("GFN_TO_MEMSLOT FAILED!\n");
 257                 print_func_exit();
 258                 return gpa | HPA_ERR_MASK;
 259         }
 260         page = gfn_to_page(slot, gpa >> PAGE_SHIFT);
 261         printk("Page is %p\n", page);
 262         print_func_exit();
 263         printk("gpa_to_hpa: return %016lx\n",  ((hpa_t) page2ppn(page) << PAGE_SHIFT)
 264                 | (gpa & (PAGE_SIZE - 1)));
 265         return ((hpa_t) page2ppn(page) << PAGE_SHIFT)
 266                 | (gpa & (PAGE_SIZE - 1));
 267 }
 268
 269 hpa_t gva_to_hpa(struct litevm_vcpu * vcpu, gva_t gva)
 270 {
 271         print_func_entry();
 272         gpa_t gpa = vcpu->mmu.gva_to_gpa(vcpu, gva);
 273
 274         if (gpa == UNMAPPED_GVA) {
 275                 print_func_exit();
 276                 return UNMAPPED_GVA;
 277         }
 278         print_func_exit();
 279         return gpa_to_hpa(vcpu, gpa);
 280 }
 281
 282 static void release_pt_page_64(struct litevm_vcpu *vcpu, hpa_t page_hpa,
 283                                                            int level)
 284 {
 285         print_func_entry();
 286         ASSERT(vcpu);
 287         ASSERT(VALID_PAGE(page_hpa));
 288         ASSERT(level <= PT64_ROOT_LEVEL && level > 0);
 289
 290         if (level == 1)
 291                 memset(KADDR(page_hpa), 0, PAGE_SIZE);
 292         else {
 293                 uint64_t *pos;
 294                 uint64_t *end;
 295
 296                 for (pos = KADDR(page_hpa), end = pos + PT64_ENT_PER_PAGE;
 297                          pos != end; pos++) {
 298                         uint64_t current_ent = *pos;
 299
 300                         *pos = 0;
 301                         if (is_present_pte(current_ent))
 302                                 release_pt_page_64(vcpu,
 303                                                                    current_ent &
 304                                                                    PT64_BASE_ADDR_MASK, level - 1);
 305                 }
 306         }
 307         litevm_mmu_free_page(vcpu, page_hpa);
 308         print_func_exit();
 309 }
 310
 311 static void nonpaging_new_cr3(struct litevm_vcpu *vcpu)
 312 {
 313         print_func_entry();
 314         print_func_exit();
 315 }
 316
 317 static int nonpaging_map(struct litevm_vcpu *vcpu, gva_t v, hpa_t p)
 318 {
 319         print_func_entry();
 320         int level = PT32E_ROOT_LEVEL;
 321         hpa_t table_addr = vcpu->mmu.root_hpa;
 322 printk("nonpaging_map: v %016lx, p %016lx\n", v, p);
 323 hexdump(KADDR(p), 32);
 324
 325         for (;; level--) {
 326                 uint32_t index = PT64_INDEX(v, level);
 327                 uint64_t *table;
 328
 329                 ASSERT(VALID_PAGE(table_addr));
 330                 table = KADDR(table_addr);
 331
 332                 if (level == 1) {
 333                         mark_page_dirty(vcpu->litevm, v >> PAGE_SHIFT);
 334                         page_header_update_slot(vcpu->litevm, table, v);
 335                         table[index] = p | PT_PRESENT_MASK | PT_WRITABLE_MASK |
 336                                 PT_USER_MASK;
 337                         print_func_exit();
 338                         return 0;
 339                 }
 340
 341                 if (table[index] == 0) {
 342                         hpa_t new_table = litevm_mmu_alloc_page(vcpu, &table[index]);
 343
 344                         if (!VALID_PAGE(new_table)) {
 345                                 pgprintk("nonpaging_map: ENOMEM\n");
 346                                 print_func_exit();
 347                                 return -ENOMEM;
 348                         }
 349
 350                         if (level == PT32E_ROOT_LEVEL)
 351                                 table[index] = new_table | PT_PRESENT_MASK;
 352                         else
 353                                 table[index] = new_table | PT_PRESENT_MASK |
 354                                         PT_WRITABLE_MASK | PT_USER_MASK;
 355                 }
 356                 table_addr = table[index] & PT64_BASE_ADDR_MASK;
 357         }
 358         print_func_exit();
 359 }
 360
 361 static void nonpaging_flush(struct litevm_vcpu *vcpu)
 362 {
 363         print_func_entry();
 364         hpa_t root = vcpu->mmu.root_hpa;
 365
 366         ++litevm_stat.tlb_flush;
 367         pgprintk("nonpaging_flush\n");
 368         ASSERT(VALID_PAGE(root));
 369         release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level);
 370         root = litevm_mmu_alloc_page(vcpu, 0);
 371         ASSERT(VALID_PAGE(root));
 372         vcpu->mmu.root_hpa = root;
 373         if (is_paging())
 374                 root |= (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK));
 375         vmcs_writel(GUEST_CR3, root);
 376         print_func_exit();
 377 }
 378
 379 static gpa_t nonpaging_gva_to_gpa(struct litevm_vcpu *vcpu, gva_t vaddr)
 380 {
 381         print_func_entry();
 382         print_func_exit();
 383         return vaddr;
 384 }
 385
 386 static int nonpaging_page_fault(struct litevm_vcpu *vcpu, gva_t gva,
 387                                                                 uint32_t error_code)
 388 {
 389         print_func_entry();
 390         int ret;
 391         gpa_t addr = gva;
 392
 393 printk("nonpaging_page_fault: %016llx\n", gva);
 394         ASSERT(vcpu);
 395         ASSERT(VALID_PAGE(vcpu->mmu.root_hpa));
 396
 397         for (;;) {
 398                 hpa_t paddr;
 399
 400                 paddr = gpa_to_hpa(vcpu, addr & PT64_BASE_ADDR_MASK);
 401
 402                 if (is_error_hpa(paddr)) {
 403                         print_func_exit();
 404                         return 1;
 405                 }
 406
 407                 ret = nonpaging_map(vcpu, addr & PAGE_MASK, paddr);
 408                 if (ret) {
 409                         nonpaging_flush(vcpu);
 410                         continue;
 411                 }
 412                 break;
 413         }
 414         print_func_exit();
 415         return ret;
 416 }
 417
 418 static void nonpaging_inval_page(struct litevm_vcpu *vcpu, gva_t addr)
 419 {
 420         print_func_entry();
 421         print_func_exit();
 422 }
 423
 424 static void nonpaging_free(struct litevm_vcpu *vcpu)
 425 {
 426         print_func_entry();
 427         hpa_t root;
 428
 429         ASSERT(vcpu);
 430         root = vcpu->mmu.root_hpa;
 431         if (VALID_PAGE(root))
 432                 release_pt_page_64(vcpu, root, vcpu->mmu.shadow_root_level);
 433         vcpu->mmu.root_hpa = INVALID_PAGE;
 434         print_func_exit();
 435 }
 436
 437 static int nonpaging_init_context(struct litevm_vcpu *vcpu)
 438 {
 439         print_func_entry();
 440         struct litevm_mmu *context = &vcpu->mmu;
 441
 442         context->new_cr3 = nonpaging_new_cr3;
 443         context->page_fault = nonpaging_page_fault;
 444         context->inval_page = nonpaging_inval_page;
 445         context->gva_to_gpa = nonpaging_gva_to_gpa;
 446         context->free = nonpaging_free;
 447         context->root_level = PT32E_ROOT_LEVEL;
 448         context->shadow_root_level = PT32E_ROOT_LEVEL;
 449         context->root_hpa = litevm_mmu_alloc_page(vcpu, 0);
 450         ASSERT(VALID_PAGE(context->root_hpa));
 451         vmcs_writel(GUEST_CR3, context->root_hpa);
 452         print_func_exit();
 453         return 0;
 454 }
 455
 456 static void litevm_mmu_flush_tlb(struct litevm_vcpu *vcpu)
 457 {
 458         print_func_entry();
 459         struct litevm_mmu_page *page, *npage;
 460
 461         //list_for_each_entry_safe(page, npage, &vcpu->litevm->active_mmu_pages,
 462         LIST_FOREACH_SAFE(page, &vcpu->litevm->link, link, npage) {
 463                 if (page->global)
 464                         continue;
 465
 466                 if (!page->parent_pte)
 467                         continue;
 468
 469                 *page->parent_pte = 0;
 470                 release_pt_page_64(vcpu, page->page_hpa, 1);
 471         }
 472         ++litevm_stat.tlb_flush;
 473         print_func_exit();
 474 }
 475
 476 static void paging_new_cr3(struct litevm_vcpu *vcpu)
 477 {
 478         print_func_entry();
 479         litevm_mmu_flush_tlb(vcpu);
 480         print_func_exit();
 481 }
 482
 483 static void mark_pagetable_nonglobal(void *shadow_pte)
 484 {
 485         print_func_entry();
 486         page_header(PADDR(shadow_pte))->global = 0;
 487         print_func_exit();
 488 }
 489
 490 static inline void set_pte_common(struct litevm_vcpu *vcpu,
 491                                                                   uint64_t * shadow_pte,
 492                                                                   gpa_t gaddr, int dirty, uint64_t access_bits)
 493 {
 494         print_func_entry();
 495         hpa_t paddr;
 496
 497         *shadow_pte |= access_bits << PT_SHADOW_BITS_OFFSET;
 498         if (!dirty)
 499                 access_bits &= ~PT_WRITABLE_MASK;
 500
 501         if (access_bits & PT_WRITABLE_MASK)
 502                 mark_page_dirty(vcpu->litevm, gaddr >> PAGE_SHIFT);
 503
 504         *shadow_pte |= access_bits;
 505
 506         paddr = gpa_to_hpa(vcpu, gaddr & PT64_BASE_ADDR_MASK);
 507
 508         if (!(*shadow_pte & PT_GLOBAL_MASK))
 509                 mark_pagetable_nonglobal(shadow_pte);
 510
 511         if (is_error_hpa(paddr)) {
 512                 *shadow_pte |= gaddr;
 513                 *shadow_pte |= PT_SHADOW_IO_MARK;
 514                 *shadow_pte &= ~PT_PRESENT_MASK;
 515         } else {
 516                 *shadow_pte |= paddr;
 517                 page_header_update_slot(vcpu->litevm, shadow_pte, gaddr);
 518         }
 519         print_func_exit();
 520 }
 521
 522 static void inject_page_fault(struct litevm_vcpu *vcpu,
 523                                                           uint64_t addr, uint32_t err_code)
 524 {
 525         print_func_entry();
 526         uint32_t vect_info = vmcs_read32(IDT_VECTORING_INFO_FIELD);
 527
 528         pgprintk("inject_page_fault: 0x%llx err 0x%x\n", addr, err_code);
 529
 530         ++litevm_stat.pf_guest;
 531
 532         if (is_page_fault(vect_info)) {
 533                 printd("inject_page_fault: "
 534                            "double fault 0x%llx @ 0x%lx\n", addr, vmcs_readl(GUEST_RIP));
 535                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, 0);
 536                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
 537                                          DF_VECTOR |
 538                                          INTR_TYPE_EXCEPTION |
 539                                          INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK);
 540                 print_func_exit();
 541                 return;
 542         }
 543         vcpu->cr2 = addr;
 544         vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, err_code);
 545         vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
 546                                  PF_VECTOR |
 547                                  INTR_TYPE_EXCEPTION |
 548                                  INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK);
 549
 550         print_func_exit();
 551 }
 552
 553 static inline int fix_read_pf(uint64_t * shadow_ent)
 554 {
 555         print_func_entry();
 556         if ((*shadow_ent & PT_SHADOW_USER_MASK) && !(*shadow_ent & PT_USER_MASK)) {
 557                 /*
 558                  * If supervisor write protect is disabled, we shadow kernel
 559                  * pages as user pages so we can trap the write access.
 560                  */
 561                 *shadow_ent |= PT_USER_MASK;
 562                 *shadow_ent &= ~PT_WRITABLE_MASK;
 563
 564                 print_func_exit();
 565                 return 1;
 566
 567         }
 568         print_func_exit();
 569         return 0;
 570 }
 571
 572 static int may_access(uint64_t pte, int write, int user)
 573 {
 574         print_func_entry();
 575
 576         if (user && !(pte & PT_USER_MASK)) {
 577                 print_func_exit();
 578                 return 0;
 579         }
 580         if (write && !(pte & PT_WRITABLE_MASK)) {
 581                 print_func_exit();
 582                 return 0;
 583         }
 584         print_func_exit();
 585         return 1;
 586 }
 587
 588 /*
 589  * Remove a shadow pte.
 590  */
 591 static void paging_inval_page(struct litevm_vcpu *vcpu, gva_t addr)
 592 {
 593         print_func_entry();
 594         hpa_t page_addr = vcpu->mmu.root_hpa;
 595         int level = vcpu->mmu.shadow_root_level;
 596
 597 printk("paging_inval_page: addr %016lx\n", addr);
 598         ++litevm_stat.invlpg;
 599
 600         for (;; level--) {
 601                 uint32_t index = PT64_INDEX(addr, level);
 602                 uint64_t *table = KADDR(page_addr);
 603
 604                 if (level == PT_PAGE_TABLE_LEVEL) {
 605                         table[index] = 0;
 606                         print_func_exit();
 607                         return;
 608                 }
 609
 610                 if (!is_present_pte(table[index])) {
 611                         print_func_exit();
 612                         return;
 613                 }
 614
 615                 page_addr = table[index] & PT64_BASE_ADDR_MASK;
 616
 617                 if (level == PT_DIRECTORY_LEVEL && (table[index] & PT_SHADOW_PS_MARK)) {
 618                         table[index] = 0;
 619                         release_pt_page_64(vcpu, page_addr, PT_PAGE_TABLE_LEVEL);
 620
 621                         //flush tlb
 622                         vmcs_writel(GUEST_CR3, vcpu->mmu.root_hpa |
 623                                                 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
 624                         print_func_exit();
 625                         return;
 626                 }
 627         }
 628         print_func_exit();
 629 }
 630
 631 static void paging_free(struct litevm_vcpu *vcpu)
 632 {
 633         print_func_entry();
 634         nonpaging_free(vcpu);
 635         print_func_exit();
 636 }
 637
 638 #define PTTYPE 64
 639 #include "paging_tmpl.h"
 640 #undef PTTYPE
 641
 642 #define PTTYPE 32
 643 #include "paging_tmpl.h"
 644 #undef PTTYPE
 645
 646 static int paging64_init_context(struct litevm_vcpu *vcpu)
 647 {
 648         print_func_entry();
 649         struct litevm_mmu *context = &vcpu->mmu;
 650
 651         ASSERT(is_pae());
 652         context->new_cr3 = paging_new_cr3;
 653         context->page_fault = paging64_page_fault;
 654         context->inval_page = paging_inval_page;
 655         context->gva_to_gpa = paging64_gva_to_gpa;
 656         context->free = paging_free;
 657         context->root_level = PT64_ROOT_LEVEL;
 658         context->shadow_root_level = PT64_ROOT_LEVEL;
 659         context->root_hpa = litevm_mmu_alloc_page(vcpu, 0);
 660         ASSERT(VALID_PAGE(context->root_hpa));
 661         vmcs_writel(GUEST_CR3, context->root_hpa |
 662                                 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
 663         print_func_exit();
 664         return 0;
 665 }
 666
 667 static int paging32_init_context(struct litevm_vcpu *vcpu)
 668 {
 669         print_func_entry();
 670         struct litevm_mmu *context = &vcpu->mmu;
 671
 672         context->new_cr3 = paging_new_cr3;
 673         context->page_fault = paging32_page_fault;
 674         context->inval_page = paging_inval_page;
 675         context->gva_to_gpa = paging32_gva_to_gpa;
 676         context->free = paging_free;
 677         context->root_level = PT32_ROOT_LEVEL;
 678         context->shadow_root_level = PT32E_ROOT_LEVEL;
 679         context->root_hpa = litevm_mmu_alloc_page(vcpu, 0);
 680         ASSERT(VALID_PAGE(context->root_hpa));
 681         vmcs_writel(GUEST_CR3, context->root_hpa |
 682                                 (vcpu->cr3 & (CR3_PCD_MASK | CR3_WPT_MASK)));
 683         print_func_exit();
 684         return 0;
 685 }
 686
 687 static int paging32E_init_context(struct litevm_vcpu *vcpu)
 688 {
 689         print_func_entry();
 690         int ret;
 691
 692         if ((ret = paging64_init_context(vcpu))) {
 693                 print_func_exit();
 694                 return ret;
 695         }
 696
 697         vcpu->mmu.root_level = PT32E_ROOT_LEVEL;
 698         vcpu->mmu.shadow_root_level = PT32E_ROOT_LEVEL;
 699         print_func_exit();
 700         return 0;
 701 }
 702
 703 static int init_litevm_mmu(struct litevm_vcpu *vcpu)
 704 {
 705         print_func_entry();
 706         ASSERT(vcpu);
 707         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
 708
 709         if (!is_paging()) {
 710                 print_func_exit();
 711                 return nonpaging_init_context(vcpu);
 712         } else if (is_long_mode()) {
 713                 print_func_exit();
 714                 return paging64_init_context(vcpu);
 715         } else if (is_pae()) {
 716                 print_func_exit();
 717                 return paging32E_init_context(vcpu);
 718         } else {
 719                 print_func_exit();
 720                 return paging32_init_context(vcpu);
 721         }
 722 }
 723
 724 static void destroy_litevm_mmu(struct litevm_vcpu *vcpu)
 725 {
 726         print_func_entry();
 727         ASSERT(vcpu);
 728         if (VALID_PAGE(vcpu->mmu.root_hpa)) {
 729                 vcpu->mmu.free(vcpu);
 730                 vcpu->mmu.root_hpa = INVALID_PAGE;
 731         }
 732         print_func_exit();
 733 }
 734
 735 int litevm_mmu_reset_context(struct litevm_vcpu *vcpu)
 736 {
 737         print_func_entry();
 738         destroy_litevm_mmu(vcpu);
 739         print_func_exit();
 740         return init_litevm_mmu(vcpu);
 741 }
 742
 743 static void free_mmu_pages(struct litevm_vcpu *vcpu)
 744 {
 745         print_func_entry();
 746         /* todo: use the right macros */
 747         while (!LIST_EMPTY(&vcpu->link)) {
 748                 struct litevm_mmu_page *vmpage;
 749                 vmpage = LIST_FIRST(&vcpu->link);
 750                 LIST_REMOVE(vmpage, link);
 751                 uintptr_t ppn = vmpage->page_hpa >> PAGE_SHIFT;
 752                 page_decref(ppn2page(ppn));
 753                 assert(page_is_free(ppn));
 754                 vmpage->page_hpa = INVALID_PAGE;
 755         }
 756         print_func_exit();
 757 }
 758
 759 static int alloc_mmu_pages(struct litevm_vcpu *vcpu)
 760 {
 761         print_func_entry();
 762         int i;
 763
 764         ASSERT(vcpu);
 765
 766         /* we could try to do the contiguous alloc but it's not
 767          * necessary for them to be contiguous.
 768          */
 769         for (i = 0; i < LITEVM_NUM_MMU_PAGES; i++) {
 770                 struct page *page;
 771                 struct litevm_mmu_page *page_header = &vcpu->page_header_buf[i];
 772
 773                 if (kpage_alloc(&page) != ESUCCESS)
 774                         goto error_1;
 775                 page->pg_private = page_header;
 776                 page_header->page_hpa = (hpa_t) page2pa(page);
 777                 memset(KADDR(page_header->page_hpa), 0, PAGE_SIZE);
 778                 LIST_INSERT_HEAD(&vcpu->link, page_header, link);
 779         }
 780         print_func_exit();
 781         return 0;
 782
 783 error_1:
 784         free_mmu_pages(vcpu);
 785         print_func_exit();
 786         return -ENOMEM;
 787 }
 788
 789 int litevm_mmu_init(struct litevm_vcpu *vcpu)
 790 {
 791         print_func_entry();
 792         int r;
 793
 794         ASSERT(vcpu);
 795         ASSERT(!VALID_PAGE(vcpu->mmu.root_hpa));
 796         ASSERT(LIST_EMPTY(&vcpu->link));
 797
 798         if ((r = alloc_mmu_pages(vcpu))) {
 799                 print_func_exit();
 800                 return r;
 801         }
 802
 803         if ((r = init_litevm_mmu(vcpu))) {
 804                 free_mmu_pages(vcpu);
 805                 print_func_exit();
 806                 return r;
 807         }
 808         print_func_exit();
 809         return 0;
 810 }
 811
 812 void litevm_mmu_destroy(struct litevm_vcpu *vcpu)
 813 {
 814         print_func_entry();
 815         ASSERT(vcpu);
 816
 817         destroy_litevm_mmu(vcpu);
 818         free_mmu_pages(vcpu);
 819         print_func_exit();
 820 }
 821
 822 void litevm_mmu_slot_remove_write_access(struct litevm *litevm, int slot)
 823 {
 824         print_func_entry();
 825         struct litevm_mmu_page *page, *link;
 826
 827         LIST_FOREACH(page, &litevm->link, link) {
 828                 int i;
 829                 uint64_t *pt;
 830
 831                 if (!GET_BITMASK_BIT((uint8_t *) & page->slot_bitmap, slot))
 832                         continue;
 833
 834                 pt = KADDR(page->page_hpa);
 835                 for (i = 0; i < PT64_ENT_PER_PAGE; ++i)
 836                         /* avoid RMW */
 837                         if (pt[i] & PT_WRITABLE_MASK)
 838                                 pt[i] &= ~PT_WRITABLE_MASK;
 839
 840         }
 841         print_func_exit();
 842 }