akaros/kern/arch/x86/trapentry64.S
<<
>>
Prefs
   1/* See COPYRIGHT for copyright information.
   2 * The two TRAP* macros (minus the .data parts) are from the JOS project.
   3 * Everything else:
   4 * Copyright (c) 2009, 2013 The Regents of the University of California
   5 * Barret Rhoden <brho@cs.berkeley.edu>
   6 * See LICENSE for details.
   7 */
   8#include <arch/mmu.h>
   9#include <arch/trap.h>
  10#include <arch/x86.h>
  11#include <ros/memlayout.h>
  12
  13###################################################################
  14# exceptions/interrupts
  15###################################################################
  16
  17/* The TRAPHANDLER macro defines a globally-visible function for handling
  18 * a trap.  It pushes a trap number onto the stack, then jumps to _alltraps.
  19 * It also builds this traps portion of the trap_tbl.
  20 * Use TRAPHANDLER for traps where the CPU automatically pushes an error code.
  21 */
  22#define TRAPHANDLER(name, num)                                          \
  23        .text;                                                          \
  24        .globl name;            /* define global symbol for 'name' */   \
  25        .type name, @function;  /* symbol type is function */           \
  26        .align 2;               /* align function definition */         \
  27        name:                   /* function starts here */              \
  28        pushq $(num);                                                   \
  29        jmp _alltraps;                                                  \
  30        .data;                                                          \
  31        .quad name;                                                     \
  32        .long num
  33
  34/* Use TRAPHANDLER_NOEC for traps where the CPU doesn't push an error code.
  35 * It pushes a 0 in place of the error code, so the trap frame has the same
  36 * format in either case.  */
  37#define TRAPHANDLER_NOEC(name, num)                                     \
  38        .text;                                                          \
  39        .globl name;                                                    \
  40        .type name, @function;                                          \
  41        .align 2;                                                       \
  42        name:                                                           \
  43        pushq $0;                                                       \
  44        pushq $(num);                                                   \
  45        jmp _alltraps;                                                  \
  46        .data;                                                          \
  47        .quad name;                                                     \
  48        .long num
  49
  50/* Same as NOEC, but for IRQs instead.  num is the ISR number it is mapped to */
  51#define IRQ_HANDLER(name, num)                                          \
  52        .text;                                                          \
  53        .globl name;                                                    \
  54        .type name, @function;                                          \
  55        .align 2;                                                       \
  56        name:                                                           \
  57        pushq $0;                                                       \
  58        pushq $(num);                                                   \
  59        jmp _allirqs;                                                   \
  60        .data;                                                          \
  61        .quad name;                                                     \
  62        .long num
  63
  64#define NMI_HANDLER(name, num)                                          \
  65        .text;                                                          \
  66        .globl name;                                                    \
  67        .type name, @function;                                          \
  68        .align 2;                                                       \
  69        name:                                                           \
  70        pushq $0;                                                       \
  71        pushq $(num);                                                   \
  72        jmp _nmi_entry;                                                 \
  73        .data;                                                          \
  74        .quad name;                                                     \
  75        .long num
  76
  77#define DOUBLEFAULT_HANDLER(name, num)  \
  78        .text;                                                          \
  79        .globl name;                                                    \
  80        .type name, @function;                                          \
  81        .align 2;                                                       \
  82        name:                                                           \
  83        pushq $(num);                                                   \
  84        jmp _dblf_entry;                                                \
  85        .data;                                                          \
  86        .quad name;                                                     \
  87        .long num
  88
  89/* Bare minimum IRQ handler: send a LAPIC_EOI and immediately iret. */
  90#define POKE_HANDLER(name, num)                                         \
  91        .text;                                                          \
  92        .globl name;                                                    \
  93        .type name, @function;                                          \
  94        .align 2;                                                       \
  95        name:;                                                          \
  96        pushq %rax;                                                     \
  97        pushq %rcx;                                                     \
  98        pushq %rdx;                                                     \
  99        movq $0, %rax;                                                  \
 100        movq $0, %rdx;                                                  \
 101        movq $(MSR_LAPIC_EOI), %rcx;                                    \
 102        wrmsr;                                                          \
 103        popq %rdx;                                                      \
 104        popq %rcx;                                                      \
 105        popq %rax;                                                      \
 106        iretq;                                                          \
 107        .data;                                                          \
 108        .quad name;                                                     \
 109        .long num
 110
 111# Set data for the trap_tbl symbol.  Set text for the entry_points.  The various
 112# trap/irq handler macros will toggle data on and off as needed.
 113.data
 114.globl trap_tbl
 115trap_tbl:
 116.text
 117.globl __asm_entry_points_start
 118__asm_entry_points_start:
 119
 120/* Generate entry points for the different traps.  Note that all of these bounce
 121 * off the corresponding trap.c function, such as handle_irqs, and that the name
 122 * e.g. ISR_divide_error is soley for the little stup that jumps to something
 123 * like _alltraps.
 124 *
 125 * Technically, these HANDLER entries do not need to be in numeric order.
 126 * trap.c will do a 'foreach (up to last-1), set the IDT for the number to point
 127 * to the func' in the order in which they appear in the trap tbl, so the 'last
 128 * one wins'. */
 129TRAPHANDLER_NOEC(ISR_divide_error, T_DIVIDE)
 130TRAPHANDLER_NOEC(ISR_debug_exceptions, T_DEBUG)
 131NMI_HANDLER(ISR_NMI, T_NMI)
 132TRAPHANDLER_NOEC(ISR_breakpoint, T_BRKPT)
 133TRAPHANDLER_NOEC(ISR_overflow, T_OFLOW)
 134TRAPHANDLER_NOEC(ISR_bounds_check, T_BOUND)
 135TRAPHANDLER_NOEC(ISR_invalid_opcode, T_ILLOP)
 136TRAPHANDLER_NOEC(ISR_device_not_available, T_DEVICE)
 137DOUBLEFAULT_HANDLER(ISR_double_fault, T_DBLFLT)
 138/* 9 reserved */
 139TRAPHANDLER(ISR_invalid_TSS, T_TSS)
 140TRAPHANDLER(ISR_segment_not_present, T_SEGNP)
 141TRAPHANDLER(ISR_stack_exception, T_STACK)
 142TRAPHANDLER(ISR_general_protection_fault, T_GPFLT)
 143TRAPHANDLER(ISR_page_fault, T_PGFLT)
 144/* 15 reserved */
 145TRAPHANDLER_NOEC(ISR_floating_point_error, T_FPERR)
 146TRAPHANDLER(ISR_alignment_check, T_ALIGN)
 147TRAPHANDLER_NOEC(ISR_machine_check, T_MCHK)
 148TRAPHANDLER_NOEC(ISR_simd_error, T_SIMDERR)
 149/* 20 - 31 reserved */
 150/* 32-47 are PIC/8259 IRQ vectors */
 151IRQ_HANDLER(IRQ0, 32)
 152IRQ_HANDLER(IRQ1, 33)
 153IRQ_HANDLER(IRQ2, 34)
 154IRQ_HANDLER(IRQ3, 35)
 155IRQ_HANDLER(IRQ4, 36)
 156IRQ_HANDLER(IRQ5, 37)
 157IRQ_HANDLER(IRQ6, 38)
 158IRQ_HANDLER(IRQ7, 39)
 159IRQ_HANDLER(IRQ8, 40)
 160IRQ_HANDLER(IRQ9, 41)
 161IRQ_HANDLER(IRQ10, 42)
 162IRQ_HANDLER(IRQ11, 43)
 163IRQ_HANDLER(IRQ12, 44)
 164IRQ_HANDLER(IRQ13, 45)
 165IRQ_HANDLER(IRQ14, 46)
 166IRQ_HANDLER(IRQ15, 47)
 167TRAPHANDLER_NOEC(ISR_syscall, T_SYSCALL)
 168/* 49-223 are IOAPIC routing vectors (from IOAPIC to LAPIC) */
 169IRQ_HANDLER(IRQ17, 49)
 170IRQ_HANDLER(IRQ18, 50)
 171IRQ_HANDLER(IRQ19, 51)
 172IRQ_HANDLER(IRQ20, 52)
 173IRQ_HANDLER(IRQ21, 53)
 174IRQ_HANDLER(IRQ22, 54)
 175IRQ_HANDLER(IRQ23, 55)
 176IRQ_HANDLER(IRQ24, 56)
 177IRQ_HANDLER(IRQ25, 57)
 178IRQ_HANDLER(IRQ26, 58)
 179IRQ_HANDLER(IRQ27, 59)
 180IRQ_HANDLER(IRQ28, 60)
 181IRQ_HANDLER(IRQ29, 61)
 182IRQ_HANDLER(IRQ30, 62)
 183IRQ_HANDLER(IRQ31, 63)
 184IRQ_HANDLER(IRQ32, 64)
 185IRQ_HANDLER(IRQ33, 65)
 186IRQ_HANDLER(IRQ34, 66)
 187IRQ_HANDLER(IRQ35, 67)
 188IRQ_HANDLER(IRQ36, 68)
 189IRQ_HANDLER(IRQ37, 69)
 190IRQ_HANDLER(IRQ38, 70)
 191IRQ_HANDLER(IRQ39, 71)
 192IRQ_HANDLER(IRQ40, 72)
 193IRQ_HANDLER(IRQ41, 73)
 194IRQ_HANDLER(IRQ42, 74)
 195IRQ_HANDLER(IRQ43, 75)
 196IRQ_HANDLER(IRQ44, 76)
 197IRQ_HANDLER(IRQ45, 77)
 198IRQ_HANDLER(IRQ46, 78)
 199IRQ_HANDLER(IRQ47, 79)
 200IRQ_HANDLER(IRQ48, 80)
 201IRQ_HANDLER(IRQ49, 81)
 202IRQ_HANDLER(IRQ50, 82)
 203IRQ_HANDLER(IRQ51, 83)
 204IRQ_HANDLER(IRQ52, 84)
 205IRQ_HANDLER(IRQ53, 85)
 206IRQ_HANDLER(IRQ54, 86)
 207IRQ_HANDLER(IRQ55, 87)
 208IRQ_HANDLER(IRQ56, 88)
 209IRQ_HANDLER(IRQ57, 89)
 210IRQ_HANDLER(IRQ58, 90)
 211IRQ_HANDLER(IRQ59, 91)
 212IRQ_HANDLER(IRQ60, 92)
 213IRQ_HANDLER(IRQ61, 93)
 214IRQ_HANDLER(IRQ62, 94)
 215IRQ_HANDLER(IRQ63, 95)
 216IRQ_HANDLER(IRQ64, 96)
 217IRQ_HANDLER(IRQ65, 97)
 218IRQ_HANDLER(IRQ66, 98)
 219IRQ_HANDLER(IRQ67, 99)
 220IRQ_HANDLER(IRQ68, 100)
 221IRQ_HANDLER(IRQ69, 101)
 222IRQ_HANDLER(IRQ70, 102)
 223IRQ_HANDLER(IRQ71, 103)
 224IRQ_HANDLER(IRQ72, 104)
 225IRQ_HANDLER(IRQ73, 105)
 226IRQ_HANDLER(IRQ74, 106)
 227IRQ_HANDLER(IRQ75, 107)
 228IRQ_HANDLER(IRQ76, 108)
 229IRQ_HANDLER(IRQ77, 109)
 230IRQ_HANDLER(IRQ78, 110)
 231IRQ_HANDLER(IRQ79, 111)
 232IRQ_HANDLER(IRQ80, 112)
 233IRQ_HANDLER(IRQ81, 113)
 234IRQ_HANDLER(IRQ82, 114)
 235IRQ_HANDLER(IRQ83, 115)
 236IRQ_HANDLER(IRQ84, 116)
 237IRQ_HANDLER(IRQ85, 117)
 238IRQ_HANDLER(IRQ86, 118)
 239IRQ_HANDLER(IRQ87, 119)
 240IRQ_HANDLER(IRQ88, 120)
 241IRQ_HANDLER(IRQ89, 121)
 242IRQ_HANDLER(IRQ90, 122)
 243IRQ_HANDLER(IRQ91, 123)
 244IRQ_HANDLER(IRQ92, 124)
 245IRQ_HANDLER(IRQ93, 125)
 246IRQ_HANDLER(IRQ94, 126)
 247IRQ_HANDLER(IRQ95, 127)
 248IRQ_HANDLER(IRQ96, 128)
 249IRQ_HANDLER(IRQ97, 129)
 250IRQ_HANDLER(IRQ98, 130)
 251IRQ_HANDLER(IRQ99, 131)
 252IRQ_HANDLER(IRQ100, 132)
 253IRQ_HANDLER(IRQ101, 133)
 254IRQ_HANDLER(IRQ102, 134)
 255IRQ_HANDLER(IRQ103, 135)
 256IRQ_HANDLER(IRQ104, 136)
 257IRQ_HANDLER(IRQ105, 137)
 258IRQ_HANDLER(IRQ106, 138)
 259IRQ_HANDLER(IRQ107, 139)
 260IRQ_HANDLER(IRQ108, 140)
 261IRQ_HANDLER(IRQ109, 141)
 262IRQ_HANDLER(IRQ110, 142)
 263IRQ_HANDLER(IRQ111, 143)
 264IRQ_HANDLER(IRQ112, 144)
 265IRQ_HANDLER(IRQ113, 145)
 266IRQ_HANDLER(IRQ114, 146)
 267IRQ_HANDLER(IRQ115, 147)
 268IRQ_HANDLER(IRQ116, 148)
 269IRQ_HANDLER(IRQ117, 149)
 270IRQ_HANDLER(IRQ118, 150)
 271IRQ_HANDLER(IRQ119, 151)
 272IRQ_HANDLER(IRQ120, 152)
 273IRQ_HANDLER(IRQ121, 153)
 274IRQ_HANDLER(IRQ122, 154)
 275IRQ_HANDLER(IRQ123, 155)
 276IRQ_HANDLER(IRQ124, 156)
 277IRQ_HANDLER(IRQ125, 157)
 278IRQ_HANDLER(IRQ126, 158)
 279IRQ_HANDLER(IRQ127, 159)
 280IRQ_HANDLER(IRQ128, 160)
 281IRQ_HANDLER(IRQ129, 161)
 282IRQ_HANDLER(IRQ130, 162)
 283IRQ_HANDLER(IRQ131, 163)
 284IRQ_HANDLER(IRQ132, 164)
 285IRQ_HANDLER(IRQ133, 165)
 286IRQ_HANDLER(IRQ134, 166)
 287IRQ_HANDLER(IRQ135, 167)
 288IRQ_HANDLER(IRQ136, 168)
 289IRQ_HANDLER(IRQ137, 169)
 290IRQ_HANDLER(IRQ138, 170)
 291IRQ_HANDLER(IRQ139, 171)
 292IRQ_HANDLER(IRQ140, 172)
 293IRQ_HANDLER(IRQ141, 173)
 294IRQ_HANDLER(IRQ142, 174)
 295IRQ_HANDLER(IRQ143, 175)
 296IRQ_HANDLER(IRQ144, 176)
 297IRQ_HANDLER(IRQ145, 177)
 298IRQ_HANDLER(IRQ146, 178)
 299IRQ_HANDLER(IRQ147, 179)
 300IRQ_HANDLER(IRQ148, 180)
 301IRQ_HANDLER(IRQ149, 181)
 302IRQ_HANDLER(IRQ150, 182)
 303IRQ_HANDLER(IRQ151, 183)
 304IRQ_HANDLER(IRQ152, 184)
 305IRQ_HANDLER(IRQ153, 185)
 306IRQ_HANDLER(IRQ154, 186)
 307IRQ_HANDLER(IRQ155, 187)
 308IRQ_HANDLER(IRQ156, 188)
 309IRQ_HANDLER(IRQ157, 189)
 310IRQ_HANDLER(IRQ158, 190)
 311IRQ_HANDLER(IRQ159, 191)
 312IRQ_HANDLER(IRQ160, 192)
 313IRQ_HANDLER(IRQ161, 193)
 314IRQ_HANDLER(IRQ162, 194)
 315IRQ_HANDLER(IRQ163, 195)
 316IRQ_HANDLER(IRQ164, 196)
 317IRQ_HANDLER(IRQ165, 197)
 318IRQ_HANDLER(IRQ166, 198)
 319IRQ_HANDLER(IRQ167, 199)
 320IRQ_HANDLER(IRQ168, 200)
 321IRQ_HANDLER(IRQ169, 201)
 322IRQ_HANDLER(IRQ170, 202)
 323IRQ_HANDLER(IRQ171, 203)
 324IRQ_HANDLER(IRQ172, 204)
 325IRQ_HANDLER(IRQ173, 205)
 326IRQ_HANDLER(IRQ174, 206)
 327IRQ_HANDLER(IRQ175, 207)
 328IRQ_HANDLER(IRQ176, 208)
 329IRQ_HANDLER(IRQ177, 209)
 330IRQ_HANDLER(IRQ178, 210)
 331IRQ_HANDLER(IRQ179, 211)
 332IRQ_HANDLER(IRQ180, 212)
 333IRQ_HANDLER(IRQ181, 213)
 334IRQ_HANDLER(IRQ182, 214)
 335IRQ_HANDLER(IRQ183, 215)
 336IRQ_HANDLER(IRQ184, 216)
 337IRQ_HANDLER(IRQ185, 217)
 338IRQ_HANDLER(IRQ186, 218)
 339IRQ_HANDLER(IRQ187, 219)
 340IRQ_HANDLER(IRQ188, 220)
 341IRQ_HANDLER(IRQ189, 221)
 342IRQ_HANDLER(IRQ190, 222)
 343IRQ_HANDLER(IRQ191, 223)
 344/* 224-239 are OS IPI vectors (0xe0-0xef) */
 345IRQ_HANDLER(IRQ192, I_SMP_CALL0)
 346IRQ_HANDLER(IRQ193, I_SMP_CALL1)
 347IRQ_HANDLER(IRQ194, I_SMP_CALL2)
 348IRQ_HANDLER(IRQ195, I_SMP_CALL3)
 349IRQ_HANDLER(IRQ196, I_SMP_CALL4)
 350IRQ_HANDLER(IRQ197, 229)
 351IRQ_HANDLER(IRQ198, 230)
 352IRQ_HANDLER(IRQ199, 231)
 353IRQ_HANDLER(IRQ200, 232)
 354IRQ_HANDLER(IRQ201, 233)
 355IRQ_HANDLER(IRQ202, 234)
 356IRQ_HANDLER(IRQ203, 235)
 357IRQ_HANDLER(IRQ204, I_TESTING)
 358POKE_HANDLER(IRQ205, I_POKE_GUEST)
 359POKE_HANDLER(IRQ206, I_POKE_CORE)
 360IRQ_HANDLER(IRQ207, I_KERNEL_MSG)
 361/* 240-255 are LAPIC vectors (0xf0-0xff), hightest priority class */
 362IRQ_HANDLER(IRQ208, 240)
 363IRQ_HANDLER(IRQ209, 241)
 364IRQ_HANDLER(IRQ210, 242)
 365IRQ_HANDLER(IRQ211, 243)
 366IRQ_HANDLER(IRQ212, 244)
 367IRQ_HANDLER(IRQ213, 245)
 368IRQ_HANDLER(IRQ214, 246)
 369IRQ_HANDLER(IRQ215, 247)
 370IRQ_HANDLER(IRQ216, 248)
 371IRQ_HANDLER(IRQ217, 249)
 372IRQ_HANDLER(IRQ218, 250)
 373IRQ_HANDLER(IRQ219, 251)
 374IRQ_HANDLER(IRQ220, 252)
 375IRQ_HANDLER(IRQ221, 253)
 376IRQ_HANDLER(IRQ222, 254)
 377IRQ_HANDLER(IRQ223, 255)
 378/* But make sure default is last!! */
 379TRAPHANDLER_NOEC(ISR_default, T_DEFAULT)
 380
 381.data
 382.globl trap_tbl_end
 383trap_tbl_end:
 384
 385.text
 386_alltraps:
 387        cld
 388        pushq %r15
 389        pushq %r14
 390        pushq %r13
 391        pushq %r12
 392        pushq %r11
 393        pushq %r10
 394        pushq %r9
 395        pushq %r8
 396        pushq %rdi
 397        pushq %rsi
 398        pushq %rbp
 399        pushq %rdx
 400        pushq %rcx
 401        pushq %rbx
 402        pushq %rax
 403        cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax
 404        je trap_all_tf
 405        # this is a user TF.  we need to swapgs to get the kernel's gs and mark
 406        # the context as partial
 407        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 408        movl $0x1, 0xac(%rsp)   # 0xac - diff btw tf_padding0 and tf_rax
 409trap_all_tf:
 410        pushq $0                # fsbase space
 411        pushq $0                # gsbase space
 412        movq $0, %rbp           # so we can backtrace to this point
 413        movq %rsp, %rdi
 414        call trap
 415        # the return paths are only used by the kernel
 416        addq $0x10, %rsp        # skip fs/gs base
 417        popq %rax
 418        popq %rbx
 419        popq %rcx
 420        popq %rdx
 421        popq %rbp
 422        popq %rsi
 423        popq %rdi
 424        popq %r8
 425        popq %r9
 426        popq %r10
 427        popq %r11
 428        popq %r12
 429        popq %r13
 430        popq %r14
 431        popq %r15
 432        addq $0x10, %rsp        # skip trapno and err
 433        iretq
 434
 435# might merge this with _alltraps
 436_allirqs:
 437        cld
 438        pushq %r15
 439        pushq %r14
 440        pushq %r13
 441        pushq %r12
 442        pushq %r11
 443        pushq %r10
 444        pushq %r9
 445        pushq %r8
 446        pushq %rdi
 447        pushq %rsi
 448        pushq %rbp
 449        pushq %rdx
 450        pushq %rcx
 451        pushq %rbx
 452        pushq %rax
 453        cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax
 454        je irq_all_tf
 455        # this is a user TF.  we need to swapgs to get the kernel's gs and mark
 456        # the context as partial
 457        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 458        movl $0x1, 0xac(%rsp)   # 0xac - diff btw tf_padding0 and tf_rax
 459irq_all_tf:
 460        pushq $0                # fsbase space
 461        pushq $0                # gsbase space
 462        movq $0, %rbp           # so we can backtrace to this point
 463        movq %rsp, %rdi
 464        call handle_irq
 465        # the return paths are only used by the kernel
 466        addq $0x10, %rsp        # skip fs/gs base
 467        popq %rax
 468        popq %rbx
 469        popq %rcx
 470        popq %rdx
 471        popq %rbp
 472        popq %rsi
 473        popq %rdi
 474        popq %r8
 475        popq %r9
 476        popq %r10
 477        popq %r11
 478        popq %r12
 479        popq %r13
 480        popq %r14
 481        popq %r15
 482        addq $0x10, %rsp        # skip trapno and err
 483        iretq
 484
 485# Similar to the NMI handler, we come in on a special stack, but we can trust
 486# the contents of GS for kernel contexts.  This is mostly true.  If we double
 487# faulted in this file, for instance, then GS could be garbage.  But that should
 488# never happen.  Most double faults will be for running into a guard page on a
 489# stack.
 490#
 491# Since the double fault is so bad, we just want to get into the kernel's C code
 492# where we can run some diagnostics.  We (currently) won't try to recover, since
 493# it's probably a kernel bug.
 494_dblf_entry:
 495        cld
 496        pushq %r15
 497        pushq %r14
 498        pushq %r13
 499        pushq %r12
 500        pushq %r11
 501        pushq %r10
 502        pushq %r9
 503        pushq %r8
 504        pushq %rdi
 505        pushq %rsi
 506        pushq %rbp
 507        pushq %rdx
 508        pushq %rcx
 509        pushq %rbx
 510        pushq %rax
 511        cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax
 512        je dblf_all_tf
 513        # this is a user TF.  we need to swapgs to get the kernel's gs and mark
 514        # the context as partial
 515        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 516        movl $0x1, 0xac(%rsp)   # 0xac - diff btw tf_padding0 and tf_rax
 517dblf_all_tf:
 518        pushq $0                # fsbase space
 519        pushq $0                # gsbase space
 520        movq $0, %rbp           # so we can backtrace to this point
 521        movq %rsp, %rdi
 522        call handle_double_fault
 523dblf_spin:
 524        jmp dblf_spin
 525
 526# Unlike normal trap and IRQ handlers, both user and kernel TFs are handled the
 527# similarly.  Both come in here and return from here.  We cannot do anything
 528# fancy like proc_restartcore() from NMI context.
 529#
 530# All NMIs will come in fresh on the same stack (IST1, per-core).  We don't need
 531# to find a stackpointer, but we do need to find GS.
 532#
 533# Regardless of whether or not the interrupted context was in the kernel, we
 534# can't trust GS.  Basically we can never tell from looking at GS and KERN_GS
 535# whether swapgs occurred, since the user could have put kernel addrs in their
 536# GS.  But we can use the stack for storage to bootstrap GS.  %rsp at entry
 537# points to pcpui *.
 538#
 539# We can also tell if GS was set correctly or not and avoid the wrmsr calls.
 540# This isn't a huge deal.  But note that we don't know whether or not the kernel
 541# actually set the gsbase.  We just know if it was correct or not.  The user
 542# could have set it and the kernel hadn't had a chance to swapgs yet.  The NMI
 543# handler doesn't care, since all TFs come in and go out via this asm.
 544#
 545# If we want to be paranoid, we can completely ignore user TFs and just save and
 546# restore GS with the same mechanism we use for the kernel.  But since we came
 547# in on a user TF, we can use swapgs.  Note that even for nested NMI handlers,
 548# we have a user TF for only the first time, and only the last exit will swapgs
 549# back to the way it was initially.
 550_nmi_entry:
 551        cld
 552        pushq %r15
 553        pushq %r14
 554        pushq %r13
 555        pushq %r12
 556        pushq %r11
 557        pushq %r10
 558        pushq %r9
 559        pushq %r8
 560        pushq %rdi
 561        pushq %rsi
 562        pushq %rbp
 563        pushq %rdx
 564        pushq %rcx
 565        pushq %rbx
 566        pushq %rax
 567        cmpw $GD_KT, 0x90(%rsp) # 0x90 - diff btw tf_cs and tf_rax
 568        je nmi_kern_tf
 569        # this is a user TF.  we need to swapgs to get the kernel's gs and mark
 570        # the context as partial
 571        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 572        movl $0x1, 0xac(%rsp)   # 0xac - diff btw tf_padding0 and tf_rax
 573        pushq $0                # fsbase space
 574        pushq $0                # gsbase space
 575        jmp nmi_all_tf
 576nmi_kern_tf:
 577        # this is a kernel TF.  but we don't know if they set up gs yet, so
 578        # we'll save and restore whatever they had loaded and use our own
 579        pushq $0                # fsbase space
 580        # Get the current GS base into rax
 581        movl $MSR_GS_BASE, %ecx
 582        rdmsr
 583        shlq $32, %rdx
 584        orq %rdx, %rax
 585        # Get the real GS base from the top of the stack.  This was set in
 586        # smp_boot, and our rsp pointed to it when we entered the kernel.
 587        movq 0xb8(%rsp), %rdx   # 0xb8 from fs_base to the top
 588        # Compare them.  If they are the same, we can just push 0 for gsbase
 589        # (which later will mean "no need to restore GS".
 590        cmpq %rdx, %rax
 591        je nmi_gs_ok
 592        # They weren't the same.  Save the old one and set the new one.
 593        pushq %rax              # gsbase space
 594        movq %rdx, %rax
 595        shrq $32, %rdx
 596        andl $0xffffffff, %eax
 597        wrmsr
 598        jmp nmi_all_tf
 599nmi_gs_ok:
 600        pushq $0                # gsbase space
 601nmi_all_tf:
 602        # At this point, GS is set correctly, either due to swapgs (user TF),
 603        # wrmsr (kern TF with bad GS), or it was already fine (and gsbase in the
 604        # TF = 0).
 605        movq $0, %rbp           # so we can backtrace to this point
 606        movq %rsp, %rdi
 607        call handle_nmi
 608        # Unlike in normal IRQs/Traps, both user and kernel contexts return via
 609        # this path.
 610        cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase
 611        je nmi_kern_restore_gs
 612        # User TF.  Restore whatever was there with swapgs.  We don't care what
 613        # it was, nor do we care what was in the TF.
 614        swapgs                  # user's GS is now in MSR_GS_BASE
 615        addq $0x10, %rsp        # skip gs/fs base
 616        jmp nmi_popal
 617nmi_kern_restore_gs:
 618        popq %rax               # fetch saved gsbase
 619        addq $0x08, %rsp        # skip fs base
 620        cmpq $0, %rax
 621        je nmi_popal
 622        # gsbase in the TF != 0, which means we need to restore that gsbase
 623        movl $MSR_GS_BASE, %ecx
 624        movq %rax, %rdx
 625        shrq $32, %rdx
 626        andl $0xffffffff, %eax
 627        wrmsr
 628nmi_popal:
 629        popq %rax
 630        popq %rbx
 631        popq %rcx
 632        popq %rdx
 633        popq %rbp
 634        popq %rsi
 635        popq %rdi
 636        popq %r8
 637        popq %r9
 638        popq %r10
 639        popq %r11
 640        popq %r12
 641        popq %r13
 642        popq %r14
 643        popq %r15
 644        addq $0x10, %rsp        # skip trapno and err
 645        iretq
 646
 647.globl __nmi_pop_ok_start;
 648.globl __nmi_pop_ok_end;
 649.globl __nmi_pop_fail_start;
 650.globl __nmi_pop_fail_end;
 651
 652# extern void nmi_try_to_pop(struct hw_trapframe *tf, int *status,
 653#                            int old_val, int new_val);
 654#
 655# __nmi_bottom_half calls this to atomically pop a hw_tf (%rdi) and set
 656# &pcpui->nmi_status (%rsi) with compare and swap to NMI_NORMAL_OPN (%ecx) given
 657# that it was NMI_IN_PROGRESS (%edx)
 658#
 659# (Careful, nmi_status is an int, not a long.)
 660#
 661# If the real NMI handler interrupts us, it'll move us to the fail section of
 662# the code.  That code is identical to 'ok', up until ok's final statement.
 663#
 664# In that event, we'll need a little help returning: specifically to bootstrap
 665# pcpui and our current stackpointer.  pcpui is already saved near the top of
 666# stack.  We'll save rsp ourselves.
 667.globl nmi_try_to_pop;
 668.type nmi_try_to_pop, @function;
 669nmi_try_to_pop:
 670__nmi_pop_ok_start:
 671        # careful only to use caller-saved or argument registers before saving
 672        movl %edx, %eax                 # load old_val into eax for the CAS
 673        cmpxchgl %ecx, (%rsi)   # no need for LOCK, since an NMI would serialize
 674        jz nmi_ok_cas_worked    # ZF = 1 on successful CAS
 675        ret
 676nmi_ok_cas_worked:
 677        # save callee-saved regs (the pops below clobber them, and we might
 678        # return)
 679        pushq %rbp
 680        pushq %rbx
 681        pushq %r12
 682        pushq %r13
 683        pushq %r14
 684        pushq %r15
 685        # We need to save the current rsp into the scratch space at the top of
 686        # the stack.  This assumes we're within the top page of our stack, which
 687        # should always be true.  Careful not to use rdi, which still has an
 688        # argument.
 689        movq %rsp, %rbx
 690        # Want to round rbx up to PGSIZE, then subtract 8, to get our slot.
 691        movq $0xfff, %rax
 692        notq %rax               # rax = 0xfffffffffffff000
 693        andq %rax, %rbx         # round down rbx
 694        addq $0x1000, %rbx      # add PGSIZE, assuming rsp was not page aligned
 695        subq $0x8, %rbx         # point to the scratch space
 696        movq %rsp, (%rbx)       # save rsp in the scratch space
 697        # We jump our rsp to the base of the HW_TF.  This is still on the same
 698        # stack, just farther back than where our caller is.  We need to be
 699        # careful to not clobber the stack.  Otherwise we'll have chaos.
 700        movq %rdi, %rsp
 701        # From here down is the same as the normal NMI exit path, but with 'ok'
 702        # in the symbol names.
 703        cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase
 704        je nmi_ok_kern_restore_gs
 705        # User TF.  Restore whatever was there with swapgs.  We don't care what
 706        # it was, nor do we care what was in the TF.
 707        swapgs                  # user's GS is now in MSR_GS_BASE
 708        addq $0x10, %rsp        # skip gs/fs base
 709        jmp nmi_ok_popal
 710nmi_ok_kern_restore_gs:
 711        popq %rax               # fetch saved gsbase
 712        addq $0x08, %rsp        # skip fs base
 713        cmpq $0, %rax
 714        je nmi_ok_popal
 715        # gsbase in the TF != 0, which means we need to restore that gsbase
 716        movl $MSR_GS_BASE, %ecx
 717        movq %rax, %rdx
 718        shrq $32, %rdx
 719        andl $0xffffffff, %eax
 720        wrmsr
 721nmi_ok_popal:
 722        popq %rax
 723        popq %rbx
 724        popq %rcx
 725        popq %rdx
 726        popq %rbp
 727        popq %rsi
 728        popq %rdi
 729        popq %r8
 730        popq %r9
 731        popq %r10
 732        popq %r11
 733        popq %r12
 734        popq %r13
 735        popq %r14
 736        popq %r15
 737        addq $0x10, %rsp        # skip trapno and err
 738        iretq
 739__nmi_pop_ok_end:
 740
 741# This is the 'fail' case.  It is identical to the 'ok' case, up until the
 742# iretq, other than 'ok' replaced with 'fail'.  In place of iretq, we undo the
 743# entire operation.
 744__nmi_pop_fail_start:
 745        # careful only to use caller-saved or argument registers before saving
 746        movl %edx, %eax         # load old_val into eax for the CAS
 747        cmpxchgl %ecx, (%rsi)   # no need for LOCK, since an NMI would serialize
 748        jz nmi_fail_cas_worked  # ZF = 1 on successful CAS
 749        ret
 750nmi_fail_cas_worked:
 751        # save callee-saved regs (the pops below clobber them, and we might
 752        # return)
 753        pushq %rbp
 754        pushq %rbx
 755        pushq %r12
 756        pushq %r13
 757        pushq %r14
 758        pushq %r15
 759        # We need to save the current rsp into the scratch space at the top of
 760        # the stack.  This assumes we're within the top page of our stack, which
 761        # should always be true.  Careful not to use rdi, which still has an
 762        # argument.
 763        movq %rsp, %rbx
 764        # Want to round rbx up to PGSIZE, then subtract 8, to get our slot.
 765        movq $0xfff, %rax
 766        notq %rax               # rax = 0xfffffffffffff000
 767        andq %rax, %rbx         # round down rbx
 768        addq $0x1000, %rbx      # add PGSIZE, assuming rsp was not page aligned
 769        subq $0x8, %rbx         # point to the scratch space
 770        movq %rsp, (%rbx)       # save rsp in the scratch space
 771        # We jump our rsp to the base of the HW_TF.  This is still on the same
 772        # stack, just farther back than where our caller is.  We need to be
 773        # careful to not clobber the stack.  Otherwise we'll have chaos.
 774        movq %rdi, %rsp
 775        # From here down is the same as the normal NMI exit path and the ok
 776        # path, but with 'fail' in the symbol names.
 777        cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase
 778        je nmi_fail_kern_restore_gs
 779        # User TF.  Restore whatever was there with swapgs.  We don't care what
 780        # it was, nor do we care what was in the TF.
 781        swapgs                  # user's GS is now in MSR_GS_BASE
 782        addq $0x10, %rsp        # skip gs/fs base
 783        jmp nmi_fail_popal
 784nmi_fail_kern_restore_gs:
 785        popq %rax               # fetch saved gsbase
 786        addq $0x08, %rsp        # skip fs base
 787        cmpq $0, %rax
 788        je nmi_fail_popal
 789        # gsbase in the TF != 0, which means we need to restore that gsbase
 790        movl $MSR_GS_BASE, %ecx
 791        movq %rax, %rdx
 792        shrq $32, %rdx
 793        andl $0xffffffff, %eax
 794        wrmsr
 795nmi_fail_popal:
 796        popq %rax
 797        popq %rbx
 798        popq %rcx
 799        popq %rdx
 800        popq %rbp
 801        popq %rsi
 802        popq %rdi
 803        popq %r8
 804        popq %r9
 805        popq %r10
 806        popq %r11
 807        popq %r12
 808        popq %r13
 809        popq %r14
 810        popq %r15
 811        addq $0x10, %rsp        # skip trapno and err
 812        # Here's is where we differ from OK.  Time to undo everything and return
 813        # rsp currently is pointing at tf->tf_rip.  Remember that we don't want
 814        # to write anything to the stack - everything in the TF is still the way
 815        # it was when we started to pop.
 816        #
 817        # First off, let's get the stack addr of the pcpui pointer loaded
 818        movq %rsp, %rbx
 819        movq $0xfff, %rax
 820        notq %rax               # rax = 0xfffffffffffff000
 821        andq %rax, %rbx         # round down rbx
 822        addq $0x1000, %rbx      # add PGSIZE, assuming rsp was not page aligned
 823        subq $0x10, %rbx        # point to the pcpui pointer
 824        # Now let's start to unwind
 825        subq $0x98, %rsp        # jump from rip to tf_gsbase (top of hw_tf)
 826        # Need to restore gs, just like on an NMI entry
 827        cmpw $GD_KT, 0xa0(%rsp) # 0xa0 - diff btw tf_cs and tf_gsbase
 828        je nmi_pop_fail_kern_tf
 829        # This is a user TF.  We need to swapgs to get the kernel's gs
 830        # We don't need to mark the context as partial (we never do for NMIs,
 831        # actually), and in general, we don't want to write anything on the
 832        # stack.
 833        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 834        jmp nmi_pop_fail_all_tf
 835nmi_pop_fail_kern_tf:
 836        # Kernel TF.  We basically need to do the same thing on entry, since we
 837        # might have restored some weird GS base.  We can tell based on
 838        # tf_gsbase 0 for gsbase means we didn't need to change GS
 839        cmpq $0, (%rsp)
 840        je nmi_pop_fail_gs_fine
 841        # rbx points to where pcpui* is stored
 842        mov (%rbx), %rdx
 843        movl $MSR_GS_BASE, %ecx
 844        movq %rdx, %rax
 845        shrq $32, %rdx
 846        andl $0xffffffff, %eax
 847        wrmsr
 848nmi_pop_fail_gs_fine:
 849nmi_pop_fail_all_tf:
 850        addq $0x8, %rbx         # move to the scratch slot, holding rsp
 851        mov (%rbx), %rsp
 852        # restore callee-saved regs
 853        popq %r15
 854        popq %r14
 855        popq %r13
 856        popq %r12
 857        popq %rbx
 858        popq %rbp
 859        ret
 860        # sweet jeebus.
 861__nmi_pop_fail_end:
 862
 863
 864.globl sysenter_handler;
 865.type sysenter_handler, @function;
 866
 867sysenter_handler:
 868#ifndef CONFIG_NOFASTCALL_FSBASE
 869        # Do a quick TLS / FS base change, never changing stacks.
 870        # When rdi has the magic number, rsi has the new base
 871        movabs $FASTCALL_SETFSBASE, %rax
 872        cmp %rax, %rdi
 873        jne normal_syscall      # could profile this and jump differently
 874        # need to check rsi, make sure it is canonical (will enfore below ULIM).
 875        # need to either do this check, or handle the kernel GP fault on wrmsr.
 876        movq %rsi, %rdi
 877        shrq $47, %rdi
 878        cmp $0, %rdi
 879        jne fastcall_pop
 880        # need to use cx, dx, and ax for the wrmsr.  dx and ax are free.
 881        movq %rcx, %rdi         # save rcx, the retaddr
 882        movq %rsi, %rdx
 883        movq %rsi, %rax
 884        shrq $32, %rdx
 885        andl $0xffffffff, %eax
 886        movl $MSR_FS_BASE, %ecx
 887        wrmsr
 888        movq %rdi, %rcx         # restore retaddr
 889fastcall_pop:
 890        rex.w sysret
 891normal_syscall:
 892#endif
 893        # cld is handled by the SFMASK
 894        swapgs                  # user's GS is now in MSR_KERNEL_GS_BASE
 895        movq %gs:0, %rsp
 896        # Saving the FPU callee-saved state for now.  Might be able to have the
 897        # preempt handler deal with it.
 898        pushq $0                # space for mxcsr, fpucw, and padding0
 899        movw $0x1, 0x6(%rsp)    # tf_padding0 = 1, partial context
 900        fnstcw 0x4(%rsp)
 901        stmxcsr (%rsp)
 902        pushq %rdx              # rsp, saved by userspace
 903        pushq %rcx              # rip, saved by hardware
 904        pushq %r15
 905        pushq %r14
 906        pushq %r13
 907        pushq %r12
 908        pushq %rbp
 909        pushq %rbx
 910        pushq $0                # fsbase space
 911        pushq $0                # gsbase space
 912        movq $0, %rbp           # so we can backtrace to this point
 913        movq %rsp, %rdx
 914        # arg0, rdi: struct sysc*.  arg1, rsi: count.  arg2, rdx: sw_tf
 915        call sysenter_callwrapper
 916        # return via pop_tf, never this path
 917sysenter_spin:
 918        jmp sysenter_spin
 919
 920.globl vmexit_handler;
 921.type vmexit_handler, @function;
 922vmexit_handler:
 923        # rflags has all flags = 0, so cli and cld already.
 924        # HOST_GS_BASE and RSP is set by the hardware
 925        # Set default values.  Most of these will be set in C later.
 926        pushq $0                # guest_pa
 927        pushq $0                # guest_va
 928        pushq $0                # intrinfo2 and 1
 929        pushq $0                # exit_qual + exit_reason
 930        pushq $0                # pad + trap_inject
 931        pushq $0                # flags + guest_pcorid
 932        pushq $0                # cr3
 933        pushq $0                # cr2
 934        pushq $0                # rsp
 935        pushq $0                # rflags
 936        pushq $0                # rip
 937        # Save register state
 938        pushq %r15
 939        pushq %r14
 940        pushq %r13
 941        pushq %r12
 942        pushq %r11
 943        pushq %r10
 944        pushq %r9
 945        pushq %r8
 946        pushq %rdi
 947        pushq %rsi
 948        pushq %rbp
 949        pushq %rdx
 950        pushq %rcx
 951        pushq %rbx
 952        pushq %rax
 953        movq $0, %rbp           # so we can backtrace to this point
 954        movq %rsp, %rdi
 955        call handle_vmexit
 956vmexit_spin:
 957        jmp vmexit_spin
 958
 959.globl __asm_entry_points_end
 960__asm_entry_points_end:
 961