3 #include <ros/common.h>
4 #include <ros/arch/msr-index.h>
7 #include <arch/fixup.h>
9 /* Model Specific Registers */
10 // TODO: figure out which are intel specific, and name them accordingly
11 #define IA32_APIC_BASE 0x1b
12 /* These two are intel-only */
13 #define IA32_FEATURE_CONTROL 0x3a
14 #define IA32_MISC_ENABLE 0x1a0
16 #define IA32_MTRR_DEF_TYPE 0x2ff
17 #define IA32_MTRR_PHYSBASE0 0x200
18 #define IA32_MTRR_PHYSMASK0 0x201
19 #define IA32_MTRR_PHYSBASE1 0x202
20 #define IA32_MTRR_PHYSMASK1 0x203
21 #define IA32_MTRR_PHYSBASE2 0x204
22 #define IA32_MTRR_PHYSMASK2 0x205
23 #define IA32_MTRR_PHYSBASE3 0x206
24 #define IA32_MTRR_PHYSMASK3 0x207
25 #define IA32_MTRR_PHYSBASE4 0x208
26 #define IA32_MTRR_PHYSMASK4 0x209
27 #define IA32_MTRR_PHYSBASE5 0x20a
28 #define IA32_MTRR_PHYSMASK5 0x20b
29 #define IA32_MTRR_PHYSBASE6 0x20c
30 #define IA32_MTRR_PHYSMASK6 0x20d
31 #define IA32_MTRR_PHYSBASE7 0x20e
32 #define IA32_MTRR_PHYSMASK7 0x20f
34 #define MSR_APIC_ENABLE 0x00000800
35 #define MSR_APIC_BASE_ADDRESS 0x0000000FFFFFF000
37 #define IA32_EFER_MSR 0xc0000080
38 # define IA32_EFER_SYSCALL (1 << 0)
39 # define IA32_EFER_IA32E_EN (1 << 8)
40 # define IA32_EFER_IA32E_ACT (1 << 10)
41 # define IA32_EFER_EXE_DIS_BIT (1 << 11)
43 #define MSR_TSC_AUX 0xc0000103
45 #define MSR_FS_BASE 0xc0000100
46 #define MSR_GS_BASE 0xc0000101
47 #define MSR_KERN_GS_BASE 0xc0000102
49 #define MSR_STAR 0xc0000081
50 #define MSR_LSTAR 0xc0000082
51 #define MSR_CSTAR 0xc0000083
52 #define MSR_SFMASK 0xc0000084
55 #define CPUID_PSE_SUPPORT 0x00000008
58 #define MAX_NUM_CORES 255
60 #define X86_REG_BP "rbp"
61 #define X86_REG_SP "rsp"
62 #define X86_REG_IP "rip"
63 #define X86_REG_AX "rax"
64 #define X86_REG_BX "rbx"
65 #define X86_REG_CX "rcx"
66 #define X86_REG_DX "rdx"
69 /* Various flags defined: can be included from assembler. */
74 #define X86_EFLAGS_CF 0x00000001 /* Carry Flag */
75 #define X86_EFLAGS_BIT1 0x00000002 /* Bit 1 - always on */
76 #define X86_EFLAGS_PF 0x00000004 /* Parity Flag */
77 #define X86_EFLAGS_AF 0x00000010 /* Auxiliary carry Flag */
78 #define X86_EFLAGS_ZF 0x00000040 /* Zero Flag */
79 #define X86_EFLAGS_SF 0x00000080 /* Sign Flag */
80 #define X86_EFLAGS_TF 0x00000100 /* Trap Flag */
81 #define X86_EFLAGS_IF 0x00000200 /* Interrupt Flag */
82 #define X86_EFLAGS_DF 0x00000400 /* Direction Flag */
83 #define X86_EFLAGS_OF 0x00000800 /* Overflow Flag */
84 #define X86_EFLAGS_IOPL 0x00003000 /* IOPL mask */
85 #define X86_EFLAGS_NT 0x00004000 /* Nested Task */
86 #define X86_EFLAGS_RF 0x00010000 /* Resume Flag */
87 #define X86_EFLAGS_VM 0x00020000 /* Virtual Mode */
88 #define X86_EFLAGS_AC 0x00040000 /* Alignment Check */
89 #define X86_EFLAGS_VIF 0x00080000 /* Virtual Interrupt Flag */
90 #define X86_EFLAGS_VIP 0x00100000 /* Virtual Interrupt Pending */
91 #define X86_EFLAGS_ID 0x00200000 /* CPUID detection flag */
94 * Basic CPU control in CR0
96 #define X86_CR0_PE 0x00000001 /* Protection Enable */
97 #define X86_CR0_MP 0x00000002 /* Monitor Coprocessor */
98 #define X86_CR0_EM 0x00000004 /* Emulation */
99 #define X86_CR0_TS 0x00000008 /* Task Switched */
100 #define X86_CR0_ET 0x00000010 /* Extension Type */
101 #define X86_CR0_NE 0x00000020 /* Numeric Error */
102 #define X86_CR0_WP 0x00010000 /* Write Protect */
103 #define X86_CR0_AM 0x00040000 /* Alignment Mask */
104 #define X86_CR0_NW 0x20000000 /* Not Write-through */
105 #define X86_CR0_CD 0x40000000 /* Cache Disable */
106 #define X86_CR0_PG 0x80000000 /* Paging */
109 * Paging options in CR3
111 #define X86_CR3_PWT 0x00000008 /* Page Write Through */
112 #define X86_CR3_PCD 0x00000010 /* Page Cache Disable */
113 #define X86_CR3_PCID_MASK 0x00000fff /* PCID Mask */
116 * Intel CPU features in CR4
118 #define X86_CR4_VME 0x00000001 /* enable vm86 extensions */
119 #define X86_CR4_PVI 0x00000002 /* virtual interrupts flag enable */
120 #define X86_CR4_TSD 0x00000004 /* disable time stamp at ipl 3 */
121 #define X86_CR4_DE 0x00000008 /* enable debugging extensions */
122 #define X86_CR4_PSE 0x00000010 /* enable page size extensions */
123 #define X86_CR4_PAE 0x00000020 /* enable physical address extensions */
124 #define X86_CR4_MCE 0x00000040 /* Machine check enable */
125 #define X86_CR4_PGE 0x00000080 /* enable global pages */
126 #define X86_CR4_PCE 0x00000100 /* enable performance counters at ipl 3 */
127 #define X86_CR4_OSFXSR 0x00000200 /* enable fast FPU save and restore */
128 #define X86_CR4_OSXMMEXCPT 0x00000400 /* enable unmasked SSE exceptions */
129 #define X86_CR4_VMXE 0x00002000 /* enable VMX virtualization */
130 #define X86_CR4_RDWRGSFS 0x00010000 /* enable RDWRGSFS support */
131 #define X86_CR4_PCIDE 0x00020000 /* enable PCID support */
132 #define X86_CR4_OSXSAVE 0x00040000 /* enable xsave and xrestore */
133 #define X86_CR4_SMEP 0x00100000 /* enable SMEP support */
134 #define X86_CR4_SMAP 0x00200000 /* enable SMAP support */
136 /* MWAIT C-state hints. The names might not be right for different processors.
137 * For instance, the Linux idle driver for a Haswell calls the mwait for 0x10
139 #define X86_MWAIT_C1 0x00
140 #define X86_MWAIT_C2 0x10
141 #define X86_MWAIT_C3 0x20
142 #define X86_MWAIT_C4 0x30
143 #define X86_MWAIT_C5 0x40
144 #define X86_MWAIT_C6 0x50
147 * x86-64 Task Priority Register, CR8
149 #define X86_CR8_TPR 0x0000000F /* task priority register */
151 #ifndef __ASSEMBLER__
153 static inline uint8_t inb(int port) __attribute__((always_inline));
154 static inline void insb(int port, void *addr, int cnt)
155 __attribute__((always_inline));
156 static inline uint16_t inw(int port) __attribute__((always_inline));
157 static inline void insw(int port, void *addr, int cnt)
158 __attribute__((always_inline));
159 static inline uint32_t inl(int port) __attribute__((always_inline));
160 static inline void insl(int port, void *addr, int cnt)
161 __attribute__((always_inline));
162 static inline void outb(int port, uint8_t data) __attribute__((always_inline));
163 static inline void outsb(int port, const void *addr, int cnt)
164 __attribute__((always_inline));
165 static inline void outw(int port, uint16_t data) __attribute__((always_inline));
166 static inline void outsw(int port, const void *addr, int cnt)
167 __attribute__((always_inline));
168 static inline void outsl(int port, const void *addr, int cnt)
169 __attribute__((always_inline));
170 static inline void outl(int port, uint32_t data) __attribute__((always_inline));
171 static inline void lidt(void *p) __attribute__((always_inline));
172 static inline void lldt(uint16_t sel) __attribute__((always_inline));
173 static inline void ltr(uint16_t sel) __attribute__((always_inline));
174 static inline void lcr0(unsigned long val) __attribute__((always_inline));
175 static inline unsigned long rcr0(void) __attribute__((always_inline));
176 static inline unsigned long rcr2(void) __attribute__((always_inline));
177 static inline void lcr3(unsigned long val) __attribute__((always_inline));
178 static inline unsigned long rcr3(void) __attribute__((always_inline));
179 static inline void lcr4(unsigned long val) __attribute__((always_inline));
180 static inline unsigned long rcr4(void) __attribute__((always_inline));
182 static inline void lxcr0(uint64_t xcr0) __attribute__((always_inline));
183 static inline int safe_lxcr0(uint64_t xcr0) __attribute__((always_inline));
184 static inline uint64_t rxcr0(void) __attribute__((always_inline));
186 static inline unsigned long read_flags(void) __attribute__((always_inline));
187 static inline void write_eflags(unsigned long eflags)
188 __attribute__((always_inline));
189 static inline unsigned long read_bp(void) __attribute__((always_inline));
190 static inline unsigned long read_pc(void) __attribute__((always_inline));
191 static inline unsigned long read_sp(void) __attribute__((always_inline));
192 static inline void cpuid(uint32_t info1, uint32_t info2, uint32_t *eaxp,
193 uint32_t *ebxp, uint32_t *ecxp, uint32_t *edxp)
194 __attribute__((always_inline));
195 static inline uint32_t cpuid_ecx(uint32_t op) __attribute__((always_inline));
196 static inline uint64_t read_msr(uint32_t reg) __attribute__((always_inline));
197 static inline void write_msr(uint32_t reg, uint64_t val)
198 __attribute__((always_inline));
199 /* if we have mm64s, change the hpet helpers */
200 static inline void write_mmreg8(uintptr_t reg, uint8_t val)
201 __attribute__((always_inline));
202 static inline uint8_t read_mmreg8(uintptr_t reg) __attribute__((always_inline));
203 static inline void write_mmreg32(uintptr_t reg, uint32_t val)
204 __attribute__((always_inline));
205 static inline uint32_t read_mmreg32(uintptr_t reg)
206 __attribute__((always_inline));
207 static inline void wbinvd(void) __attribute__((always_inline));
208 static inline void __cpu_relax(void) __attribute__((always_inline));
210 void set_pstate(unsigned int pstate);
211 void set_fastest_pstate(void);
212 unsigned int get_pstate(void);
213 void set_cstate(unsigned int cstate);
214 unsigned int get_cstate(void);
216 static inline uint8_t inb(int port)
220 asm volatile("inb %w1,%0" : "=a" (data) : "d" (port));
224 static inline void insb(int port, void *addr, int cnt)
226 asm volatile("cld\n\trepne\n\tinsb"
227 : "=D" (addr), "=c" (cnt)
228 : "d" (port), "0" (addr), "1" (cnt)
232 static inline uint16_t inw(int port)
236 asm volatile("inw %w1,%0" : "=a" (data) : "d" (port));
240 static inline void insw(int port, void *addr, int cnt)
242 asm volatile("cld\n\trepne\n\tinsw"
243 : "=D" (addr), "=c" (cnt)
244 : "d" (port), "0" (addr), "1" (cnt)
248 static inline uint32_t inl(int port)
252 asm volatile("inl %w1,%0" : "=a" (data) : "d" (port));
256 static inline void insl(int port, void *addr, int cnt)
258 asm volatile("cld\n\trepne\n\tinsl"
259 : "=D" (addr), "=c" (cnt)
260 : "d" (port), "0" (addr), "1" (cnt)
264 static inline void outb(int port, uint8_t data)
266 asm volatile("outb %0,%w1" : : "a" (data), "d" (port));
269 static inline void outsb(int port, const void *addr, int cnt)
271 asm volatile("cld\n\trepne\n\toutsb"
272 : "=S" (addr), "=c" (cnt)
273 : "d" (port), "0" (addr), "1" (cnt)
277 static inline void outw(int port, uint16_t data)
279 asm volatile("outw %0,%w1" : : "a" (data), "d" (port));
282 static inline void outsw(int port, const void *addr, int cnt)
284 asm volatile("cld\n\trepne\n\toutsw"
285 : "=S" (addr), "=c" (cnt)
286 : "d" (port), "0" (addr), "1" (cnt)
290 static inline void outsl(int port, const void *addr, int cnt)
292 asm volatile("cld\n\trepne\n\toutsl"
293 : "=S" (addr), "=c" (cnt)
294 : "d" (port), "0" (addr), "1" (cnt)
298 static inline void outl(int port, uint32_t data)
300 asm volatile("outl %0,%w1" : : "a" (data), "d" (port));
303 static inline void lidt(void *p)
305 asm volatile("lidt (%0)" : : "r" (p));
308 static inline void lldt(uint16_t sel)
310 asm volatile("lldt %0" : : "r" (sel));
313 static inline void ltr(uint16_t sel)
315 asm volatile("ltr %0" : : "r" (sel));
318 static inline void lcr0(unsigned long val)
320 asm volatile("mov %0,%%cr0" : : "r" (val));
323 static inline unsigned long rcr0(void)
326 asm volatile("mov %%cr0,%0" : "=r" (val));
330 static inline void lcr2(unsigned long val)
332 asm volatile("mov %0,%%cr2" : : "r" (val));
335 static inline unsigned long rcr2(void)
338 asm volatile("mov %%cr2,%0" : "=r" (val));
342 static inline void lcr3(unsigned long val)
344 asm volatile("mov %0,%%cr3" : : "r" (val));
347 static inline unsigned long rcr3(void)
350 asm volatile("mov %%cr3,%0" : "=r" (val));
354 static inline void lcr4(unsigned long val)
356 asm volatile("mov %0,%%cr4" : : "r" (val));
359 static inline unsigned long rcr4(void)
362 asm volatile("mov %%cr4,%0" : "=r" (cr4));
366 static inline void lxcr0(uint64_t xcr0)
372 asm volatile("xsetbv"
374 : "a"(eax), "c" (0), "d"(edx));
377 static inline int safe_lxcr0(uint64_t xcr0)
384 asm volatile(ASM_STAC ";"
387 ".section .fixup, \"ax\";"
393 : "a"(eax), "c" (0), "d"(edx),
394 "i" (-EINVAL), "0" (err));
399 static inline uint64_t rxcr0(void)
403 asm volatile("xgetbv"
404 : "=a"(eax), "=d"(edx)
406 return ((uint64_t)edx << 32) | eax;
409 static inline unsigned long read_flags(void)
411 unsigned long eflags;
413 asm volatile("pushf; pop %0" : "=r" (eflags));
417 static inline void write_eflags(unsigned long eflags)
419 asm volatile("push %0; popf" : : "r" (eflags));
422 static inline unsigned long read_bp(void)
426 asm volatile("mov %%"X86_REG_BP",%0" : "=r" (bp));
430 static inline unsigned long read_pc(void)
434 asm volatile("call 1f; 1: pop %0" : "=r"(ip));
438 static inline unsigned long read_sp(void)
442 asm volatile("mov %%"X86_REG_SP",%0" : "=r" (sp));
446 static inline void cpuid(uint32_t info1, uint32_t info2, uint32_t *eaxp,
447 uint32_t *ebxp, uint32_t *ecxp, uint32_t *edxp)
449 uint32_t eax, ebx, ecx, edx;
451 /* Can select with both eax (info1) and ecx (info2) */
453 : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx)
454 : "a" (info1), "c" (info2));
465 static inline uint32_t cpuid_ecx(uint32_t op)
469 cpuid(op, 0, NULL, NULL, &ecx, NULL);
473 // Might need to mfence rdmsr. supposedly wrmsr serializes, but not for x2APIC
474 static inline uint64_t read_msr(uint32_t reg)
478 asm volatile("rdmsr; mfence" : "=d"(edx), "=a"(eax) : "c"(reg));
479 return (uint64_t)edx << 32 | eax;
482 static inline void write_msr(uint32_t reg, uint64_t val)
484 asm volatile("wrmsr" : : "d"(val >> 32), "a"(val & 0xFFFFFFFF), "c"(reg));
487 static inline void write_mmreg8(uintptr_t reg, uint8_t val)
489 *((volatile uint8_t*)reg) = val;
492 static inline uint8_t read_mmreg8(uintptr_t reg)
494 return *((volatile uint8_t*)reg);
497 static inline void write_mmreg32(uintptr_t reg, uint32_t val)
499 *((volatile uint32_t*)reg) = val;
502 static inline uint32_t read_mmreg32(uintptr_t reg)
504 return *((volatile uint32_t*)reg);
507 static inline void wbinvd(void)
509 asm volatile("wbinvd");
512 /* this version of cpu_relax is needed to resolve some circular dependencies
513 * with arch/arch.h and arch/apic.h */
514 static inline void __cpu_relax(void)
516 // in case the compiler doesn't serialize for pause, the "m" will make
517 // sure no memory is reordered around this instruction.
518 asm volatile("pause" : : : "memory");
521 #endif /* !__ASSEMBLER__ */