4 #include <ros/arch/arch.h>
5 #include <ros/common.h>
10 #define ARCH_CL_SIZE 64
12 static inline void breakpoint(void) __attribute__((always_inline));
13 static inline void invlpg(void *addr) __attribute__((always_inline));
14 static inline void tlbflush(void) __attribute__((always_inline));
15 static inline void icache_flush_page(void *va, void *kva)
16 __attribute__((always_inline));
17 static inline uint64_t read_tsc(void) __attribute__((always_inline));
18 static inline uint64_t read_tscp(void) __attribute__((always_inline));
19 static inline uint64_t read_tsc_serialized(void) __attribute__((always_inline));
20 static inline void enable_irq(void) __attribute__((always_inline));
21 static inline void disable_irq(void) __attribute__((always_inline));
22 static inline void enable_irqsave(int8_t *state) __attribute__((always_inline));
23 static inline void disable_irqsave(int8_t *state)
24 __attribute__((always_inline));
25 static inline void cpu_relax(void) __attribute__((always_inline));
26 static inline void cpu_halt(void) __attribute__((always_inline));
27 static inline void clflush(uintptr_t* addr) __attribute__((always_inline));
28 static inline int irq_is_enabled(void) __attribute__((always_inline));
29 static inline int get_hw_coreid(uint32_t coreid) __attribute__((always_inline));
30 static inline int hw_core_id(void) __attribute__((always_inline));
31 static inline int get_os_coreid(int hw_coreid) __attribute__((always_inline));
32 static inline int core_id(void) __attribute__((always_inline));
33 static inline int core_id_early(void) __attribute__((always_inline));
34 static inline void cache_flush(void) __attribute__((always_inline));
35 static inline void reboot(void)
36 __attribute__((always_inline)) __attribute__((noreturn));
39 void send_ipi(uint32_t os_coreid, uint8_t vector);
41 void print_cpuinfo(void);
42 void show_mapping(uintptr_t start, size_t size);
44 /* declared in smp.c */
45 extern int hw_coreid_lookup[MAX_NUM_CPUS];
46 extern int os_coreid_lookup[MAX_NUM_CPUS];
48 static inline void breakpoint(void)
53 static inline void invlpg(void *addr)
55 asm volatile("invlpg (%0)" : : "r" (addr) : "memory");
58 static inline void tlbflush(void)
61 asm volatile("mov %%cr3,%0" : "=r" (cr3));
62 asm volatile("mov %0,%%cr3" : : "r" (cr3));
65 static inline void icache_flush_page(void *va, void *kva)
67 // x86 handles self-modifying code (mostly) without SW support
70 static inline uint64_t read_tsc(void)
73 asm volatile("rdtsc" : "=d"(edx), "=a"(eax));
74 return (uint64_t)edx << 32 | eax;
77 /* non-core-id reporting style (it is in ecx) */
78 static inline uint64_t read_tscp(void)
81 asm volatile("rdtscp" : "=d"(edx), "=a"(eax) : : X86_REG_CX);
82 return (uint64_t)edx << 32 | eax;
85 /* Check out k/a/x86/rdtsc_test.c for more info */
86 static inline uint64_t read_tsc_serialized(void)
88 asm volatile("lfence" ::: "memory"); /* mfence on amd? */
92 static inline void enable_irq(void)
97 static inline void disable_irq(void)
102 static inline void enable_irqsave(int8_t *state)
104 // *state tracks the number of nested enables and disables
105 // initial value of state: 0 = first run / no favorite
106 // > 0 means more enabled calls have been made
107 // < 0 means more disabled calls have been made
108 // Mostly doing this so we can call disable_irqsave first if we want
110 // one side or another "gets a point" if interrupts were already the
111 // way it wanted to go. o/w, state stays at 0. if the state was not 0
112 // then, enabling/disabling isn't even an option. just increment/decrement
114 // if enabling is winning or tied, make sure it's enabled
115 if ((*state == 0) && !irq_is_enabled())
121 static inline void disable_irqsave(int8_t *state)
123 if ((*state == 0) && irq_is_enabled())
129 static inline void cpu_relax(void)
134 /* This doesn't atomically enable interrupts and then halt, like we want, so
135 * x86 needs to use a custom helper in the irq handler in trap.c. */
136 static inline void cpu_halt(void)
138 asm volatile("sti; hlt" : : : "memory");
141 static inline void clflush(uintptr_t* addr)
143 asm volatile("clflush %0" : : "m"(*addr));
146 static inline int irq_is_enabled(void)
148 return read_flags() & FL_IF;
151 /* os_coreid -> hw_coreid */
152 static inline int get_hw_coreid(uint32_t coreid)
154 return hw_coreid_lookup[coreid];
157 static inline int hw_core_id(void)
159 return lapic_get_id();
162 /* hw_coreid -> os_coreid */
163 static inline int get_os_coreid(int hw_coreid)
165 return os_coreid_lookup[hw_coreid];
168 #ifdef CONFIG_FAST_COREID
169 static inline int core_id(void)
172 asm volatile ("rdtscp" : "=c"(ret) : : "eax", "edx");
176 /* core_id() returns the OS core number, not to be confused with the
177 * hardware-specific core identifier (such as the lapic id) returned by
179 static inline int core_id(void)
181 return get_os_coreid(hw_core_id());
183 #endif /* CONFIG_FAST_COREID */
185 static inline int core_id_early(void)
192 static inline void cache_flush(void)
197 static inline void reboot(void)
200 asm volatile ("mov $0, %"X86_REG_SP"; int $0");
204 #endif /* !ROS_INC_ARCH_H */