c15f7c8b1f5be85fb91ef57022265471f023d5e3
[akaros.git] / kern / lib / percpu_counter.c
1 /*
2  * Fast batching percpu counters.
3  */
4
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
9 #include <linux/cpu.h>
10 #include <linux/module.h>
11 #include <linux/debugobjects.h>
12
13 #ifdef CONFIG_HOTPLUG_CPU
14 static LIST_HEAD(percpu_counters);
15 static DEFINE_SPINLOCK(percpu_counters_lock);
16 #endif
17
18 #ifdef CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER
19
20 static struct debug_obj_descr percpu_counter_debug_descr;
21
22 static bool percpu_counter_fixup_free(void *addr, enum debug_obj_state state)
23 {
24         struct percpu_counter *fbc = addr;
25
26         switch (state) {
27         case ODEBUG_STATE_ACTIVE:
28                 percpu_counter_destroy(fbc);
29                 debug_object_free(fbc, &percpu_counter_debug_descr);
30                 return true;
31         default:
32                 return false;
33         }
34 }
35
36 static struct debug_obj_descr percpu_counter_debug_descr = {
37         .name           = "percpu_counter",
38         .fixup_free     = percpu_counter_fixup_free,
39 };
40
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
42 {
43         debug_object_init(fbc, &percpu_counter_debug_descr);
44         debug_object_activate(fbc, &percpu_counter_debug_descr);
45 }
46
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
48 {
49         debug_object_deactivate(fbc, &percpu_counter_debug_descr);
50         debug_object_free(fbc, &percpu_counter_debug_descr);
51 }
52
53 #else   /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)
55 { }
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)
57 { }
58 #endif  /* CONFIG_DEBUG_OBJECTS_PERCPU_COUNTER */
59
60 void percpu_counter_set(struct percpu_counter *fbc, int64_t amount)
61 {
62         int cpu;
63         unsigned long flags;
64
65         spin_lock_irqsave(&fbc->lock);
66         for_each_possible_cpu(cpu) {
67                 int32_t *pcount = per_cpu_ptr(fbc->counters, cpu);
68                 *pcount = 0;
69         }
70         fbc->count = amount;
71         spin_unlock_irqsave(&fbc->lock);
72 }
73 EXPORT_SYMBOL(percpu_counter_set);
74
75 /**
76  * This function is both preempt and irq safe. The former is due to explicit
77  * preemption disable. The latter is guaranteed by the fact that the slow path
78  * is explicitly protected by an irq-safe spinlock whereas the fast patch uses
79  * this_cpu_add which is irq-safe by definition. Hence there is no need muck
80  * with irq state before calling this one
81  */
82 void percpu_counter_add_batch(struct percpu_counter *fbc, int64_t amount,
83                               int32_t batch)
84 {
85         int64_t count;
86
87         preempt_disable();
88         count = __this_cpu_read(*fbc->counters) + amount;
89         if (count >= batch || count <= -batch) {
90                 unsigned long flags;
91                 spin_lock_irqsave(&fbc->lock);
92                 fbc->count += count;
93                 __this_cpu_sub(*fbc->counters, count - amount);
94                 spin_unlock_irqsave(&fbc->lock);
95         } else {
96                 this_cpu_add(*fbc->counters, amount);
97         }
98         preempt_enable();
99 }
100 EXPORT_SYMBOL(percpu_counter_add_batch);
101
102 /*
103  * Add up all the per-cpu counts, return the result.  This is a more accurate
104  * but much slower version of percpu_counter_read_positive()
105  */
106 int64_t __percpu_counter_sum(struct percpu_counter *fbc)
107 {
108         int64_t ret;
109         int cpu;
110         unsigned long flags;
111
112         spin_lock_irqsave(&fbc->lock);
113         ret = fbc->count;
114         for_each_online_cpu(cpu) {
115                 int32_t *pcount = per_cpu_ptr(fbc->counters, cpu);
116                 ret += *pcount;
117         }
118         spin_unlock_irqsave(&fbc->lock);
119         return ret;
120 }
121 EXPORT_SYMBOL(__percpu_counter_sum);
122
123 int __percpu_counter_init(struct percpu_counter *fbc, int64_t amount, gfp_t gfp,
124                           struct lock_class_key *key)
125 {
126         unsigned long flags __maybe_unused;
127
128         spinlock_init_irqsave(&fbc->lock);
129         lockdep_set_class(&fbc->lock, key);
130         fbc->count = amount;
131         fbc->counters = alloc_percpu_gfp(int32_t, gfp);
132         if (!fbc->counters)
133                 return -ENOMEM;
134
135         debug_percpu_counter_activate(fbc);
136
137 #ifdef CONFIG_HOTPLUG_CPU
138         INIT_LIST_HEAD(&fbc->list);
139         spin_lock_irqsave(&percpu_counters_lock);
140         list_add(&fbc->list, &percpu_counters);
141         spin_unlock_irqsave(&percpu_counters_lock);
142 #endif
143         return 0;
144 }
145 EXPORT_SYMBOL(__percpu_counter_init);
146
147 void percpu_counter_destroy(struct percpu_counter *fbc)
148 {
149         unsigned long flags __maybe_unused;
150
151         if (!fbc->counters)
152                 return;
153
154         debug_percpu_counter_deactivate(fbc);
155
156 #ifdef CONFIG_HOTPLUG_CPU
157         spin_lock_irqsave(&percpu_counters_lock);
158         list_del(&fbc->list);
159         spin_unlock_irqsave(&percpu_counters_lock);
160 #endif
161         free_percpu(fbc->counters);
162         fbc->counters = NULL;
163 }
164 EXPORT_SYMBOL(percpu_counter_destroy);
165
166 int percpu_counter_batch __read_mostly = 32;
167 EXPORT_SYMBOL(percpu_counter_batch);
168
169 static int compute_batch_value(unsigned int cpu)
170 {
171         int nr = num_online_cpus();
172
173         percpu_counter_batch = MAX(32, nr * 2);
174         return 0;
175 }
176
177 static int percpu_counter_cpu_dead(unsigned int cpu)
178 {
179 #ifdef CONFIG_HOTPLUG_CPU
180         struct percpu_counter *fbc;
181
182         compute_batch_value(cpu);
183
184         spin_lock_irqsave(&percpu_counters_lock);
185         list_for_each_entry(fbc, &percpu_counters, list) {
186                 int32_t *pcount;
187
188                 spin_lock_irqsave(&fbc->lock);
189                 pcount = per_cpu_ptr(fbc->counters, cpu);
190                 fbc->count += *pcount;
191                 *pcount = 0;
192                 spin_unlock_irqsave(&fbc->lock);
193         }
194         spin_unlock_irqsave(&percpu_counters_lock);
195 #endif
196         return 0;
197 }
198
199 /*
200  * Compare counter against given value.
201  * Return 1 if greater, 0 if equal and -1 if less
202  */
203 int __percpu_counter_compare(struct percpu_counter *fbc, int64_t rhs,
204                              int32_t batch)
205 {
206         int64_t count;
207
208         count = percpu_counter_read(fbc);
209         /* Check to see if rough count will be sufficient for comparison */
210         if (abs(count - rhs) > (batch * num_online_cpus())) {
211                 if (count > rhs)
212                         return 1;
213                 else
214                         return -1;
215         }
216         /* Need to use precise count */
217         count = percpu_counter_sum(fbc);
218         if (count > rhs)
219                 return 1;
220         else if (count < rhs)
221                 return -1;
222         else
223                 return 0;
224 }
225 EXPORT_SYMBOL(__percpu_counter_compare);
226
227 static int __init percpu_counter_startup(void)
228 {
229         int ret;
230
231         ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "lib/percpu_cnt:online",
232                                 compute_batch_value, NULL);
233         warn_on(ret < 0);
234         ret = cpuhp_setup_state_nocalls(CPUHP_PERCPU_CNT_DEAD,
235                                         "lib/percpu_cnt:dead", NULL,
236                                         percpu_counter_cpu_dead);
237         warn_on(ret < 0);
238         return 0;
239 }
240 module_init(percpu_counter_startup);