-#ifndef ROS_KERN_ATOMIC_H
-#define ROS_KERN_ATOMIC_H
+/* Copyright (c) 2009-2011 The Regents of the University of California
+ * Barret Rhoden <brho@cs.berkeley.edu>
+ * See LICENSE for details.
+ *
+ * Kernel atomics and locking functions.
+ *
+ * The extern inline declarations are arch-dependent functions. We do this
+ * so that each arch can either static inline or just have a regular function,
+ * whichever is appropriate. The actual implementation usually will be in
+ * arch/atomic.h (for inlines).
+ *
+ * The static inlines are defined farther down in the file (as always). */
+
+#pragma once
#include <ros/common.h>
#include <ros/atomic.h>
#include <arch/mmu.h>
-#include <arch/atomic.h>
#include <arch/arch.h>
#include <assert.h>
-static inline void
-(SLOCK(0) spin_lock_irqsave)(spinlock_t RACY*SAFE lock);
-static inline void
-(SUNLOCK(0) spin_unlock_irqsave)(spinlock_t RACY*SAFE lock);
-static inline bool spin_lock_irq_enabled(spinlock_t *SAFE lock);
+/* Atomics */
+extern inline void atomic_init(atomic_t *number, long val);
+extern inline long atomic_read(atomic_t *number);
+extern inline void atomic_set(atomic_t *number, long val);
+extern inline void atomic_add(atomic_t *number, long val);
+extern inline void atomic_inc(atomic_t *number);
+extern inline void atomic_dec(atomic_t *number);
+extern inline long atomic_fetch_and_add(atomic_t *number, long val);
+extern inline void atomic_and(atomic_t *number, long mask);
+extern inline void atomic_or(atomic_t *number, long mask);
+extern inline long atomic_swap(atomic_t *addr, long val);
+extern inline bool atomic_cas(atomic_t *addr, long exp_val, long new_val);
+extern inline bool atomic_cas_ptr(void **addr, void *exp_val, void *new_val);
+extern inline bool atomic_cas_u32(uint32_t *addr, uint32_t exp_val,
+ uint32_t new_val);
+extern inline bool atomic_add_not_zero(atomic_t *number, long val);
+extern inline bool atomic_sub_and_test(atomic_t *number, long val);
+
+/* Spin locks */
+struct spinlock {
+ volatile uint32_t rlock;
+#ifdef CONFIG_SPINLOCK_DEBUG
+ uintptr_t call_site;
+ uint32_t calling_core;
+ bool irq_okay;
+#endif
+};
+typedef struct spinlock spinlock_t;
+#define SPINLOCK_INITIALIZER {0}
+
+#ifdef CONFIG_SPINLOCK_DEBUG
+#define SPINLOCK_INITIALIZER_IRQSAVE {0, .irq_okay = TRUE}
+#else
+#define SPINLOCK_INITIALIZER_IRQSAVE SPINLOCK_INITIALIZER
+#endif
+
+/* Arch dependent helpers/funcs: */
+extern inline void __spinlock_init(spinlock_t *lock);
+extern inline bool spin_locked(spinlock_t *lock);
+extern inline void __spin_lock(spinlock_t *lock);
+extern inline void __spin_unlock(spinlock_t *lock);
+
+/* So we can inline a __spin_lock if we want. Even though we don't need this
+ * if we're debugging, its helpful to keep the include at the same place for
+ * all builds. */
+#include <arch/atomic.h>
+
+#ifdef CONFIG_SPINLOCK_DEBUG
+/* Arch indep, in k/s/atomic.c */
+void spin_lock(spinlock_t *lock);
+bool spin_trylock(spinlock_t *lock);
+void spin_unlock(spinlock_t *lock);
+void spinlock_debug(spinlock_t *lock);
+
+#else
+/* Just inline the arch-specific __ versions */
+static inline void spin_lock(spinlock_t *lock)
+{
+ __spin_lock(lock);
+}
+static inline bool spin_trylock(spinlock_t *lock)
+{
+ return __spin_trylock(lock);
+}
+
+static inline void spin_unlock(spinlock_t *lock)
+{
+ __spin_unlock(lock);
+}
+
+static inline void spinlock_debug(spinlock_t *lock)
+{
+}
+
+#endif /* CONFIG_SPINLOCK_DEBUG */
+
+/* Inlines, defined below */
+static inline void spinlock_init(spinlock_t *lock);
+static inline void spinlock_init_irqsave(spinlock_t *lock);
+static inline void spin_lock_irqsave(spinlock_t *lock);
+static inline bool spin_trylock_irqsave(spinlock_t *lock);
+static inline void spin_unlock_irqsave(spinlock_t *lock);
+static inline bool spin_lock_irq_enabled(spinlock_t *lock);
+
+/* Hash locks (array of spinlocks). Most all users will want the default one,
+ * so point your pointer to one of them, though you could always kmalloc a
+ * bigger one. In the future, they might be growable, etc, which init code may
+ * care about. */
+struct hashlock {
+ unsigned int nr_entries;
+ struct spinlock locks[];
+};
+#define HASHLOCK_DEFAULT_SZ 53 /* nice prime, might be a bit large */
+struct small_hashlock {
+ unsigned int nr_entries;
+ struct spinlock locks[HASHLOCK_DEFAULT_SZ];
+};
+
+void hashlock_init(struct hashlock *hl, unsigned int nr_entries);
+void hashlock_init_irqsave(struct hashlock *hl, unsigned int nr_entries);
+void hash_lock(struct hashlock *hl, long key);
+void hash_unlock(struct hashlock *hl, long key);
+void hash_lock_irqsave(struct hashlock *hl, long key);
+void hash_unlock_irqsave(struct hashlock *hl, long key);
+
+/* Seq locks */
/* An example seq lock, built from the counter. I don't particularly like this,
* since it forces you to use a specific locking type. */
typedef struct seq_lock {
static inline seq_ctr_t read_seqbegin(seqlock_t *lock);
static inline bool read_seqretry(seqlock_t *lock, seq_ctr_t ctr);
+/* Post work and poke synchronization. This is a wait-free way to make sure
+ * some code is run, usually by the calling core, but potentially by any core.
+ * Under contention, everyone just posts work, and one core will carry out the
+ * work. Callers post work (the meaning of which is particular to their
+ * subsystem), then call this function. The function is not run concurrently
+ * with itself.
+ *
+ * In the future, this may send RKMs to LL cores to ensure the work is done
+ * somewhere, but not necessarily on the calling core. Will reserve 'flags'
+ * for that. */
+struct poke_tracker {
+ atomic_t need_to_run;
+ atomic_t run_in_progress;
+ void (*func)(void *);
+};
+void poke(struct poke_tracker *tracker, void *arg);
+
+static inline void poke_init(struct poke_tracker *tracker, void (*func)(void*))
+{
+ tracker->need_to_run = 0;
+ tracker->run_in_progress = 0;
+ tracker->func = func;
+}
+#define POKE_INITIALIZER(f) {.func = f}
+
+/* Arch-specific implementations / declarations go here */
+#include <arch/atomic.h>
+
#define MAX_SPINS 1000000000
/* Will spin for a little while, but not deadlock if it never happens */
typedef struct checklist_mask {
// only need an uint8_t, but we need the bits[] to be word aligned
uint32_t size;
- volatile uint8_t (COUNT(BYTES_FOR_BITMASK(size)) bits)[MAX_NUM_CPUS];
+ volatile uint8_t bits[MAX_NUM_CORES];
} checklist_mask_t;
// mask contains an unspecified array, so it needs to be at the bottom
// eagle-eyed readers may know why this might have been needed. 2009-09-04
//volatile uint8_t (COUNT(BYTES_FOR_BITMASK(size)) bits)[];
};
-typedef struct checklist RACY checklist_t;
+typedef struct checklist checklist_t;
#define ZEROS_ARRAY(size) {[0 ... ((size)-1)] 0}
#define DEFAULT_CHECKLIST_MASK(sz) {(sz), ZEROS_ARRAY(BYTES_FOR_BITMASK(sz))}
-#define DEFAULT_CHECKLIST(sz) {SPINLOCK_INITIALIZER, DEFAULT_CHECKLIST_MASK(sz)}
+#define DEFAULT_CHECKLIST(sz) {SPINLOCK_INITIALIZER_IRQSAVE, \
+ DEFAULT_CHECKLIST_MASK(sz)}
#define INIT_CHECKLIST(nm, sz) \
checklist_t nm = DEFAULT_CHECKLIST(sz);
#define INIT_CHECKLIST_MASK(nm, sz) \
int release_checklist(checklist_t* list);
int checklist_is_locked(checklist_t* list);
int checklist_is_clear(checklist_t* list);
+int checklist_is_full(checklist_t* list);
void reset_checklist(checklist_t* list);
void down_checklist(checklist_t* list);
// TODO - do we want to adjust the size? (YES, don't want to check it all)
volatile uint8_t ready;
};
-typedef struct barrier RACY barrier_t;
+typedef struct barrier barrier_t;
-void init_barrier(barrier_t*COUNT(1) barrier, uint32_t count);
+void init_barrier(barrier_t *barrier, uint32_t count);
void reset_barrier(barrier_t* barrier);
void waiton_barrier(barrier_t* barrier);
/* Spinlock bit flags */
#define SPINLOCK_IRQ_EN 0x80000000
+static inline void spinlock_init(spinlock_t *lock)
+{
+ __spinlock_init(lock);
+#ifdef CONFIG_SPINLOCK_DEBUG
+ lock->call_site = 0;
+ lock->calling_core = 0;
+ lock->irq_okay = FALSE;
+#endif
+}
+
+static inline void spinlock_init_irqsave(spinlock_t *lock)
+{
+ __spinlock_init(lock);
+#ifdef CONFIG_SPINLOCK_DEBUG
+ lock->call_site = 0;
+ lock->calling_core = 0;
+ lock->irq_okay = TRUE;
+#endif
+}
+
// If ints are enabled, disable them and note it in the top bit of the lock
// There is an assumption about releasing locks in order here...
-static inline void spin_lock_irqsave(spinlock_t *SAFE lock)
+static inline void spin_lock_irqsave(spinlock_t *lock)
{
uint32_t irq_en;
irq_en = irq_is_enabled();
lock->rlock |= SPINLOCK_IRQ_EN;
}
+static inline bool spin_trylock_irqsave(spinlock_t *lock)
+{
+ uint32_t irq_en = irq_is_enabled();
+
+ disable_irq();
+ if (!spin_trylock(lock)) {
+ if (irq_en)
+ enable_irq();
+ return FALSE;
+ }
+ if (irq_en)
+ lock->rlock |= SPINLOCK_IRQ_EN;
+ return TRUE;
+}
+
// if the high bit of the lock is set, then re-enable interrupts
// (note from asw: you're lucky this works, you little-endian jerks)
-static inline void spin_unlock_irqsave(spinlock_t *SAFE lock)
+static inline void spin_unlock_irqsave(spinlock_t *lock)
{
if (spin_lock_irq_enabled(lock)) {
spin_unlock(lock);
/* Returns whether or not unlocking this lock should enable interrupts or not.
* Is meaningless on locks that weren't locked with irqsave. */
-static inline bool spin_lock_irq_enabled(spinlock_t *SAFE lock)
+static inline bool spin_lock_irq_enabled(spinlock_t *lock)
{
return lock->rlock & SPINLOCK_IRQ_EN;
}
* concurrent write. */
static inline void __seq_start_write(seq_ctr_t *seq_ctr)
{
-#ifdef _CONFIG_SEQLOCK_DEBUG_
+#ifdef CONFIG_SEQLOCK_DEBUG
assert(*seq_ctr % 2 == 0);
#endif
(*seq_ctr)++;
static inline void __seq_end_write(seq_ctr_t *seq_ctr)
{
-#ifdef _CONFIG_SEQLOCK_DEBUG_
+#ifdef CONFIG_SEQLOCK_DEBUG
assert(*seq_ctr % 2 == 1);
#endif
/* Need to prevent the compiler (and some arches) from reordering older
static inline seq_ctr_t read_seqbegin(seqlock_t *lock)
{
- return lock->r_ctr;
+ seq_ctr_t retval = lock->r_ctr;
+ rmb(); /* don't want future reads to come before our ctr read */
+ return retval;
}
static inline bool read_seqretry(seqlock_t *lock, seq_ctr_t ctr)
{
return seqctr_retry(lock->r_ctr, ctr);
}
-
-#endif /* !ROS_KERN_ATOMIC_H */
projects / akaros.git / blobdiff