[sched] New thread locking scheme.

include/bitstring.h

@@ -51,6 +51,9 @@
  * works."
  *  /s/ Perry E. Metzger, 2 Feb 98
  */
+#include <stddef.h>
+#include <stdint.h>
+
 typedef unsigned char bitstr_t;
 
 /* internal macros */

include/sys/context.h

@@ -5,6 +5,7 @@
 #include <sys/types.h>
 #include <stdbool.h>
 
+typedef struct mtx mtx_t;
 typedef struct thread thread_t;
 typedef struct __ucontext ucontext_t;
 typedef struct ctx ctx_t;
@@ -47,7 +48,7 @@ void mcontext_restart_syscall(mcontext_t *ctx);
  *
  * \note must be called with interrupts disabled!
  */
-void ctx_switch(thread_t *from, thread_t *to);
+void ctx_switch(thread_t *from, thread_t *to, mtx_t *mtx);
 
 /* Implementation of setcontext syscall. */
 int do_setcontext(thread_t *td, ucontext_t *uc);

include/sys/mutex.h

@@ -60,6 +60,8 @@ typedef struct mtx {
 #define MTX_DEFINE(mutexname, type)                                            \
   mtx_t mutexname = MTX_INITIALIZER(mutexname, type)
 
+void init_mtx(void);
+
 /*! \brief Initializes mutex.
  *
  * \note Every mutex has to be initialized before it is used. */
@@ -120,4 +122,6 @@ DEFINE_CLEANUP_FUNCTION(mtx_t *, mtx_unlock);
 #define WITH_MTX_LOCK(mtx_p)                                                   \
   WITH_STMT(mtx_t, mtx_lock, CLEANUP_FUNCTION(mtx_unlock), mtx_p)
 
+extern mtx_t blocked_lock;
+
 #endif /* !_SYS_MUTEX_H_ */

include/sys/pcpu.h

@@ -1,9 +1,9 @@
 #ifndef _SYS_PCPU_H_
 #define _SYS_PCPU_H_
 
+#include <stdbool.h>
 #include <machine/types.h>
 #include <machine/pcpu.h>
-#include <stdbool.h>
 
 typedef struct thread thread_t;
 typedef struct pmap pmap_t;

include/sys/runq.h

@@ -3,19 +3,19 @@
 
 #ifdef _KERNEL
 
+#include <bitstring.h>
 #include <sys/cdefs.h>
 #include <sys/queue.h>
 
 typedef struct thread thread_t;
+typedef TAILQ_HEAD(, thread) thread_list_t;
 
-/* TODO How to prevent tests from using following values? */
 #define RQ_NQS 64 /* Number of run queues. */
 #define RQ_PPQ 4  /* Priorities per queue. */
 
-TAILQ_HEAD(rq_head, thread);
-
-typedef struct {
-  struct rq_head rq_queues[RQ_NQS];
+typedef struct runq {
+  bitstr_t bit_decl(rq_status, RQ_NQS);
+  thread_list_t rq_queues[RQ_NQS];
 } runq_t;
 
 /* Initialize a run structure. */

include/sys/sched.h

@@ -43,6 +43,8 @@ static inline void __preempt_enable(void *data) {
 #define WITH_NO_PREEMPTION                                                     \
   WITH_STMT(void, __preempt_disable, __preempt_enable, NULL)
 
+void sched_init(thread_t *td);
+
 /*! \brief Add new thread to the scheduler.
  *
  * The thread will be set runnable. */

include/sys/sleepq.h

@@ -13,6 +13,10 @@ typedef struct sleepq sleepq_t;
 /*! \brief Called during kernel initialization. */
 void init_sleepq(void);
 
+void sleepq_lock(void *wchan);
+
+void sleepq_unlock(void *wchan);
+
 /*! \brief Allocates sleep queue entry. */
 sleepq_t *sleepq_alloc(void);
 
@@ -54,7 +58,7 @@ bool sleepq_broadcast(void *wchan);
 /*! \brief Break thread's sleep.
  *
  * \returns true on success
- * \returns false if the thread has not been asleep
+ * \returns false if the thread has not been aborted
  */
 bool sleepq_abort(thread_t *td);
 

include/sys/thread.h

@@ -214,6 +214,14 @@ void thread_reap(void);
  * Must be called with acquired td_lock. */
 void thread_continue(thread_t *td);
 
+static inline bool thread_lock_eq(thread_t *td, mtx_t *mtx) {
+  return td->td_lock == mtx;
+}
+
+void thread_lock_set(thread_t *td, mtx_t *mtx);
+
+mtx_t *thread_lock_block(thread_t *td);
+
 /* Please use following functions to read state of a thread! */
 static inline bool td_is_ready(thread_t *td) {
   return td->td_state == TDS_READY;

include/sys/turnstile.h

@@ -4,11 +4,16 @@
 #include <sys/cdefs.h>
 #include <sys/priority.h>
 
+typedef struct mtx mtx_t;
 typedef struct turnstile turnstile_t;
 
 /*! \brief Called during kernel initialization. */
 void init_turnstile(void);
 
+void turnstile_lock(mtx_t *mtx);
+
+void turnstile_unlock(mtx_t *mtx);
+
 /*! \brief Allocates turnstile entry. */
 turnstile_t *turnstile_alloc(void);
 
@@ -28,18 +33,12 @@ void turnstile_destroy(turnstile_t *ts);
  * \note Requires td_spin acquired. */
 void turnstile_adjust(thread_t *td, prio_t oldprio);
 
-/* Provide turnstile that we're going to block on. */
-turnstile_t *turnstile_take(void *wchan);
-
-/* Release turnstile in case we decided not to block on it. */
-void turnstile_give(turnstile_t *ts);
-
 /* Block the current thread on given turnstile. This function will perform
  * context switch and release turnstile when woken up. */
-void turnstile_wait(turnstile_t *ts, thread_t *owner, const void *waitpt);
+void turnstile_wait(mtx_t *mtx, const void *waitpt);
 
-/* Wakeup all threads waiting on given channel and adjust the priority of the
+/* Wakeup all threads waiting on given mutex and adjust the priority of the
  * current thread appropriately. */
-void turnstile_broadcast(void *wchan);
+void turnstile_broadcast(mtx_t *mtx);
 
 #endif /* !_SYS_TURNSTILE_H_ */

sys/drv/rtc.c

@@ -69,8 +69,11 @@ static intr_filter_t rtc_intr(void *data) {
   rtc_state_t *rtc = data;
   uint8_t regc = rtc_read(rtc->regs, MC_REGC);
   if (regc & MC_REGC_PF) {
-    if (rtc->counter++ & 1)
+    if (rtc->counter++ & 1) {
+      sleepq_lock(rtc);
       sleepq_signal(rtc);
+      sleepq_unlock(rtc);
+    }
     return IF_FILTERED;
   }
   return IF_STRAY;
@@ -80,6 +83,7 @@ static int rtc_time_read(devnode_t *dev, uio_t *uio) {
   rtc_state_t *rtc = dev->data;
   tm_t t;
 
+  sleepq_lock(rtc);
   sleepq_wait(rtc, NULL);
   rtc_gettime(rtc->regs, &t);
   int count = snprintf(rtc->asctime, RTC_ASCTIME_SIZE, "%d %d %d %d %d %d",

sys/kern/callout.c

@@ -7,7 +7,6 @@
 #include <sys/sleepq.h>
 #include <sys/thread.h>
 #include <sys/sched.h>
-#include <sys/interrupt.h>
 #include <sys/time.h>
 
 /* Note: If the difference in time between ticks is greater than the number of
@@ -53,14 +52,16 @@ static void callout_thread(void *arg) {
   while (true) {
     callout_t *elem;
 
-    WITH_INTR_DISABLED {
-      while (TAILQ_EMPTY(&delegated)) {
-        sleepq_wait(&delegated, NULL);
-      }
+    sleepq_lock(&delegated);
 
-      elem = TAILQ_FIRST(&delegated);
-      TAILQ_REMOVE(&delegated, elem, c_link);
+    while (TAILQ_EMPTY(&delegated)) {
+      sleepq_wait(&delegated, NULL);
+      sleepq_lock(&delegated);
     }
+    elem = TAILQ_FIRST(&delegated);
+    TAILQ_REMOVE(&delegated, elem, c_link);
+
+    sleepq_unlock(&delegated);
 
     assert(callout_is_active(elem));
     assert(!callout_is_pending(elem));
@@ -163,6 +164,8 @@ bool callout_stop(callout_t *handle) {
  * current position and delegate them to callout thread.
  */
 void callout_process(systime_t time) {
+  SCOPED_MTX_LOCK(&ci.lock);
+
   unsigned int last_bucket;
   unsigned int current_bucket = ci.last % CALLOUT_BUCKETS;
 
@@ -174,8 +177,7 @@ void callout_process(systime_t time) {
     last_bucket = time % CALLOUT_BUCKETS;
   }
 
-  /* We are in kernel's bottom half. */
-  assert(intr_disabled());
+  sleepq_lock(&delegated);
 
   while (true) {
     callout_list_t *head = ci_list(current_bucket);
@@ -204,12 +206,18 @@ void callout_process(systime_t time) {
   if (!TAILQ_EMPTY(&delegated)) {
     sleepq_signal(&delegated);
   }
+
+  sleepq_unlock(&delegated);
 }
 
 bool callout_drain(callout_t *handle) {
-  SCOPED_INTR_DISABLED();
-  if (!callout_is_pending(handle) && !callout_is_active(handle))
+  sleepq_lock(handle);
+
+  if (!callout_is_pending(handle) && !callout_is_active(handle)) {
+    sleepq_unlock(handle);
     return false;
+  }
+
   while (callout_is_pending(handle) || callout_is_active(handle))
     sleepq_wait(handle, NULL);
   return true;

sys/kern/condvar.c

@@ -1,48 +1,45 @@
 #include <sys/condvar.h>
+#include <sys/mutex.h>
 #include <sys/sleepq.h>
-#include <sys/sched.h>
-#include <sys/interrupt.h>
 
 void cv_init(condvar_t *cv, const char *name) {
   cv->name = name;
   cv->waiters = 0;
 }
 
 void cv_wait(condvar_t *cv, mtx_t *m) {
-  WITH_INTR_DISABLED {
-    cv->waiters++;
-    mtx_unlock(m);
-    /* If we got interrupted here and an interrupt filter called
-     * cv_signal, we would have a lost wakeup, so we need interrupts
-     * to be disabled. Same goes for cv_wait_timed. */
-    sleepq_wait(cv, __caller(0));
-  }
-  _mtx_lock(m, __caller(0));
+  sleepq_lock(cv);
+  cv->waiters++;
+  mtx_unlock(m);
+  sleepq_wait(cv, __caller(0));
+  mtx_lock(m);
 }
 
 int cv_wait_timed(condvar_t *cv, mtx_t *m, systime_t timeout) {
-  int status;
-  WITH_INTR_DISABLED {
-    cv->waiters++;
-    mtx_unlock(m);
-    status = sleepq_wait_timed(cv, __caller(0), timeout);
-  }
-  _mtx_lock(m, __caller(0));
+  sleepq_lock(cv);
+  cv->waiters++;
+  mtx_unlock(m);
+  int status = sleepq_wait_timed(cv, __caller(0), timeout);
+  mtx_lock(m);
   return status;
 }
 
 void cv_signal(condvar_t *cv) {
-  SCOPED_NO_PREEMPTION();
-  if (cv->waiters > 0) {
-    cv->waiters--;
-    sleepq_signal(cv);
-  }
+  sleepq_lock(cv);
+  if (!cv->waiters)
+    goto end;
+  cv->waiters--;
+  sleepq_signal(cv);
+end:
+  sleepq_unlock(cv);
 }
 
 void cv_broadcast(condvar_t *cv) {
-  SCOPED_NO_PREEMPTION();
-  if (cv->waiters > 0) {
-    cv->waiters = 0;
-    sleepq_broadcast(cv);
-  }
+  sleepq_lock(cv);
+  if (!cv->waiters)
+    goto end;
+  cv->waiters--;
+  sleepq_broadcast(cv);
+end:
+  sleepq_unlock(cv);
 }

sys/kern/interrupt.c

@@ -209,7 +209,9 @@ void intr_event_run_handlers(intr_event_t *ie) {
 
   if (ie_status & IF_DELEGATE) {
     ie_disable(ie);
+    sleepq_lock(ie);
     sleepq_signal(ie);
+    sleepq_unlock(ie);
   }
 
   if (ie_status == IF_STRAY)
@@ -269,13 +271,10 @@ static void intr_thread(void *arg) {
     }
 
     /* If there are still handlers assigned to the interrupt event, enable
-     * interrupts and wait for a wakeup. We do it with interrupts disabled
-     * to prevent the wakeup from being lost. */
-    WITH_INTR_DISABLED {
-      if (!TAILQ_EMPTY(&ie->ie_handlers))
-        ie_enable(ie);
-
-      sleepq_wait(ie, NULL);
-    }
+     * interrupts and wait for a wakeup. */
+    sleepq_lock(ie);
+    if (!TAILQ_EMPTY(&ie->ie_handlers))
+      ie_enable(ie);
+    sleepq_wait(ie, NULL);
   }
 }

sys/kern/main.c

@@ -92,6 +92,7 @@ __noreturn void kernel_init(void) {
   /* Make dispatcher & scheduler structures ready for use. */
   init_sleepq();
   init_turnstile();
+  init_mtx();
   lockdep_init();
   init_thread0();
   init_sched();