63#include "opt_sleepqueue_profiling.h"
71#include <sys/kernel.h>
78#include <sys/signalvar.h>
79#include <sys/sleepqueue.h>
81#include <sys/sysctl.h>
87#include <machine/atomic.h>
100#define SC_TABLESIZE 256
103#define SC_MASK (SC_TABLESIZE - 1)
105#define SC_HASH(wc) ((((uintptr_t)(wc) >> SC_SHIFT) ^ (uintptr_t)(wc)) & \
107#define SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]
132 const void *sq_wchan;
135 struct lock_object *sq_lock;
142#ifdef SLEEPQUEUE_PROFILING
148#ifdef SLEEPQUEUE_PROFILING
149static SYSCTL_NODE(_debug, OID_AUTO, sleepq, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
152 CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
153 "sleepq chain stats");
154static u_int sleepq_max_depth;
155SYSCTL_UINT(_debug_sleepq, OID_AUTO, max_depth, CTLFLAG_RD, &sleepq_max_depth,
156 0,
"maxmimum depth achieved of a single chain");
158static void sleepq_profile(
const char *wmesg);
159static int prof_enabled;
171static void sleepq_dtor(
void *mem,
int size,
void *arg);
175 int pri,
int srqflags);
188#ifdef SLEEPQUEUE_PROFILING
190init_sleepqueue_profiling(
void)
193 struct sysctl_oid *chain_oid;
197 snprintf(chain_name,
sizeof(chain_name),
"%u", i);
198 chain_oid = SYSCTL_ADD_NODE(NULL,
199 SYSCTL_STATIC_CHILDREN(_debug_sleepq_chains), OID_AUTO,
200 chain_name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
201 "sleepq chain stats");
202 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
204 SYSCTL_ADD_UINT(NULL, SYSCTL_CHILDREN(chain_oid), OID_AUTO,
210SYSINIT(sleepqueue_profiling, SI_SUB_LOCK, SI_ORDER_ANY,
211 init_sleepqueue_profiling, NULL);
230 NULL, sleepq_dtor,
sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
232 NULL, NULL,
sleepq_init, NULL, UMA_ALIGN_CACHE, 0);
267 mtx_lock_spin(&sc->sc_lock);
281 KASSERT(wchan != NULL, (
"%s: invalid NULL wait channel", __func__));
283 mtx_assert(&sc->sc_lock, MA_OWNED);
284 LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
285 if (sq->sq_wchan == wchan)
299 mtx_unlock_spin(&sc->sc_lock);
309sleepq_add(
const void *wchan,
struct lock_object *lock,
const char *wmesg,
310 int flags,
int queue)
318 mtx_assert(&sc->sc_lock, MA_OWNED);
319 MPASS(td->td_sleepqueue != NULL);
320 MPASS(wchan != NULL);
324 if (__predict_false(!THREAD_CAN_SLEEP())) {
326 epoch_trace_list(curthread);
329 (
"%s: td %p to sleep on wchan %p with sleeping prohibited",
330 __func__, td, wchan));
345 sq = td->td_sleepqueue;
348 (
"thread's sleep queue %d is not empty", i));
350 (
"thread's sleep queue %d count mismatches", i));
352 KASSERT(LIST_EMPTY(&sq->sq_free),
353 (
"thread's sleep queue has a non-empty free list"));
354 KASSERT(sq->sq_wchan == NULL, (
"stale sq_wchan pointer"));
357#ifdef SLEEPQUEUE_PROFILING
359 if (sc->sc_depth > sc->sc_max_depth) {
360 sc->sc_max_depth = sc->sc_depth;
361 if (sc->sc_max_depth > sleepq_max_depth)
362 sleepq_max_depth = sc->sc_max_depth;
365 sq = td->td_sleepqueue;
366 LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
367 sq->sq_wchan = wchan;
368 sq->sq_type =
flags & SLEEPQ_TYPE;
370 MPASS(wchan == sq->sq_wchan);
371 MPASS(lock == sq->sq_lock);
372 MPASS((
flags & SLEEPQ_TYPE) == sq->sq_type);
373 LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
376 TAILQ_INSERT_TAIL(&sq->
sq_blocked[queue], td, td_slpq);
378 td->td_sleepqueue = NULL;
379 td->td_sqqueue = queue;
380 td->td_wchan = wchan;
381 td->td_wmesg = wmesg;
382 if (
flags & SLEEPQ_INTERRUPTIBLE) {
384 td->td_flags |= TDF_SINTR;
386 td->td_flags &= ~TDF_TIMEOUT;
404 mtx_assert(&sc->sc_lock, MA_OWNED);
405 MPASS(TD_ON_SLEEPQ(td));
406 MPASS(td->td_sleepqueue == NULL);
407 MPASS(wchan != NULL);
408 if (cold && td == &thread0)
409 panic(
"timed sleep before timers are working");
410 KASSERT(td->td_sleeptimo == 0, (
"td %d %p td_sleeptimo %jx",
411 td->td_tid, td, (uintmax_t)td->td_sleeptimo));
428 KASSERT(wchan != NULL, (
"%s: invalid NULL wait channel", __func__));
442 mtx_assert(&sc->sc_lock, MA_OWNED);
444 if ((td->td_pflags & TDP_WAKEUP) != 0) {
445 td->td_pflags &= ~TDP_WAKEUP;
455 if ((td->td_flags & (TDF_NEEDSIGCHK | TDF_NEEDSUSPCHK)) == 0)
459 mtx_unlock_spin(&sc->sc_lock);
462 CTR3(KTR_PROC,
"sleepq catching signals: thread %p (pid %ld, %s)",
463 (
void *)td, (
long)p->p_pid, td->td_name);
474 mtx_lock_spin(&sc->sc_lock);
488 mtx_lock_spin(&sc->sc_lock);
511 mtx_assert(&sc->sc_lock, MA_OWNED);
512 MPASS(wchan != NULL);
516 THREAD_LOCK_ASSERT(td, MA_OWNED);
517 mtx_assert(&sc->sc_lock, MA_OWNED);
530 if (TD_ON_SLEEPQ(td)) {
534 MPASS(td->td_lock != &sc->sc_lock);
535 mtx_unlock_spin(&sc->sc_lock);
555 mtx_assert(&sc->sc_lock, MA_OWNED);
556 THREAD_LOCK_ASSERT(td, MA_OWNED);
562 if (td->td_sleepqueue != NULL) {
563 mtx_unlock_spin(&sc->sc_lock);
587 rtc_changed = td->td_rtcgen != 0 && td->td_rtcgen !=
rtc_generation;
588 if ((td->td_flags & TDF_TIMEOUT) || rtc_changed) {
592 MPASS(TD_ON_SLEEPQ(td));
595 mtx_unlock_spin(&sc->sc_lock);
599#ifdef SLEEPQUEUE_PROFILING
601 sleepq_profile(td->td_wmesg);
603 MPASS(td->td_sleepqueue == NULL);
606 SDT_PROBE0(sched, , , sleep);
609 KASSERT(TD_IS_RUNNING(td), (
"running but not TDS_RUNNING"));
610 CTR3(KTR_PROC,
"sleepq resume: thread %p (pid %ld, %s)",
611 (
void *)td, (
long)td->td_proc->p_pid, (
void *)td->td_name);
625 if (td->td_sleeptimo != 0) {
626 if (td->td_sleeptimo <= sbinuptime())
628 td->td_sleeptimo = 0;
642 KASSERT((td->td_flags & TDF_SINTR) == 0,
643 (
"thread %p still in interruptible sleep?", td));
645 return (td->td_intrval);
657 MPASS(!(td->td_flags & TDF_SINTR));
687 MPASS(!(td->td_flags & TDF_SINTR));
702 int rcatch, rvalt, rvals;
724 MPASS(wchan != NULL);
749 MPASS(sq->sq_wchan != NULL);
750 MPASS(td->td_wchan == sq->sq_wchan);
753 mtx_assert(&sc->sc_lock, MA_OWNED);
764 if (!TD_IS_SLEEPING(td)) {
774 if ((srqflags & SRQ_HOLD) == 0 && drop)
775 mtx_unlock_spin(&sc->sc_lock);
778 MPASS(pri == 0 || (pri >= PRI_MIN && pri <= PRI_MAX));
779 if (pri != 0 && td->td_priority > pri &&
780 PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
789 if (TD_IS_SLEEPING(td)) {
806 MPASS(sq->sq_wchan != NULL);
807 MPASS(td->td_wchan == sq->sq_wchan);
808 MPASS(td->td_sqqueue <
NR_SLEEPQS && td->td_sqqueue >= 0);
809 THREAD_LOCK_ASSERT(td, MA_OWNED);
811 mtx_assert(&sc->sc_lock, MA_OWNED);
813 SDT_PROBE2(sched, , ,
wakeup, td, td->td_proc);
817 TAILQ_REMOVE(&sq->
sq_blocked[td->td_sqqueue], td, td_slpq);
824 if (LIST_EMPTY(&sq->sq_free)) {
825 td->td_sleepqueue = sq;
829#ifdef SLEEPQUEUE_PROFILING
833 td->td_sleepqueue = LIST_FIRST(&sq->sq_free);
834 LIST_REMOVE(td->td_sleepqueue, sq_hash);
836 if ((td->td_flags & TDF_TIMEOUT) == 0 && td->td_sleeptimo != 0 &&
837 td->td_lock == &sc->sc_lock) {
855 callout_stop(&td->td_slpcallout);
860 td->td_flags &= ~(TDF_SINTR | TDF_TIMEOUT);
862 CTR3(KTR_PROC,
"sleepq_wakeup: thread %p (pid %ld, %s)",
863 (
void *)td, (
long)td->td_proc->p_pid, td->td_name);
873 MPASS(TD_ON_SLEEPQ(td));
875 wchan = td->td_wchan;
877 mtx_lock_spin(&sc->sc_lock);
882 mtx_unlock_spin(&sc->sc_lock);
891sleepq_dtor(
void *mem,
int size,
void *arg)
919 LIST_INIT(&sq->sq_free);
931 struct threadqueue *head;
932 struct thread *td, *besttd;
935 CTR2(KTR_PROC,
"sleepq_signal(%p, %d)", wchan,
flags);
936 KASSERT(wchan != NULL, (
"%s: invalid NULL wait channel", __func__));
940 if (
flags & SLEEPQ_DROP)
944 KASSERT(sq->sq_type == (
flags & SLEEPQ_TYPE),
945 (
"%s: mismatch between sleep/wakeup and cv_*", __func__));
948 if (
flags & SLEEPQ_UNFAIR) {
955 besttd = TAILQ_LAST_FAST(head, thread, td_slpq);
956 while (besttd->td_lock != &sc->sc_lock) {
957 td = TAILQ_PREV_FAST(besttd, head, thread, td_slpq);
969 besttd = td = TAILQ_FIRST(head);
970 while ((td = TAILQ_NEXT(td, td_slpq)) != NULL) {
971 if (td->td_priority < besttd->td_priority)
975 MPASS(besttd != NULL);
977 (
flags & SLEEPQ_DROP) ? 0 : SRQ_HOLD);
978 return (wakeup_swapper);
996 CTR2(KTR_PROC,
"sleepq_broadcast(%p, %d)", wchan,
flags);
997 KASSERT(wchan != NULL, (
"%s: invalid NULL wait channel", __func__));
1002 KASSERT(sq->sq_type == (
flags & SLEEPQ_TYPE),
1003 (
"%s: mismatch between sleep/wakeup and cv_*", __func__));
1013 bool (*matches)(
struct thread *),
int pri)
1015 struct thread *td, *tdn;
1025 TAILQ_FOREACH_SAFE(td, &sq->
sq_blocked[queue], td_slpq, tdn) {
1031 return (wakeup_swapper);
1048 CTR3(KTR_PROC,
"sleepq_timeout: thread %p (pid %ld, %s)",
1049 (
void *)td, (
long)td->td_proc->p_pid, (
void *)td->td_name);
1052 if (td->td_sleeptimo == 0 ||
1053 td->td_sleeptimo > td->td_slpcallout.c_time) {
1057 }
else if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) {
1062 wchan = td->td_wchan;
1064 THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock);
1067 td->td_flags |= TDF_TIMEOUT;
1072 }
else if (TD_ON_SLEEPQ(td)) {
1079 td->td_flags |= TDF_TIMEOUT;
1100 MPASS(wchan != NULL);
1102 mtx_lock_spin(&sc->sc_lock);
1109 if (!TD_ON_SLEEPQ(td) || td->td_wchan != wchan) {
1110 mtx_unlock_spin(&sc->sc_lock);
1117 MPASS(td->td_wchan == wchan);
1135 THREAD_LOCK_ASSERT(td, MA_OWNED);
1136 MPASS(TD_ON_SLEEPQ(td));
1137 MPASS(td->td_flags & TDF_SINTR);
1138 MPASS((intrval == 0 && (td->td_flags & TDF_SIGWAIT) != 0) ||
1139 intrval == EINTR || intrval == ERESTART);
1145 if (td->td_flags & TDF_TIMEOUT) {
1150 CTR3(KTR_PROC,
"sleepq_abort: thread %p (pid %ld, %s)",
1151 (
void *)td, (
long)td->td_proc->p_pid, (
void *)td->td_name);
1152 td->td_intrval = intrval;
1159 if (!TD_IS_SLEEPING(td)) {
1163 wchan = td->td_wchan;
1164 MPASS(wchan != NULL);
1177 int i, wakeup_swapper;
1181 if (LIST_EMPTY(&sc->sc_queues)) {
1184 mtx_lock_spin(&sc->sc_lock);
1185 LIST_FOREACH_SAFE(sq, &sc->sc_queues, sq_hash, sq1) {
1191 mtx_unlock_spin(&sc->sc_lock);
1193 if (wakeup_swapper) {
1206sleepq_sbuf_print_stacks(
struct sbuf *sb,
const void *wchan,
int queue,
1207 int *count_stacks_printed)
1209 struct thread *td, *td_next;
1212 struct sbuf **td_infos;
1213 int i, stack_idx, error, stacks_to_allocate;
1219 KASSERT(wchan != NULL, (
"%s: invalid NULL wait channel", __func__));
1222 stacks_to_allocate = 10;
1223 for (i = 0; i < 3 && !finished ; i++) {
1236 st =
malloc(
sizeof(
struct stack *) * stacks_to_allocate,
1238 for (stack_idx = 0; stack_idx < stacks_to_allocate;
1243 td_infos =
malloc(
sizeof(
struct sbuf *) * stacks_to_allocate,
1245 for (stack_idx = 0; stack_idx < stacks_to_allocate;
1247 td_infos[stack_idx] =
sbuf_new(NULL, NULL,
1248 MAXCOMLEN +
sizeof(
struct thread *) * 2 + 40,
1263 TAILQ_FOREACH_SAFE(td, &sq->
sq_blocked[queue], td_slpq,
1265 if (stack_idx >= stacks_to_allocate)
1269 (void)stack_save_td(
st[stack_idx], td);
1272 td->td_tid, td->td_name, td);
1281 for (i = 0; i < stack_idx; i++) {
1289 *count_stacks_printed = stack_idx;
1295 for (stack_idx = 0; stack_idx < stacks_to_allocate;
1298 for (stack_idx = 0; stack_idx < stacks_to_allocate;
1302 free(td_infos, M_TEMP);
1303 stacks_to_allocate *= 10;
1306 if (!finished && error == 0)
1313#ifdef SLEEPQUEUE_PROFILING
1314#define SLEEPQ_PROF_LOCATIONS 1024
1315#define SLEEPQ_SBUFSIZE 512
1317 LIST_ENTRY(sleepq_prof) sp_link;
1318 const char *sp_wmesg;
1324struct sqphead sleepq_prof_free;
1326static struct sleepq_prof sleepq_profent[SLEEPQ_PROF_LOCATIONS];
1327static struct mtx sleepq_prof_lock;
1328MTX_SYSINIT(sleepq_prof_lock, &sleepq_prof_lock,
"sleepq_prof", MTX_SPIN);
1331sleepq_profile(
const char *wmesg)
1333 struct sleepq_prof *sp;
1335 mtx_lock_spin(&sleepq_prof_lock);
1336 if (prof_enabled == 0)
1338 LIST_FOREACH(sp, &sleepq_hash[
SC_HASH(wmesg)], sp_link)
1339 if (sp->sp_wmesg == wmesg)
1341 sp = LIST_FIRST(&sleepq_prof_free);
1344 sp->sp_wmesg = wmesg;
1345 LIST_REMOVE(sp, sp_link);
1346 LIST_INSERT_HEAD(&sleepq_hash[
SC_HASH(wmesg)], sp, sp_link);
1350 mtx_unlock_spin(&sleepq_prof_lock);
1355sleepq_prof_reset(
void)
1357 struct sleepq_prof *sp;
1361 mtx_lock_spin(&sleepq_prof_lock);
1362 enabled = prof_enabled;
1365 LIST_INIT(&sleepq_hash[i]);
1366 LIST_INIT(&sleepq_prof_free);
1367 for (i = 0; i < SLEEPQ_PROF_LOCATIONS; i++) {
1368 sp = &sleepq_profent[i];
1369 sp->sp_wmesg = NULL;
1371 LIST_INSERT_HEAD(&sleepq_prof_free, sp, sp_link);
1373 prof_enabled = enabled;
1374 mtx_unlock_spin(&sleepq_prof_lock);
1378enable_sleepq_prof(SYSCTL_HANDLER_ARGS)
1386 if (req->newptr == NULL)
1388 if (v == prof_enabled)
1391 sleepq_prof_reset();
1392 mtx_lock_spin(&sleepq_prof_lock);
1394 mtx_unlock_spin(&sleepq_prof_lock);
1400reset_sleepq_prof_stats(SYSCTL_HANDLER_ARGS)
1408 if (req->newptr == NULL)
1412 sleepq_prof_reset();
1418dump_sleepq_prof_stats(SYSCTL_HANDLER_ARGS)
1420 struct sleepq_prof *sp;
1431 enabled = prof_enabled;
1432 mtx_lock_spin(&sleepq_prof_lock);
1434 mtx_unlock_spin(&sleepq_prof_lock);
1436 LIST_FOREACH(sp, &sleepq_hash[i], sp_link) {
1438 sp->sp_wmesg, sp->sp_count);
1441 mtx_lock_spin(&sleepq_prof_lock);
1442 prof_enabled = enabled;
1443 mtx_unlock_spin(&sleepq_prof_lock);
1451 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NEEDGIANT, NULL, 0,
1452 dump_sleepq_prof_stats,
"A",
1453 "Sleepqueue profiling statistics");
1455 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
1456 reset_sleepq_prof_stats,
"I",
1457 "Reset sleepqueue profiling statistics");
1459 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0,
1460 enable_sleepq_prof,
"I",
1461 "Enable sleepqueue profiling");
1465DB_SHOW_COMMAND(sleepq, db_show_sleepqueue)
1470 struct lock_object *lock;
1483 wchan = (
void *)
addr;
1485 LIST_FOREACH(sq, &sc->sc_queues, sq_hash)
1486 if (sq->sq_wchan == wchan)
1499 db_printf(
"Unable to locate a sleep queue via %p\n", (
void *)
addr);
1502 db_printf(
"Wait channel: %p\n", sq->sq_wchan);
1503 db_printf(
"Queue type: %d\n", sq->sq_type);
1507 db_printf(
"Associated Interlock: %p - (%s) %s\n", lock,
1508 LOCK_CLASS(lock)->lc_name, lock->lo_name);
1511 db_printf(
"Blocked threads:\n");
1513 db_printf(
"\nQueue[%d]:\n", i);
1515 db_printf(
"\tempty\n");
1519 db_printf(
"\t%p (tid %d, pid %d, \"%s\")\n", td,
1520 td->td_tid, td->td_proc->p_pid,
1528DB_SHOW_ALIAS(
sleepqueue, db_show_sleepqueue);
device_property_type_t type
SYSCTL_NODE(_kern, OID_AUTO, binmisc, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "Image activator for miscellaneous binaries")
SYSCTL_PROC(_kern_binmisc, OID_AUTO, add, CTLFLAG_MPSAFE|CTLTYPE_STRUCT|CTLFLAG_WR, NULL, IBC_ADD, sysctl_kern_binmisc, "S,ximgact_binmisc_entry", "Add an activator entry")
SYSINIT(imgact_binmisc, SI_SUB_EXEC, SI_ORDER_MIDDLE, imgact_binmisc_init, NULL)
static struct bt_table st
SYSCTL_UINT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RWTUN, &idletick, 0, "Run periodic events when idle")
MTX_SYSINIT(et_eventtimers_init, &et_eventtimers_mtx, "et_mtx", MTX_DEF)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
void thread_lock_block_wait(struct thread *td)
void thread_lock_set(struct thread *td, struct mtx *new)
static struct pollrec pr[POLL_LIST_LEN]
void panic(const char *fmt,...)
int sig_ast_needsigchk(struct thread *td)
int sig_ast_checksusp(struct thread *td)
void mi_switch(int flags)
int setrunnable(struct thread *td, int srqflags)
void wakeup(const void *ident)
int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
struct sbuf * sbuf_new_for_sysctl(struct sbuf *s, char *buf, int length, struct sysctl_req *req)
volatile int rtc_generation
int callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t prec, callout_func_t *ftn, void *arg, int cpu, int flags)
void callout_when(sbintime_t sbt, sbintime_t precision, int flags, sbintime_t *res, sbintime_t *prec_res)
void sched_prio(struct thread *td, u_char prio)
void sched_sleep(struct thread *td, int pri)
u_int sq_blockedcnt[NR_SLEEPQS]
struct threadqueue sq_blocked[NR_SLEEPQS]
int snprintf(char *str, size_t size, const char *format,...)
int sbuf_finish(struct sbuf *s)
void sbuf_delete(struct sbuf *s)
int sbuf_printf(struct sbuf *s, const char *fmt,...)
char * sbuf_data(struct sbuf *s)
int sbuf_error(const struct sbuf *s)
struct sbuf * sbuf_new(struct sbuf *s, char *buf, int length, int flags)
void sleepq_release(const void *wchan)
static int sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri, int srqflags)
void init_sleepqueues(void)
struct sleepqueue * sleepq_lookup(const void *wchan)
int sleepq_wait_sig(const void *wchan, int pri)
static int sleepq_check_signals(void)
int sleepq_broadcast(const void *wchan, int flags, int pri, int queue)
static int sleepq_catch_signals(const void *wchan, int pri)
void sleepq_remove_nested(struct thread *td)
int sleepq_remove_matching(struct sleepqueue *sq, int queue, bool(*matches)(struct thread *), int pri)
static bool match_any(struct thread *td __unused)
int sleepq_timedwait_sig(const void *wchan, int pri)
void sleepq_remove(struct thread *td, const void *wchan)
struct sleepqueue_chain __aligned(CACHE_LINE_SIZE)
static struct sleepqueue_chain sleepq_chains[SC_TABLESIZE]
void sleepq_chains_remove_matching(bool(*matches)(struct thread *))
struct sleepqueue * sleepq_alloc(void)
int sleepq_type(const void *wchan)
int sleepq_signal(const void *wchan, int flags, int pri, int queue)
int sleepq_timedwait(const void *wchan, int pri)
CTASSERT(powerof2(SC_TABLESIZE))
u_int sleepq_sleepcnt(const void *wchan, int queue)
static int sleepq_init(void *mem, int size, int flags)
void sleepq_add(const void *wchan, struct lock_object *lock, const char *wmesg, int flags, int queue)
static uma_zone_t sleepq_zone
static void sleepq_timeout(void *arg)
void sleepq_set_timeout_sbt(const void *wchan, sbintime_t sbt, sbintime_t pr, int flags)
void sleepq_wait(const void *wchan, int pri)
static void sleepq_switch(const void *wchan, int pri)
void sleepq_lock(const void *wchan)
static int sleepq_check_timeout(void)
static void sleepq_remove_thread(struct sleepqueue *sq, struct thread *td)
int sleepq_abort(struct thread *td, int intrval)
void sleepq_free(struct sleepqueue *sq)
static int sleepq_check_ast_sc_locked(struct thread *td, struct sleepqueue_chain *sc)
SDT_PROBE_DECLARE(sched,,, sleep)
void stack_destroy(struct stack *st)
struct stack * stack_create(int flags)
void stack_sbuf_print(struct sbuf *sb, const struct stack *st)