42#include "opt_ktrace.h"
47#include <sys/blockcount.h>
48#include <sys/condvar.h>
50#include <sys/kernel.h>
55#include <sys/resourcevar.h>
58#include <sys/signalvar.h>
59#include <sys/sleepqueue.h>
62#include <sys/sysctl.h>
63#include <sys/sysproto.h>
64#include <sys/vmmeter.h>
67#include <sys/ktrace.h>
73#include <machine/cpu.h>
85 { {0, 0, 0}, FSCALE };
91 0.9200444146293232 * FSCALE,
92 0.9834714538216174 * FSCALE,
93 0.9944598480048967 * FSCALE,
97SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE,
98 "Fixed-point scale factor used for calculating load average values");
100static void loadav(
void *arg);
136 const char *wmesg, sbintime_t sbt, sbintime_t
pr,
int flags)
139 struct lock_class *
class;
140 uintptr_t lock_state;
141 int catch, pri, rval, sleepq_flags;
142 WITNESS_SAVE_DECL(lock_witness);
147 if (KTRPOINT(td, KTR_CSW))
150 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, lock,
151 "Sleeping on \"%s\"", wmesg);
152 KASSERT(sbt != 0 || mtx_owned(&
Giant) || lock != NULL ||
154 (
"sleeping without a lock"));
155 KASSERT(ident != NULL, (
"_sleep: NULL ident"));
156 KASSERT(TD_IS_RUNNING(td), (
"_sleep: curthread not running"));
158 KASSERT(lock != NULL && lock != &
Giant.lock_object,
159 (
"PDROP requires a non-Giant lock"));
161 class = LOCK_CLASS(lock);
165 if (SCHEDULER_STOPPED_TD(td)) {
166 if (lock != NULL &&
priority & PDROP)
167 class->lc_unlock(lock);
173 KASSERT(!TD_ON_SLEEPQ(td), (
"recursive sleep"));
175 if ((uintptr_t)ident >= (uintptr_t)&
pause_wchan[0] &&
176 (uintptr_t)ident <= (uintptr_t)&
pause_wchan[MAXCPU - 1])
177 sleepq_flags = SLEEPQ_PAUSE;
179 sleepq_flags = SLEEPQ_SLEEP;
181 sleepq_flags |= SLEEPQ_INTERRUPTIBLE;
184 CTR5(KTR_PROC,
"sleep: thread %ld (pid %ld, %s) on %s (%p)",
185 td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
187 if (lock == &
Giant.lock_object)
188 mtx_assert(&
Giant, MA_OWNED);
190 if (lock != NULL && lock != &
Giant.lock_object &&
191 !(class->lc_flags & LC_SLEEPABLE)) {
192 KASSERT(!(class->lc_flags & LC_SPINLOCK),
193 (
"spin locks can only use msleep_spin"));
194 WITNESS_SAVE(lock, lock_witness);
195 lock_state =
class->lc_unlock(lock);
209 sleepq_add(ident, lock, wmesg, sleepq_flags, 0);
212 if (lock != NULL && class->lc_flags & LC_SLEEPABLE) {
214 WITNESS_SAVE(lock, lock_witness);
215 lock_state =
class->lc_unlock(lock);
218 if (sbt != 0 &&
catch)
229 if (KTRPOINT(td, KTR_CSW))
233 if (lock != NULL && lock != &
Giant.lock_object && !(
priority & PDROP)) {
234 class->lc_lock(lock, lock_state);
235 WITNESS_RESTORE(lock, lock_witness);
243 sbintime_t sbt, sbintime_t
pr,
int flags)
247 WITNESS_SAVE_DECL(
mtx);
250 KASSERT(
mtx != NULL, (
"sleeping without a mutex"));
251 KASSERT(ident != NULL, (
"msleep_spin_sbt: NULL ident"));
252 KASSERT(TD_IS_RUNNING(td), (
"msleep_spin_sbt: curthread not running"));
254 if (SCHEDULER_STOPPED_TD(td))
258 CTR5(KTR_PROC,
"msleep_spin: thread %ld (pid %ld, %s) on %s (%p)",
259 td->td_tid, td->td_proc->p_pid, td->td_name, wmesg, ident);
262 mtx_assert(
mtx, MA_OWNED | MA_NOTRECURSED);
263 WITNESS_SAVE(&
mtx->lock_object,
mtx);
264 mtx_unlock_spin(
mtx);
269 sleepq_add(ident, &
mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0);
281 if (KTRPOINT(td, KTR_CSW)) {
289 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
"Sleeping on \"%s\"",
300 if (KTRPOINT(td, KTR_CSW))
305 WITNESS_RESTORE(&
mtx->lock_object,
mtx);
319 KASSERT(sbt >= 0, (
"pause_sbt: timeout must be >= 0"));
325 if ((cold && curthread == &thread0) ||
kdb_active ||
326 SCHEDULER_STOPPED()) {
331 while (sbt >= SBT_1S) {
336 sbt = howmany(sbt, SBT_1US);
339 return (EWOULDBLOCK);
342 (
flags & C_CATCH) ? PCATCH : 0, wmesg, sbt,
pr,
flags));
356 if (wakeup_swapper) {
357 KASSERT(ident != &
proc0,
358 (
"wakeup and wakeup_swapper and proc0"));
374 wakeup_swapper =
sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_DROP, 0, 0);
385 wakeup_swapper =
sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR |
398 KASSERT(_BLOCKCOUNT_WAITERS(old),
399 (
"%s: no waiters on %p", __func__, bc));
401 if (atomic_cmpset_int(&bc->__count, _BLOCKCOUNT_WAITERS_FLAG, 0))
418 uintptr_t lock_state;
423 KASSERT(lock != &
Giant.lock_object,
424 (
"%s: cannot use Giant as the interlock", __func__));
426 catch = (prio & PCATCH) != 0;
427 drop = (prio & PDROP) != 0;
438 if (atomic_load_acq_int(&bc->__count) == 0) {
439 if (lock != NULL && drop)
440 LOCK_CLASS(lock)->lc_unlock(lock);
448 lock_state = LOCK_CLASS(lock)->lc_unlock(lock);
449 old = blockcount_read(bc);
452 if (_BLOCKCOUNT_COUNT(old) == 0) {
456 if (_BLOCKCOUNT_WAITERS(old))
458 }
while (!atomic_fcmpset_int(&bc->__count, &old,
459 old | _BLOCKCOUNT_WAITERS_FLAG));
460 sleepq_add(wchan, NULL, wmesg,
catch ? SLEEPQ_INTERRUPTIBLE : 0, 0);
470 if (lock != NULL && !drop)
471 LOCK_CLASS(lock)->lc_lock(lock, lock_state);
479 thread_unlock(curthread);
482 panic(
"%s: did not reenter debugger", __func__);
493 uint64_t runtime, new_switchtime;
497 THREAD_LOCK_ASSERT(td, MA_OWNED | MA_NOTRECURSED);
498 KASSERT(!TD_ON_RUNQ(td), (
"mi_switch: called by old code"));
500 if (!TD_ON_LOCK(td) && !TD_IS_RUNNING(td))
501 mtx_assert(&
Giant, MA_NOTOWNED);
503 KASSERT(td->td_critnest == 1 || KERNEL_PANICKED(),
504 (
"mi_switch: switch in a critical section"));
505 KASSERT((
flags & (SW_INVOL | SW_VOL)) != 0,
506 (
"mi_switch: switch must be voluntary or involuntary"));
513 if (SCHEDULER_STOPPED_TD(td))
515 if (
flags & SW_VOL) {
516 td->td_ru.ru_nvcsw++;
517 td->td_swvoltick =
ticks;
519 td->td_ru.ru_nivcsw++;
520 td->td_swinvoltick =
ticks;
523 SCHED_STAT_INC(sched_switch_stats[
flags & SW_TYPE_MASK]);
530 runtime = new_switchtime - PCPU_GET(switchtime);
531 td->td_runtime += runtime;
532 td->td_incruntime += runtime;
533 PCPU_SET(switchtime, new_switchtime);
536 PCPU_SET(switchticks,
ticks);
537 CTR4(KTR_PROC,
"mi_switch: old thread %ld (td_sched %p, pid %ld, %s)",
538 td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
540 if (SDT_PROBES_ENABLED() &&
541 ((
flags & SW_PREEMPT) != 0 || ((
flags & SW_INVOL) != 0 &&
542 (
flags & SW_TYPE_MASK) == SWT_NEEDRESCHED)))
543 SDT_PROBE0(sched, , , preempt);
546 CTR4(KTR_PROC,
"mi_switch: new thread %ld (td_sched %p, pid %ld, %s)",
547 td->td_tid, td_get_sched(td), td->td_proc->p_pid, td->td_name);
552 if ((td = PCPU_GET(deadthread))) {
553 PCPU_SET(deadthread, NULL);
571 THREAD_LOCK_ASSERT(td, MA_OWNED);
572 KASSERT(td->td_proc->p_state != PRS_ZOMBIE,
573 (
"setrunnable: pid %d is a zombie", td->td_proc->p_pid));
576 switch (TD_GET_STATE(td)) {
581 KASSERT((td->td_flags & TDF_INMEM) != 0,
582 (
"setrunnable: td %p not in mem, flags 0x%X inhibit 0x%X",
583 td, td->td_flags, td->td_inhibitors));
592 if (td->td_inhibitors == TDI_SWAPPED &&
593 (td->td_flags & TDF_SWAPINREQ) == 0) {
594 td->td_flags |= TDF_SWAPINREQ;
599 panic(
"setrunnable: state 0x%x", TD_GET_STATE(td));
601 if ((srqflags & (SRQ_HOLD | SRQ_HOLDTD)) == 0)
620 for (i = 0; i < 3; i++)
621 avg->ldavg[i] = (
cexp[i] * avg->ldavg[i] +
622 nrun * FSCALE * (FSCALE -
cexp[i])) >> FSHIFT;
630 SBT_1US * (4000000 + (
int)(random() % 2000001)), SBT_1US,
631 loadav, NULL, C_DIRECT_EXEC | C_PREL(32));
648 return ((u_int)
ticks - (u_int)curthread->td_swvoltick >=
hogticks);
667 if (prio == PRI_USER)
668 prio = td->td_user_pri;
683 if (PRI_BASE(td->td_pri_class) == PRI_TIMESHARE)
686 td->td_retval[0] = 0;
693 td->td_retval[0] = td->td_oncpu;
struct mtx __exclusive_cache_line Giant
static struct pollrec pr[POLL_LIST_LEN]
void panic(const char *fmt,...)
int sys_yield(struct thread *td, struct yield_args *uap)
void kern_yield(int prio)
void _blockcount_wakeup(blockcount_t *bc, u_int old)
static void kdb_switch(void)
void mi_switch(int flags)
static void loadav(void *arg)
int sys_sched_getcpu(struct thread *td, struct sched_getcpu_args *uap)
static const char pause_wchan[MAXCPU]
int pause_sbt(const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
void wakeup_any(const void *ident)
int _sleep(const void *ident, struct lock_object *lock, int priority, const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
SYSCTL_INT(_kern, OID_AUTO, fscale, CTLFLAG_RD, SYSCTL_NULL_INT_PTR, FSCALE, "Fixed-point scale factor used for calculating load average values")
int msleep_spin_sbt(const void *ident, struct mtx *mtx, const char *wmesg, sbintime_t sbt, sbintime_t pr, int flags)
static void synch_setup(void *dummy)
SYSINIT(synch_setup, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, synch_setup, NULL)
SDT_PROVIDER_DECLARE(sched)
SDT_PROBE_DEFINE(sched,,, preempt)
int setrunnable(struct thread *td, int srqflags)
static struct callout loadav_callout
struct loadavg averunnable
void wakeup(const void *ident)
void wakeup_one(const void *ident)
int _blockcount_sleep(blockcount_t *bc, struct lock_object *lock, const char *wmesg, int prio)
static void sleepinit(void *unused)
void thread_stash(struct thread *td)
void callout_init(struct callout *c, int mpsafe)
void sched_prio(struct thread *td, u_char prio)
void sched_switch(struct thread *td, int flags)
void sched_wakeup(struct thread *td, int srqflags)
u_char __read_frequently kdb_active
void sleepq_release(const void *wchan)
void init_sleepqueues(void)
int sleepq_wait_sig(const void *wchan, int pri)
int sleepq_broadcast(const void *wchan, int flags, int pri, int queue)
int sleepq_timedwait_sig(const void *wchan, int pri)
int sleepq_signal(const void *wchan, int flags, int pri, int queue)
int sleepq_timedwait(const void *wchan, int pri)
void sleepq_add(const void *wchan, struct lock_object *lock, const char *wmesg, int flags, int queue)
void sleepq_set_timeout_sbt(const void *wchan, sbintime_t sbt, sbintime_t pr, int flags)
void sleepq_wait(const void *wchan, int pri)
void sleepq_lock(const void *wchan)