#include <sys/cdefs.h>
#include "opt_sleepqueue_profiling.h"
#include "opt_ddb.h"
#include "opt_sched.h"
#include "opt_stack.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sbuf.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepqueue.h>
#include <sys/stack.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <machine/atomic.h>
#include <vm/uma.h>

Include dependency graph for subr_sleepqueue.c:

Data Structures
struct	sleepqueue

struct	sleepqueue_chain

Macros
#define	SC_TABLESIZE 256

#define	SC_MASK (SC_TABLESIZE - 1)

#define	SC_SHIFT 8

#define	SC_HASH(wc)

#define	SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]

#define	NR_SLEEPQS 2

Functions
	__FBSDID ("$FreeBSD$")

	CTASSERT (powerof2(SC_TABLESIZE))

struct sleepqueue_chain	__aligned (CACHE_LINE_SIZE)

static int	sleepq_catch_signals (const void *wchan, int pri)

static int	sleepq_check_signals (void)

static int	sleepq_check_timeout (void)

static int	sleepq_init (void *mem, int size, int flags)

static int	sleepq_resume_thread (struct sleepqueue sq, struct thread td, int pri, int srqflags)

static void	sleepq_remove_thread (struct sleepqueue sq, struct thread td)

static void	sleepq_switch (const void *wchan, int pri)

static void	sleepq_timeout (void *arg)

	SDT_PROBE_DECLARE (sched,,, sleep)

	SDT_PROBE_DECLARE (sched,,, wakeup)

void	init_sleepqueues (void)

struct sleepqueue *	sleepq_alloc (void)

void	sleepq_free (struct sleepqueue *sq)

void	sleepq_lock (const void *wchan)

struct sleepqueue *	sleepq_lookup (const void *wchan)

void	sleepq_release (const void *wchan)

void	sleepq_add (const void wchan, struct lock_object lock, const char *wmesg, int flags, int queue)

void	sleepq_set_timeout_sbt (const void *wchan, sbintime_t sbt, sbintime_t pr, int flags)

u_int	sleepq_sleepcnt (const void *wchan, int queue)

static int	sleepq_check_ast_sc_locked (struct thread td, struct sleepqueue_chain sc)

void	sleepq_wait (const void *wchan, int pri)

int	sleepq_wait_sig (const void *wchan, int pri)

int	sleepq_timedwait (const void *wchan, int pri)

int	sleepq_timedwait_sig (const void *wchan, int pri)

int	sleepq_type (const void *wchan)

void	sleepq_remove_nested (struct thread *td)

int	sleepq_signal (const void *wchan, int flags, int pri, int queue)

static bool	match_any (struct thread *td __unused)

int	sleepq_broadcast (const void *wchan, int flags, int pri, int queue)

int	sleepq_remove_matching (struct sleepqueue sq, int queue, bool(matches)(struct thread *), int pri)

void	sleepq_remove (struct thread td, const void wchan)

int	sleepq_abort (struct thread *td, int intrval)

void	sleepq_chains_remove_matching (bool(matches)(struct thread ))

Variables
static struct sleepqueue_chain	sleepq_chains [SC_TABLESIZE]

static uma_zone_t	sleepq_zone

Macro Definition Documentation

◆ NR_SLEEPQS

#define NR_SLEEPQS 2

Definition at line 108 of file subr_sleepqueue.c.

◆ SC_HASH

#define SC_HASH ( wc )

Value:

((((uintptr_t)(wc) >> SC_SHIFT) ^ (uintptr_t)(wc)) & \

SC_MASK)

SC_SHIFT

#define SC_SHIFT

Definition: subr_sleepqueue.c:104

Definition at line 105 of file subr_sleepqueue.c.

◆ SC_LOOKUP

#define SC_LOOKUP ( wc ) &sleepq_chains[SC_HASH(wc)]

Definition at line 107 of file subr_sleepqueue.c.

◆ SC_MASK

#define SC_MASK (SC_TABLESIZE - 1)

Definition at line 103 of file subr_sleepqueue.c.

◆ SC_SHIFT

#define SC_SHIFT 8

Definition at line 104 of file subr_sleepqueue.c.

◆ SC_TABLESIZE

#define SC_TABLESIZE 256

Definition at line 100 of file subr_sleepqueue.c.

Function Documentation

◆ __aligned()

struct sleepqueue_chain __aligned ( CACHE_LINE_SIZE )

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ CTASSERT()

CTASSERT ( powerof2(SC_TABLESIZE) )

◆ init_sleepqueues()

void init_sleepqueues ( void )

Definition at line 219 of file subr_sleepqueue.c.

References SC_TABLESIZE, sleepq_alloc(), sleepq_chains, sleepq_init(), and sleepq_zone.

Referenced by sleepinit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ match_any()

static bool match_any ( struct thread *td __unused )

static

Definition at line 982 of file subr_sleepqueue.c.

Referenced by sleepq_broadcast().

Here is the caller graph for this function:

◆ SDT_PROBE_DECLARE() [1/2]

SDT_PROBE_DECLARE	(	sched	,
		sleep
	)

◆ SDT_PROBE_DECLARE() [2/2]

SDT_PROBE_DECLARE	(	sched	,
		wakeup
	)

◆ sleepq_abort()

int sleepq_abort	(	struct thread *	td,
		int	intrval
	)

Definition at line 1130 of file subr_sleepqueue.c.

References sleepq_lookup(), and sleepq_resume_thread().

Referenced by sig_sleepq_abort(), sig_suspend_threads(), and weed_inhib().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_add()

void sleepq_add	(	const void *	wchan,
		struct lock_object *	lock,
		const char *	wmesg,
		int	flags,
		int	queue
	)

Definition at line 309 of file subr_sleepqueue.c.

References flags, NR_SLEEPQS, SC_LOOKUP, sleepq_lookup(), sleepqueue::sq_blocked, and sleepqueue::sq_blockedcnt.

Referenced by __lockmgr_args(), _blockcount_sleep(), _callout_stop_safe(), _cv_timedwait_sbt(), _cv_timedwait_sig_sbt(), _cv_wait(), _cv_wait_sig(), _cv_wait_unlock(), _sleep(), _sx_slock_hard(), _sx_xlock_hard(), foffset_lock(), msleep_spin_sbt(), and sleeplk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_alloc()

struct sleepqueue * sleepq_alloc ( void )

Definition at line 242 of file subr_sleepqueue.c.

References sleepq_zone.

Referenced by init_sleepqueues(), and thread_init().

Here is the caller graph for this function:

◆ sleepq_broadcast()

int sleepq_broadcast	(	const void *	wchan,
		int	flags,
		int	pri,
		int	queue
	)

Definition at line 992 of file subr_sleepqueue.c.

References flags, match_any(), NR_SLEEPQS, sleepq_lookup(), and sleepq_remove_matching().

Referenced by __lockmgr_args(), _sx_sunlock_hard(), _sx_xunlock_hard(), cv_broadcastpri(), foffset_unlock(), lockmgr_xunlock_hard(), sx_downgrade_int(), wakeup(), and wakeupshlk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_catch_signals()

static int sleepq_catch_signals	(	const void *	wchan,
		int	pri
	)

static

Definition at line 503 of file subr_sleepqueue.c.

References SC_LOOKUP, sleepq_check_ast_sc_locked(), sleepq_lookup(), sleepq_remove_thread(), and sleepq_switch().

Referenced by sleepq_timedwait_sig(), and sleepq_wait_sig().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_chains_remove_matching()

void sleepq_chains_remove_matching ( bool(*)(struct thread *) matches )

Definition at line 1173 of file subr_sleepqueue.c.

References NR_SLEEPQS, SC_TABLESIZE, sleepq_chains, and sleepq_remove_matching().

Referenced by tc_setclock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_check_ast_sc_locked()

static int sleepq_check_ast_sc_locked	(	struct thread *	td,
		struct sleepqueue_chain *	sc
	)

static

Definition at line 437 of file subr_sleepqueue.c.

References sig_ast_checksusp(), and sig_ast_needsigchk().

Referenced by sleepq_catch_signals().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_check_signals()

static int sleepq_check_signals ( void )

inlinestatic

Definition at line 637 of file subr_sleepqueue.c.

Referenced by sleepq_timedwait_sig(), and sleepq_wait_sig().

Here is the caller graph for this function:

◆ sleepq_check_timeout()

static int sleepq_check_timeout ( void )

inlinestatic

Definition at line 618 of file subr_sleepqueue.c.

References res.

Referenced by sleepq_timedwait(), and sleepq_timedwait_sig().

Here is the caller graph for this function:

◆ sleepq_free()

void sleepq_free ( struct sleepqueue * sq )

Definition at line 252 of file subr_sleepqueue.c.

References sleepq_zone.

Referenced by thread_fini().

Here is the caller graph for this function:

◆ sleepq_init()

static int sleepq_init	(	void *	mem,
		int	size,
		int	flags
	)

static

Definition at line 908 of file subr_sleepqueue.c.

References NR_SLEEPQS, sleepqueue::sq_blocked, and sleepqueue::sq_blockedcnt.

Referenced by init_sleepqueues().

Here is the caller graph for this function:

◆ sleepq_lock()

void sleepq_lock ( const void * wchan )

Definition at line 262 of file subr_sleepqueue.c.

References SC_LOOKUP.

Referenced by __lockmgr_args(), _blockcount_sleep(), _callout_stop_safe(), _cv_timedwait_sbt(), _cv_timedwait_sig_sbt(), _cv_wait(), _cv_wait_sig(), _cv_wait_unlock(), _sleep(), _sx_slock_hard(), _sx_sunlock_hard(), _sx_xlock_hard(), _sx_xunlock_hard(), cv_broadcastpri(), cv_destroy(), cv_signal(), foffset_lock(), foffset_unlock(), lockmgr_slock_hard(), lockmgr_xlock_hard(), lockmgr_xunlock_hard(), msleep_spin_sbt(), sx_downgrade_int(), wakeup(), wakeup_any(), wakeup_one(), and wakeupshlk().

Here is the caller graph for this function:

◆ sleepq_lookup()

struct sleepqueue * sleepq_lookup ( const void * wchan )

Definition at line 276 of file subr_sleepqueue.c.

References SC_LOOKUP.

Referenced by cv_destroy(), cv_signal(), sleepq_abort(), sleepq_add(), sleepq_broadcast(), sleepq_catch_signals(), sleepq_remove(), sleepq_remove_nested(), sleepq_signal(), sleepq_sleepcnt(), sleepq_switch(), sleepq_timeout(), and sleepq_type().

Here is the caller graph for this function:

◆ sleepq_release()

void sleepq_release ( const void * wchan )

Definition at line 294 of file subr_sleepqueue.c.

References SC_LOOKUP.

Referenced by __lockmgr_args(), _blockcount_sleep(), _callout_stop_safe(), _cv_timedwait_sbt(), _cv_timedwait_sig_sbt(), _cv_wait(), _cv_wait_sig(), _cv_wait_unlock(), _sleep(), _sx_slock_hard(), _sx_sunlock_hard(), _sx_xlock_hard(), _sx_xunlock_hard(), cv_broadcastpri(), cv_destroy(), cv_signal(), foffset_lock(), foffset_unlock(), lockmgr_slock_hard(), lockmgr_xlock_hard(), lockmgr_xunlock_hard(), msleep_spin_sbt(), sleepq_signal(), sx_downgrade_int(), wakeup(), and wakeupshlk().

Here is the caller graph for this function:

◆ sleepq_remove()

void sleepq_remove	(	struct thread *	td,
		const void *	wchan
	)

Definition at line 1089 of file subr_sleepqueue.c.

References SC_LOOKUP, sleepq_lookup(), and sleepq_resume_thread().

Here is the call graph for this function:

◆ sleepq_remove_matching()

int sleepq_remove_matching	(	struct sleepqueue *	sq,
		int	queue,
		bool()(struct thread )	matches,
		int	pri
	)

Definition at line 1012 of file subr_sleepqueue.c.

References sleepq_resume_thread(), and sleepqueue::sq_blocked.

Referenced by sleepq_broadcast(), and sleepq_chains_remove_matching().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_remove_nested()

void sleepq_remove_nested ( struct thread * td )

Definition at line 867 of file subr_sleepqueue.c.

References SC_LOOKUP, sleepq_lookup(), and sleepq_remove_thread().

Referenced by thread_run_flash().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_remove_thread()

static void sleepq_remove_thread	(	struct sleepqueue *	sq,
		struct thread *	td
	)

static

Definition at line 801 of file subr_sleepqueue.c.

References NR_SLEEPQS, SC_LOOKUP, sleepqueue::sq_blocked, sleepqueue::sq_blockedcnt, and wakeup().

Referenced by sleepq_catch_signals(), sleepq_remove_nested(), sleepq_resume_thread(), and sleepq_switch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_resume_thread()

static int sleepq_resume_thread	(	struct sleepqueue *	sq,
		struct thread *	td,
		int	pri,
		int	srqflags
	)

static

Definition at line 742 of file subr_sleepqueue.c.

References SC_LOOKUP, sched_prio(), setrunnable(), sleepq_remove_thread(), and thread_lock_block_wait().

Referenced by sleepq_abort(), sleepq_remove(), sleepq_remove_matching(), sleepq_signal(), and sleepq_timeout().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_set_timeout_sbt()

void sleepq_set_timeout_sbt	(	const void *	wchan,
		sbintime_t	sbt,
		sbintime_t	pr,
		int	flags
	)

Definition at line 395 of file subr_sleepqueue.c.

References callout_reset_sbt_on(), callout_when(), flags, panic(), pr, SC_LOOKUP, and sleepq_timeout().

Referenced by _cv_timedwait_sbt(), _cv_timedwait_sig_sbt(), _sleep(), and msleep_spin_sbt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_signal()

int sleepq_signal	(	const void *	wchan,
		int	flags,
		int	pri,
		int	queue
	)

Definition at line 927 of file subr_sleepqueue.c.

References flags, NR_SLEEPQS, SC_LOOKUP, sleepq_lookup(), sleepq_release(), sleepq_resume_thread(), and sleepqueue::sq_blocked.

Referenced by cv_signal(), wakeup_any(), and wakeup_one().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_sleepcnt()

u_int sleepq_sleepcnt	(	const void *	wchan,
		int	queue
	)

Definition at line 424 of file subr_sleepqueue.c.

References NR_SLEEPQS, sleepq_lookup(), and sleepqueue::sq_blockedcnt.

Referenced by __lockmgr_args(), _sx_xunlock_hard(), lockmgr_xunlock_hard(), and wakeupshlk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_switch()

static void sleepq_switch	(	const void *	wchan,
		int	pri
	)

static

Definition at line 546 of file subr_sleepqueue.c.

References mi_switch(), rtc_generation, SC_LOOKUP, sched_sleep(), sleepq_lookup(), sleepq_remove_thread(), and thread_lock_set().

Referenced by sleepq_catch_signals(), sleepq_timedwait(), and sleepq_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_timedwait()

int sleepq_timedwait	(	const void *	wchan,
		int	pri
	)

Definition at line 682 of file subr_sleepqueue.c.

References sleepq_check_timeout(), and sleepq_switch().

Referenced by _cv_timedwait_sbt(), _sleep(), msleep_spin_sbt(), and sleeplk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_timedwait_sig()

int sleepq_timedwait_sig	(	const void *	wchan,
		int	pri
	)

Definition at line 700 of file subr_sleepqueue.c.

References sleepq_catch_signals(), sleepq_check_signals(), and sleepq_check_timeout().

Referenced by _cv_timedwait_sig_sbt(), _sleep(), and sleeplk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_timeout()

static void sleepq_timeout ( void * arg )

static

Definition at line 1039 of file subr_sleepqueue.c.

References SC_LOOKUP, sleepq_lookup(), and sleepq_resume_thread().

Referenced by sleepq_set_timeout_sbt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_type()

int sleepq_type ( const void * wchan )

Definition at line 719 of file subr_sleepqueue.c.

References sleepq_lookup(), and type.

Here is the call graph for this function:

◆ sleepq_wait()

void sleepq_wait	(	const void *	wchan,
		int	pri
	)

Definition at line 652 of file subr_sleepqueue.c.

References sleepq_switch().

Referenced by __lockmgr_args(), _blockcount_sleep(), _callout_stop_safe(), _cv_wait(), _cv_wait_unlock(), _sleep(), _sx_slock_hard(), _sx_xlock_hard(), foffset_lock(), msleep_spin_sbt(), and sleeplk().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sleepq_wait_sig()

int sleepq_wait_sig	(	const void *	wchan,
		int	pri
	)

Definition at line 667 of file subr_sleepqueue.c.

References sleepq_catch_signals(), and sleepq_check_signals().

Referenced by _blockcount_sleep(), _cv_wait_sig(), _sleep(), _sx_slock_hard(), _sx_xlock_hard(), and sleeplk().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ sleepq_chains

struct sleepqueue_chain sleepq_chains[SC_TABLESIZE]

static

Definition at line 161 of file subr_sleepqueue.c.

Referenced by init_sleepqueues(), and sleepq_chains_remove_matching().

◆ sleepq_zone

uma_zone_t sleepq_zone

static

Definition at line 162 of file subr_sleepqueue.c.

Referenced by init_sleepqueues(), sleepq_alloc(), and sleepq_free().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ NR_SLEEPQS

◆ SC_HASH

◆ SC_LOOKUP

◆ SC_MASK

◆ SC_SHIFT

◆ SC_TABLESIZE

Function Documentation

◆ __aligned()

◆ __FBSDID()

◆ CTASSERT()

◆ init_sleepqueues()

◆ match_any()

◆ SDT_PROBE_DECLARE() [1/2]

◆ SDT_PROBE_DECLARE() [2/2]

◆ sleepq_abort()

◆ sleepq_add()

◆ sleepq_alloc()

◆ sleepq_broadcast()

◆ sleepq_catch_signals()

◆ sleepq_chains_remove_matching()

◆ sleepq_check_ast_sc_locked()

◆ sleepq_check_signals()

◆ sleepq_check_timeout()

◆ sleepq_free()

◆ sleepq_init()

◆ sleepq_lock()

◆ sleepq_lookup()

◆ sleepq_release()

◆ sleepq_remove()

◆ sleepq_remove_matching()

◆ sleepq_remove_nested()

◆ sleepq_remove_thread()

◆ sleepq_resume_thread()

◆ sleepq_set_timeout_sbt()

◆ sleepq_signal()

◆ sleepq_sleepcnt()

◆ sleepq_switch()

◆ sleepq_timedwait()

◆ sleepq_timedwait_sig()

◆ sleepq_timeout()

◆ sleepq_type()

◆ sleepq_wait()

◆ sleepq_wait_sig()

Variable Documentation

◆ sleepq_chains

◆ sleepq_zone