#include <sys/cdefs.h>
#include "opt_hwpmc_hooks.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resource.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/smp.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/turnstile.h>
#include <sys/umtxvar.h>
#include <sys/vmmeter.h>
#include <sys/cpuset.h>
#include <sys/sbuf.h>
#include <machine/cpu.h>
#include <machine/smp.h>

Include dependency graph for sched_ule.c:

Data Structures
struct	td_sched

struct	tdq

Macros
#define	KTR_ULE 0

#define	TS_NAME_LEN (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))

#define	TDQ_NAME_LEN (sizeof("sched lock ") + sizeof(__XSTRING(MAXCPU)))

#define	TDQ_LOADNAME_LEN (sizeof("CPU ") + sizeof(__XSTRING(MAXCPU)) - 1 + sizeof(" load"))

#define	TSF_BOUND 0x0001 /* Thread can not migrate. */

#define	TSF_XFERABLE 0x0002 /* Thread was added as transferable. */

#define	THREAD_CAN_MIGRATE(td) ((td)->td_pinned == 0)

#define	THREAD_CAN_SCHED(td, cpu) CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)

#define	PRI_TIMESHARE_RANGE (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1)

#define	PRI_INTERACT_RANGE ((PRI_TIMESHARE_RANGE - SCHED_PRI_NRESV) / 2)

#define	PRI_BATCH_RANGE (PRI_TIMESHARE_RANGE - PRI_INTERACT_RANGE)

#define	PRI_MIN_INTERACT PRI_MIN_TIMESHARE

#define	PRI_MAX_INTERACT (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE - 1)

#define	PRI_MIN_BATCH (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE)

#define	PRI_MAX_BATCH PRI_MAX_TIMESHARE

#define	SCHED_TICK_SECS 10

#define	SCHED_TICK_TARG (hz * SCHED_TICK_SECS)

#define	SCHED_TICK_MAX (SCHED_TICK_TARG + hz)

#define	SCHED_TICK_SHIFT 10

#define	SCHED_TICK_HZ(ts) ((ts)->ts_ticks >> SCHED_TICK_SHIFT)

#define	SCHED_TICK_TOTAL(ts) (max((ts)->ts_ltick - (ts)->ts_ftick, hz))

#define	SCHED_PRI_NRESV (PRIO_MAX - PRIO_MIN)

#define	SCHED_PRI_NHALF (SCHED_PRI_NRESV / 2)

#define	SCHED_PRI_MIN (PRI_MIN_BATCH + SCHED_PRI_NHALF)

#define	SCHED_PRI_MAX (PRI_MAX_BATCH - SCHED_PRI_NHALF)

#define	SCHED_PRI_RANGE (SCHED_PRI_MAX - SCHED_PRI_MIN + 1)

#define	SCHED_PRI_TICKS(ts)

#define	SCHED_PRI_NICE(nice) (nice)

#define	SCHED_SLP_RUN_MAX ((hz * 5) << SCHED_TICK_SHIFT)

#define	SCHED_SLP_RUN_FORK ((hz / 2) << SCHED_TICK_SHIFT)

#define	SCHED_INTERACT_MAX (100)

#define	SCHED_INTERACT_HALF (SCHED_INTERACT_MAX / 2)

#define	SCHED_INTERACT_THRESH (30)

#define	SCHED_SLICE_DEFAULT_DIVISOR 10 /* ~94 ms, 12 stathz ticks. */

#define	SCHED_SLICE_MIN_DIVISOR 6 /* DEFAULT/MIN = ~16 ms. */

#define	TDF_PICKCPU TDF_SCHED0 /* Thread should pick new CPU. */

#define	TDF_SLICEEND TDF_SCHED2 /* Thread time slice is over. */

#define	TDQ_RUNNING 1

#define	TDQ_IDLE 2

#define	TDQ_ID(x) (0)

#define	TDQ_SELF() (&tdq_cpu)

#define	TDQ_CPU(x) (&tdq_cpu)

#define	TDQ_LOCK_ASSERT(t, type) mtx_assert(TDQ_LOCKPTR((t)), (type))

#define	TDQ_LOCK(t) mtx_lock_spin(TDQ_LOCKPTR((t)))

#define	TDQ_LOCK_FLAGS(t, f) mtx_lock_spin_flags(TDQ_LOCKPTR((t)), (f))

#define	TDQ_TRYLOCK(t) mtx_trylock_spin(TDQ_LOCKPTR((t)))

#define	TDQ_TRYLOCK_FLAGS(t, f) mtx_trylock_spin_flags(TDQ_LOCKPTR((t)), (f))

#define	TDQ_UNLOCK(t) mtx_unlock_spin(TDQ_LOCKPTR((t)))

#define	TDQ_LOCKPTR(t) ((struct mtx *)(&(t)->tdq_lock))

#define	TDQ_IDLESPIN(tdq) 1

Functions
	__FBSDID ("$FreeBSD$")

	_Static_assert (sizeof(struct thread)+sizeof(struct td_sched)<=sizeof(struct thread0_storage), "increase struct thread0_storage.t0st_sched size")

struct tdq	__aligned (64)

static void	sched_priority (struct thread *)

static void	sched_thread_priority (struct thread *, u_char)

static int	sched_interact_score (struct thread *)

static void	sched_interact_update (struct thread *)

static void	sched_interact_fork (struct thread *)

static void	sched_pctcpu_update (struct td_sched *, int)

static struct thread *	tdq_choose (struct tdq *)

static void	tdq_setup (struct tdq *, int i)

static void	tdq_load_add (struct tdq , struct thread )

static void	tdq_load_rem (struct tdq , struct thread )

static __inline void	tdq_runq_add (struct tdq , struct thread , int)

static __inline void	tdq_runq_rem (struct tdq , struct thread )

static int	sched_shouldpreempt (int, int, int)

void	tdq_print (int cpu)

static void	runq_print (struct runq *rq)

static void	tdq_add (struct tdq , struct thread , int)

static void	sched_setup (void *dummy)

	SYSINIT (sched_setup, SI_SUB_RUN_QUEUE, SI_ORDER_FIRST, sched_setup, NULL)

static void	sched_initticks (void *dummy)

	SYSINIT (sched_initticks, SI_SUB_CLOCKS, SI_ORDER_THIRD, sched_initticks, NULL)

	SDT_PROVIDER_DEFINE (sched)

	SDT_PROBE_DEFINE3 (sched,,, change__pri, "struct thread ", "struct proc ", "uint8_t")

	SDT_PROBE_DEFINE3 (sched,,, dequeue, "struct thread ", "struct proc ", "void *")

	SDT_PROBE_DEFINE4 (sched,,, enqueue, "struct thread ", "struct proc ", "void *", "int")

	SDT_PROBE_DEFINE4 (sched,,, lend__pri, "struct thread ", "struct proc ", "uint8_t", "struct thread *")

	SDT_PROBE_DEFINE2 (sched,,, load__change, "int", "int")

	SDT_PROBE_DEFINE2 (sched,,, off__cpu, "struct thread ", "struct proc ")

	SDT_PROBE_DEFINE (sched,,, on__cpu)

	SDT_PROBE_DEFINE (sched,,, remain__cpu)

	SDT_PROBE_DEFINE2 (sched,,, surrender, "struct thread ", "struct proc ")

static int	tdq_slice (struct tdq *tdq)

static void	tdq_setlowpri (struct tdq tdq, struct thread ctd)

void	schedinit (void)

void	schedinit_ap (void)

int	sched_rr_interval (void)

void	sched_lend_prio (struct thread *td, u_char prio)

void	sched_unlend_prio (struct thread *td, u_char prio)

void	sched_prio (struct thread *td, u_char prio)

void	sched_user_prio (struct thread *td, u_char prio)

void	sched_lend_user_prio (struct thread *td, u_char prio)

void	sched_lend_user_prio_cond (struct thread *td, u_char prio)

static struct mtx *	sched_switch_migrate (struct tdq tdq, struct thread td, int flags)

static void	thread_unblock_switch (struct thread td, struct mtx mtx)

void	sched_switch (struct thread *td, int flags)

void	sched_nice (struct proc *p, int nice)

void	sched_sleep (struct thread *td, int prio)

void	sched_wakeup (struct thread *td, int srqflags)

void	sched_fork (struct thread td, struct thread child)

void	sched_fork_thread (struct thread td, struct thread child)

void	sched_class (struct thread *td, int class)

void	sched_exit (struct proc p, struct thread child)

void	sched_exit_thread (struct thread td, struct thread child)

void	sched_preempt (struct thread *td)

void	sched_userret_slowpath (struct thread *td)

void	sched_clock (struct thread *td, int cnt)

u_int	sched_estcpu (struct thread *td __unused)

int	sched_runnable (void)

struct thread *	sched_choose (void)

static void	sched_setpreempt (struct thread *td)

void	sched_add (struct thread *td, int flags)

void	sched_rem (struct thread *td)

fixpt_t	sched_pctcpu (struct thread *td)

void	sched_affinity (struct thread *td)

void	sched_bind (struct thread *td, int cpu)

void	sched_unbind (struct thread *td)

int	sched_is_bound (struct thread *td)

void	sched_relinquish (struct thread *td)

int	sched_load (void)

int	sched_sizeof_proc (void)

int	sched_sizeof_thread (void)

void	sched_idletd (void *dummy)

static struct thread *	sched_throw_grab (struct tdq *tdq)

void	sched_ap_entry (void)

void	sched_throw (struct thread *td)

void	sched_fork_exit (struct thread *td)

char *	sched_tdname (struct thread *td)

static int	sysctl_kern_quantum (SYSCTL_HANDLER_ARGS)

	SYSCTL_NODE (_kern, OID_AUTO, sched, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "Scheduler")

	SYSCTL_STRING (_kern_sched, OID_AUTO, name, CTLFLAG_RD, "ULE", 0, "Scheduler name")

	SYSCTL_PROC (_kern_sched, OID_AUTO, quantum, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, NULL, 0, sysctl_kern_quantum, "I", "Quantum for timeshare threads in microseconds")

	SYSCTL_INT (_kern_sched, OID_AUTO, slice, CTLFLAG_RW, &sched_slice, 0, "Quantum for timeshare threads in stathz ticks")

	SYSCTL_UINT (_kern_sched, OID_AUTO, interact, CTLFLAG_RW, &sched_interact, 0, "Interactivity score threshold")

	SYSCTL_INT (_kern_sched, OID_AUTO, preempt_thresh, CTLFLAG_RW, &preempt_thresh, 0, "Maximal (lowest) priority for preemption")

	SYSCTL_INT (_kern_sched, OID_AUTO, static_boost, CTLFLAG_RW, &static_boost, 0, "Assign static kernel priorities to sleeping threads")

	SYSCTL_INT (_kern_sched, OID_AUTO, idlespins, CTLFLAG_RW, &sched_idlespins, 0, "Number of times idle thread will spin waiting for new work")

	SYSCTL_INT (_kern_sched, OID_AUTO, idlespinthresh, CTLFLAG_RW, &sched_idlespinthresh, 0, "Threshold before we will permit idle thread spinning")

	SYSCTL_INT (_kern, OID_AUTO, ccpu, CTLFLAG_RD, &ccpu, 0, "Decay factor used for updating %CPU in 4BSD scheduler")

Variables
static u_int __read_mostly	sched_interact = SCHED_INTERACT_THRESH

static int __read_mostly	tickincr = 8 << SCHED_TICK_SHIFT

static int __read_mostly	realstathz = 127

static int __read_mostly	sched_slice = 10

static int __read_mostly	sched_slice_min = 1

static int __read_mostly	preempt_thresh = 0

static int __read_mostly	static_boost = PRI_MIN_BATCH

static int __read_mostly	sched_idlespins = 10000

static int __read_mostly	sched_idlespinthresh = -1

struct mtx_padalign	tdq_lock

struct cpu_group *	tdq_cg

volatile int	tdq_load

volatile int	tdq_cpu_idle

int	tdq_sysload

volatile int	tdq_transferable

volatile short	tdq_switchcnt

volatile short	tdq_oldswitchcnt

u_char	tdq_lowpri

u_char	tdq_owepreempt

u_char	tdq_idx

u_char	tdq_ridx

int	tdq_id

struct runq	tdq_realtime

struct runq	tdq_timeshare

struct runq	tdq_idle

char	tdq_name [TDQ_NAME_LEN]

static struct tdq	tdq_cpu

static int	ccpu = 0

Macro Definition Documentation

◆ KTR_ULE

#define KTR_ULE 0

Definition at line 82 of file sched_ule.c.

◆ PRI_BATCH_RANGE

#define PRI_BATCH_RANGE (PRI_TIMESHARE_RANGE - PRI_INTERACT_RANGE)

Definition at line 128 of file sched_ule.c.

◆ PRI_INTERACT_RANGE

#define PRI_INTERACT_RANGE ((PRI_TIMESHARE_RANGE - SCHED_PRI_NRESV) / 2)

Definition at line 127 of file sched_ule.c.

◆ PRI_MAX_BATCH

#define PRI_MAX_BATCH PRI_MAX_TIMESHARE

Definition at line 133 of file sched_ule.c.

◆ PRI_MAX_INTERACT

#define PRI_MAX_INTERACT (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE - 1)

Definition at line 131 of file sched_ule.c.

◆ PRI_MIN_BATCH

#define PRI_MIN_BATCH (PRI_MIN_TIMESHARE + PRI_INTERACT_RANGE)

Definition at line 132 of file sched_ule.c.

◆ PRI_MIN_INTERACT

#define PRI_MIN_INTERACT PRI_MIN_TIMESHARE

Definition at line 130 of file sched_ule.c.

◆ PRI_TIMESHARE_RANGE

#define PRI_TIMESHARE_RANGE (PRI_MAX_TIMESHARE - PRI_MIN_TIMESHARE + 1)

Definition at line 126 of file sched_ule.c.

◆ SCHED_INTERACT_HALF

#define SCHED_INTERACT_HALF (SCHED_INTERACT_MAX / 2)

Definition at line 189 of file sched_ule.c.

◆ SCHED_INTERACT_MAX

#define SCHED_INTERACT_MAX (100)

Definition at line 188 of file sched_ule.c.

◆ SCHED_INTERACT_THRESH

#define SCHED_INTERACT_THRESH (30)

Definition at line 190 of file sched_ule.c.

◆ SCHED_PRI_MAX

#define SCHED_PRI_MAX (PRI_MAX_BATCH - SCHED_PRI_NHALF)

Definition at line 167 of file sched_ule.c.

◆ SCHED_PRI_MIN

#define SCHED_PRI_MIN (PRI_MIN_BATCH + SCHED_PRI_NHALF)

Definition at line 166 of file sched_ule.c.

◆ SCHED_PRI_NHALF

#define SCHED_PRI_NHALF (SCHED_PRI_NRESV / 2)

Definition at line 165 of file sched_ule.c.

◆ SCHED_PRI_NICE

#define SCHED_PRI_NICE ( nice ) (nice)

Definition at line 172 of file sched_ule.c.

◆ SCHED_PRI_NRESV

#define SCHED_PRI_NRESV (PRIO_MAX - PRIO_MIN)

Definition at line 164 of file sched_ule.c.

◆ SCHED_PRI_RANGE

#define SCHED_PRI_RANGE (SCHED_PRI_MAX - SCHED_PRI_MIN + 1)

Definition at line 168 of file sched_ule.c.

◆ SCHED_PRI_TICKS

#define SCHED_PRI_TICKS ( ts )

Value:

(SCHED_TICK_HZ((ts)) / \

(roundup(SCHED_TICK_TOTAL((ts)), SCHED_PRI_RANGE) / SCHED_PRI_RANGE))

ts

struct timespec * ts

Definition: clock_if.m:39

SCHED_PRI_RANGE

#define SCHED_PRI_RANGE

Definition: sched_ule.c:168

SCHED_TICK_HZ

#define SCHED_TICK_HZ(ts)

Definition: sched_ule.c:149

SCHED_TICK_TOTAL

#define SCHED_TICK_TOTAL(ts)

Definition: sched_ule.c:150

Definition at line 169 of file sched_ule.c.

◆ SCHED_SLICE_DEFAULT_DIVISOR

#define SCHED_SLICE_DEFAULT_DIVISOR 10 /* ~94 ms, 12 stathz ticks. */

Definition at line 195 of file sched_ule.c.

◆ SCHED_SLICE_MIN_DIVISOR

#define SCHED_SLICE_MIN_DIVISOR 6 /* DEFAULT/MIN = ~16 ms. */

Definition at line 196 of file sched_ule.c.

◆ SCHED_SLP_RUN_FORK

#define SCHED_SLP_RUN_FORK ((hz / 2) << SCHED_TICK_SHIFT)

Definition at line 187 of file sched_ule.c.

◆ SCHED_SLP_RUN_MAX

#define SCHED_SLP_RUN_MAX ((hz * 5) << SCHED_TICK_SHIFT)

Definition at line 186 of file sched_ule.c.

◆ SCHED_TICK_HZ

#define SCHED_TICK_HZ ( ts ) ((ts)->ts_ticks >> SCHED_TICK_SHIFT)

Definition at line 149 of file sched_ule.c.

◆ SCHED_TICK_MAX

#define SCHED_TICK_MAX (SCHED_TICK_TARG + hz)

Definition at line 147 of file sched_ule.c.

◆ SCHED_TICK_SECS

#define SCHED_TICK_SECS 10

Definition at line 145 of file sched_ule.c.

◆ SCHED_TICK_SHIFT

#define SCHED_TICK_SHIFT 10

Definition at line 148 of file sched_ule.c.

◆ SCHED_TICK_TARG

#define SCHED_TICK_TARG (hz * SCHED_TICK_SECS)

Definition at line 146 of file sched_ule.c.

◆ SCHED_TICK_TOTAL

#define SCHED_TICK_TOTAL ( ts ) (max((ts)->ts_ltick - (ts)->ts_ftick, hz))

Definition at line 150 of file sched_ule.c.

◆ TDF_PICKCPU

#define TDF_PICKCPU TDF_SCHED0 /* Thread should pick new CPU. */

Definition at line 199 of file sched_ule.c.

◆ TDF_SLICEEND

#define TDF_SLICEEND TDF_SCHED2 /* Thread time slice is over. */

Definition at line 200 of file sched_ule.c.

◆ TDQ_CPU

#define TDQ_CPU ( x ) (&tdq_cpu)

Definition at line 298 of file sched_ule.c.

◆ TDQ_ID

#define TDQ_ID ( x ) (0)

Definition at line 296 of file sched_ule.c.

◆ TDQ_IDLE

#define TDQ_IDLE 2

Definition at line 263 of file sched_ule.c.

◆ TDQ_IDLESPIN

#define TDQ_IDLESPIN ( tdq ) 1

Definition at line 2900 of file sched_ule.c.

◆ TDQ_LOADNAME_LEN

#define TDQ_LOADNAME_LEN (sizeof("CPU ") + sizeof(__XSTRING(MAXCPU)) - 1 + sizeof(" load"))

Definition at line 86 of file sched_ule.c.

◆ TDQ_LOCK

#define TDQ_LOCK ( t ) mtx_lock_spin(TDQ_LOCKPTR((t)))

Definition at line 302 of file sched_ule.c.

◆ TDQ_LOCK_ASSERT

#define TDQ_LOCK_ASSERT	(	t,
		type
	)	mtx_assert(TDQ_LOCKPTR((t)), (type))

Definition at line 301 of file sched_ule.c.

◆ TDQ_LOCK_FLAGS

#define TDQ_LOCK_FLAGS	(	t,
		f
	)	mtx_lock_spin_flags(TDQ_LOCKPTR((t)), (f))

Definition at line 303 of file sched_ule.c.

◆ TDQ_LOCKPTR

#define TDQ_LOCKPTR ( t ) ((struct mtx *)(&(t)->tdq_lock))

Definition at line 307 of file sched_ule.c.

◆ TDQ_NAME_LEN

#define TDQ_NAME_LEN (sizeof("sched lock ") + sizeof(__XSTRING(MAXCPU)))

Definition at line 85 of file sched_ule.c.

◆ TDQ_RUNNING

#define TDQ_RUNNING 1

Definition at line 262 of file sched_ule.c.

◆ TDQ_SELF

#define TDQ_SELF ( ) (&tdq_cpu)

Definition at line 297 of file sched_ule.c.

◆ TDQ_TRYLOCK

#define TDQ_TRYLOCK ( t ) mtx_trylock_spin(TDQ_LOCKPTR((t)))

Definition at line 304 of file sched_ule.c.

◆ TDQ_TRYLOCK_FLAGS

#define TDQ_TRYLOCK_FLAGS	(	t,
		f
	)	mtx_trylock_spin_flags(TDQ_LOCKPTR((t)), (f))

Definition at line 305 of file sched_ule.c.

◆ TDQ_UNLOCK

#define TDQ_UNLOCK ( t ) mtx_unlock_spin(TDQ_LOCKPTR((t)))

Definition at line 306 of file sched_ule.c.

◆ THREAD_CAN_MIGRATE

#define THREAD_CAN_MIGRATE ( td ) ((td)->td_pinned == 0)

Definition at line 111 of file sched_ule.c.

◆ THREAD_CAN_SCHED

#define THREAD_CAN_SCHED	(	td,
		cpu
	)	CPU_ISSET((cpu), &(td)->td_cpuset->cs_mask)

Definition at line 112 of file sched_ule.c.

◆ TS_NAME_LEN

#define TS_NAME_LEN (MAXCOMLEN + sizeof(" td ") + sizeof(__XSTRING(UINT_MAX)))

Definition at line 84 of file sched_ule.c.

◆ TSF_BOUND

#define TSF_BOUND 0x0001 /* Thread can not migrate. */

Definition at line 108 of file sched_ule.c.

◆ TSF_XFERABLE

#define TSF_XFERABLE 0x0002 /* Thread was added as transferable. */

Definition at line 109 of file sched_ule.c.

Function Documentation

◆ __aligned()

struct tdq __aligned ( 64 )

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _Static_assert()

_Static_assert	(	sizeof(struct thread)+sizeof(struct td_sched)<=sizeof(struct thread0_storage)	,
		"increase struct thread0_storage.t0st_sched size"
	)

◆ runq_print()

static void runq_print ( struct runq * rq )

static

Definition at line 372 of file sched_ule.c.

References printf().

Referenced by tdq_print().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_add()

void sched_add	(	struct thread *	td,
		int	flags
	)

Definition at line 2671 of file sched_ule.c.

References flags, sched_priority(), sched_setpreempt(), sched_tdname(), tdq_add(), TDQ_LOCK, TDQ_LOCKPTR, TDQ_SELF, and thread_lock_set().

Referenced by sched_affinity(), sched_thread_priority(), and sched_wakeup().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_affinity()

void sched_affinity ( struct thread * td )

Definition at line 2780 of file sched_ule.c.

References sched_add(), sched_rem(), THREAD_CAN_SCHED, and ts.

Here is the call graph for this function:

◆ sched_ap_entry()

void sched_ap_entry ( void )

Definition at line 3010 of file sched_ule.c.

References cpu_ticks, sched_throw_grab(), TDQ_LOCK, TDQ_LOCKPTR, TDQ_SELF, and ticks.

Here is the call graph for this function:

◆ sched_bind()

void sched_bind	(	struct thread *	td,
		int	cpu
	)

Definition at line 2811 of file sched_ule.c.

References mi_switch(), sched_unbind(), THREAD_CAN_MIGRATE, ts, and TSF_BOUND.

Here is the call graph for this function:

◆ sched_choose()

struct thread * sched_choose ( void )

Definition at line 2598 of file sched_ule.c.

References tdq_choose(), TDQ_LOCK_ASSERT, tdq::tdq_lowpri, tdq_runq_rem(), and TDQ_SELF.

Here is the call graph for this function:

◆ sched_class()

void sched_class	(	struct thread *	td,
		int	class
	)

Definition at line 2406 of file sched_ule.c.

◆ sched_clock()

void sched_clock	(	struct thread *	td,
		int	cnt
	)

Definition at line 2501 of file sched_ule.c.

References sched_interact_update(), sched_pctcpu_update(), sched_priority(), smp_started, TDF_SLICEEND, tdq::tdq_idx, tdq::tdq_load, tdq::tdq_oldswitchcnt, tdq::tdq_ridx, TDQ_SELF, tdq_slice(), tdq::tdq_switchcnt, tdq::tdq_timeshare, tickincr, and ts.

Here is the call graph for this function:

◆ sched_estcpu()

u_int sched_estcpu ( struct thread *td __unused )

Definition at line 2562 of file sched_ule.c.

◆ sched_exit()

void sched_exit	(	struct proc *	p,
		struct thread *	child
	)

Definition at line 2419 of file sched_ule.c.

References child, sched_exit_thread(), and sched_tdname().

Here is the call graph for this function:

◆ sched_exit_thread()

void sched_exit_thread	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2437 of file sched_ule.c.

References child, sched_interact_update(), sched_priority(), and sched_tdname().

Referenced by sched_exit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_fork()

void sched_fork	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2341 of file sched_ule.c.

References child, sched_fork_thread(), sched_interact_fork(), sched_interact_update(), sched_pctcpu_update(), sched_priority(), and tickincr.

Here is the call graph for this function:

◆ sched_fork_exit()

void sched_fork_exit ( struct thread * td )

Definition at line 3063 of file sched_ule.c.

References sched_tdname(), TDQ_LOCK, TDQ_LOCKPTR, and TDQ_SELF.

Here is the call graph for this function:

◆ sched_fork_thread()

void sched_fork_thread	(	struct thread *	td,
		struct thread *	child
	)

Definition at line 2360 of file sched_ule.c.

References child, cpuset_ref(), sched_slice_min, TDQ_LOCKPTR, TDQ_SELF, tdq_slice(), ts, td_sched::ts_cpu, td_sched::ts_flags, td_sched::ts_ftick, td_sched::ts_ltick, td_sched::ts_runtime, td_sched::ts_slice, td_sched::ts_slptime, and td_sched::ts_ticks.

Referenced by sched_fork().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_idletd()

void sched_idletd ( void * dummy )

Definition at line 2907 of file sched_ule.c.

References Giant, mi_switch(), sched_idlespins, sched_idlespinthresh, tdq::tdq_cpu_idle, TDQ_IDLESPIN, tdq::tdq_load, tdq::tdq_oldswitchcnt, TDQ_SELF, and tdq::tdq_switchcnt.

Here is the call graph for this function:

◆ sched_initticks()

static void sched_initticks ( void * dummy )

static

Definition at line 1519 of file sched_ule.c.

References hogticks, hz, realstathz, sched_idlespinthresh, sched_slice, SCHED_SLICE_DEFAULT_DIVISOR, sched_slice_min, SCHED_SLICE_MIN_DIVISOR, SCHED_TICK_SHIFT, stathz, and tickincr.

◆ sched_interact_fork()

static void sched_interact_fork ( struct thread * td )

static

Definition at line 1713 of file sched_ule.c.

References SCHED_SLP_RUN_FORK, and ts.

Referenced by sched_fork().

Here is the caller graph for this function:

◆ sched_interact_score()

static int sched_interact_score ( struct thread * td )

static

Definition at line 1577 of file sched_ule.c.

References sched_interact, SCHED_INTERACT_HALF, and ts.

Referenced by sched_priority().

Here is the caller graph for this function:

◆ sched_interact_update()

static void sched_interact_update ( struct thread * td )

static

Definition at line 1668 of file sched_ule.c.

References SCHED_SLP_RUN_MAX, and ts.

Referenced by sched_clock(), sched_exit_thread(), sched_fork(), and sched_wakeup().

Here is the caller graph for this function:

◆ sched_is_bound()

int sched_is_bound ( struct thread * td )

Definition at line 2849 of file sched_ule.c.

References td_sched::ts_flags, and TSF_BOUND.

◆ sched_lend_prio()

void sched_lend_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1868 of file sched_ule.c.

References sched_thread_priority().

Referenced by sched_unlend_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_lend_user_prio()

void sched_lend_user_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1944 of file sched_ule.c.

References sched_prio().

Referenced by sched_lend_user_prio_cond().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_lend_user_prio_cond()

void sched_lend_user_prio_cond	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1960 of file sched_ule.c.

References sched_lend_user_prio().

Here is the call graph for this function:

◆ sched_load()

int sched_load ( void )

Definition at line 2869 of file sched_ule.c.

References TDQ_CPU, TDQ_SELF, and tdq_sysload.

◆ sched_nice()

void sched_nice	(	struct proc *	p,
		int	nice
	)

Definition at line 2267 of file sched_ule.c.

References sched_prio(), and sched_priority().

Here is the call graph for this function:

◆ sched_pctcpu()

fixpt_t sched_pctcpu ( struct thread * td )

Definition at line 2754 of file sched_ule.c.

References hz, sched_pctcpu_update(), SCHED_TICK_HZ, SCHED_TICK_SECS, and ts.

Here is the call graph for this function:

◆ sched_pctcpu_update()

static void sched_pctcpu_update	(	struct td_sched *	ts,
		int	run
	)

static

Definition at line 1786 of file sched_ule.c.

References SCHED_TICK_MAX, SCHED_TICK_SHIFT, SCHED_TICK_TARG, ticks, and ts.

Referenced by sched_clock(), sched_fork(), sched_pctcpu(), sched_switch(), and sched_wakeup().

Here is the caller graph for this function:

◆ sched_preempt()

void sched_preempt ( struct thread * td )

Definition at line 2455 of file sched_ule.c.

References flags, mi_switch(), TDQ_LOCK_ASSERT, tdq::tdq_lowpri, tdq::tdq_owepreempt, and TDQ_SELF.

Here is the call graph for this function:

◆ sched_prio()

void sched_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1904 of file sched_ule.c.

References sched_thread_priority(), and turnstile_adjust().

Referenced by sched_lend_user_prio(), sched_nice(), and sched_sleep().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_priority()

static void sched_priority ( struct thread * td )

static

Definition at line 1617 of file sched_ule.c.

References PRI_MAX_BATCH, PRI_MAX_INTERACT, PRI_MIN_BATCH, PRI_MIN_INTERACT, sched_interact, sched_interact_score(), SCHED_PRI_MIN, SCHED_PRI_NICE, SCHED_PRI_RANGE, SCHED_PRI_TICKS, sched_user_prio(), td_sched::ts_ftick, td_sched::ts_ltick, and td_sched::ts_ticks.

Referenced by sched_add(), sched_clock(), sched_exit_thread(), sched_fork(), and sched_nice().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_relinquish()

void sched_relinquish ( struct thread * td )

Definition at line 2859 of file sched_ule.c.

References mi_switch().

Here is the call graph for this function:

◆ sched_rem()

void sched_rem ( struct thread * td )

Definition at line 2731 of file sched_ule.c.

References sched_tdname(), TDQ_CPU, tdq_load_rem(), TDQ_LOCK_ASSERT, TDQ_LOCKPTR, tdq::tdq_lowpri, tdq_runq_rem(), and tdq_setlowpri().

Referenced by sched_affinity(), and sched_thread_priority().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_rr_interval()

int sched_rr_interval ( void )

Definition at line 1772 of file sched_ule.c.

References hz, realstathz, and sched_slice.

◆ sched_runnable()

int sched_runnable ( void )

Definition at line 2573 of file sched_ule.c.

References tdq::tdq_load, and TDQ_SELF.

◆ sched_setpreempt()

static void sched_setpreempt ( struct thread * td )

inlinestatic

Definition at line 2620 of file sched_ule.c.

References sched_shouldpreempt().

Referenced by sched_add().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_setup()

static void sched_setup ( void * dummy )

static

Definition at line 1495 of file sched_ule.c.

References tdq_load_add(), TDQ_LOCK, TDQ_LOCKPTR, tdq::tdq_lowpri, TDQ_SELF, tdq_setup(), and TDQ_UNLOCK.

Here is the call graph for this function:

◆ sched_shouldpreempt()

static int sched_shouldpreempt	(	int	pri,
		int	cpri,
		int	remote
	)

inlinestatic

Definition at line 425 of file sched_ule.c.

References preempt_thresh, and PRI_MAX_INTERACT.

Referenced by sched_setpreempt().

Here is the caller graph for this function:

◆ sched_sizeof_proc()

int sched_sizeof_proc ( void )

Definition at line 2885 of file sched_ule.c.

◆ sched_sizeof_thread()

int sched_sizeof_thread ( void )

Definition at line 2891 of file sched_ule.c.

◆ sched_sleep()

void sched_sleep	(	struct thread *	td,
		int	prio
	)

Definition at line 2286 of file sched_ule.c.

References sched_prio(), static_boost, and ticks.

Here is the call graph for this function:

◆ sched_switch()

void sched_switch	(	struct thread *	td,
		int	flags
	)

Definition at line 2135 of file sched_ule.c.

References choosethread(), flags, mtx, sched_pctcpu_update(), sched_switch_migrate(), sched_tdname(), TDF_PICKCPU, TDF_SLICEEND, tdq::tdq_load, tdq_load_rem(), TDQ_LOCK, TDQ_LOCK_ASSERT, TDQ_LOCKPTR, tdq::tdq_owepreempt, tdq_runq_add(), TDQ_SELF, tdq::tdq_switchcnt, TDQ_UNLOCK, THREAD_CAN_MIGRATE, THREAD_CAN_SCHED, thread_lock_block(), thread_unblock_switch(), ticks, and ts.

Here is the call graph for this function:

◆ sched_switch_migrate()

static struct mtx * sched_switch_migrate	(	struct tdq *	tdq,
		struct thread *	td,
		int	flags
	)

static

Definition at line 2090 of file sched_ule.c.

References flags, tdq_add(), TDQ_CPU, tdq_load_rem(), TDQ_LOCK, TDQ_LOCKPTR, TDQ_UNLOCK, THREAD_CAN_MIGRATE, and TSF_BOUND.

Referenced by sched_switch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_tdname()

char * sched_tdname ( struct thread * td )

Definition at line 3089 of file sched_ule.c.

References snprintf(), and ts.

Referenced by sched_add(), sched_exit(), sched_exit_thread(), sched_fork_exit(), sched_rem(), sched_switch(), and sched_thread_priority().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_thread_priority()

static void sched_thread_priority	(	struct thread *	td,
		u_char	prio
	)

static

Definition at line 1813 of file sched_ule.c.

References sched_add(), sched_rem(), sched_tdname(), TDQ_CPU, tdq_lowpri, tdq::tdq_lowpri, tdq_setlowpri(), and ts.

Referenced by sched_lend_prio(), sched_prio(), and sched_unlend_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_throw()

void sched_throw ( struct thread * td )

Definition at line 3036 of file sched_ule.c.

References sched_throw_grab(), tdq_load_rem(), TDQ_LOCKPTR, TDQ_SELF, and thread_lock_block().

Here is the call graph for this function:

◆ sched_throw_grab()

static struct thread * sched_throw_grab ( struct tdq * tdq )

static

Definition at line 2994 of file sched_ule.c.

References choosethread(), and TDQ_UNLOCK.

Referenced by sched_ap_entry(), and sched_throw().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sched_unbind()

void sched_unbind ( struct thread * td )

Definition at line 2835 of file sched_ule.c.

References ts, and TSF_BOUND.

Referenced by sched_bind().

Here is the caller graph for this function:

◆ sched_unlend_prio()

void sched_unlend_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1884 of file sched_ule.c.

References sched_lend_prio(), and sched_thread_priority().

Here is the call graph for this function:

◆ sched_user_prio()

void sched_user_prio	(	struct thread *	td,
		u_char	prio
	)

Definition at line 1934 of file sched_ule.c.

Referenced by sched_priority().

Here is the caller graph for this function:

◆ sched_userret_slowpath()

void sched_userret_slowpath ( struct thread * td )

Definition at line 2486 of file sched_ule.c.

References TDQ_SELF, and tdq_setlowpri().

Here is the call graph for this function:

◆ sched_wakeup()

void sched_wakeup	(	struct thread *	td,
		int	srqflags
	)

Definition at line 2309 of file sched_ule.c.

References sched_add(), sched_interact_update(), sched_pctcpu_update(), SCHED_TICK_SHIFT, ticks, and ts.

Here is the call graph for this function:

◆ schedinit()

void schedinit ( void )

Definition at line 1732 of file sched_ule.c.

References ticks, td_sched::ts_cpu, td_sched::ts_ftick, td_sched::ts_ltick, and td_sched::ts_slice.

◆ schedinit_ap()

void schedinit_ap ( void )

Definition at line 1757 of file sched_ule.c.

References TDQ_LOCKPTR, and TDQ_SELF.

◆ SDT_PROBE_DEFINE() [1/2]

SDT_PROBE_DEFINE	(	sched	,
		on__cpu
	)

◆ SDT_PROBE_DEFINE() [2/2]

SDT_PROBE_DEFINE	(	sched	,
		remain__cpu
	)

◆ SDT_PROBE_DEFINE2() [1/3]

SDT_PROBE_DEFINE2	(	sched	,
		load__change	,
		"int"	,
		"int"
	)

◆ SDT_PROBE_DEFINE2() [2/3]

SDT_PROBE_DEFINE2	(	sched	,
		off__cpu	,
		"struct thread *"	,
		"struct proc *"
	)

◆ SDT_PROBE_DEFINE2() [3/3]

SDT_PROBE_DEFINE2	(	sched	,
		surrender	,
		"struct thread *"	,
		"struct proc *"
	)

◆ SDT_PROBE_DEFINE3() [1/2]

SDT_PROBE_DEFINE3	(	sched	,
		change__pri	,
		"struct thread *"	,
		"struct proc *"	,
		"uint8_t"
	)

◆ SDT_PROBE_DEFINE3() [2/2]

SDT_PROBE_DEFINE3	(	sched	,
		dequeue	,
		"struct thread *"	,
		"struct proc *"	,
		"void *"
	)

◆ SDT_PROBE_DEFINE4() [1/2]

SDT_PROBE_DEFINE4	(	sched	,
		enqueue	,
		"struct thread *"	,
		"struct proc *"	,
		"void *"	,
		"int"
	)

◆ SDT_PROBE_DEFINE4() [2/2]

SDT_PROBE_DEFINE4	(	sched	,
		lend__pri	,
		"struct thread *"	,
		"struct proc *"	,
		"uint8_t"	,
		"struct thread *"
	)

◆ SDT_PROVIDER_DEFINE()

SDT_PROVIDER_DEFINE ( sched )

◆ SYSCTL_INT() [1/6]

SYSCTL_INT	(	_kern	,
		OID_AUTO	,
		ccpu	,
		CTLFLAG_RD	,
		&	ccpu,
		0	,
		"Decay factor used for updating %CPU in 4BSD scheduler"
	)

◆ SYSCTL_INT() [2/6]

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		idlespins	,
		CTLFLAG_RW	,
		&	sched_idlespins,
		0	,
		"Number of times idle thread will spin waiting for new work"
	)

◆ SYSCTL_INT() [3/6]

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		idlespinthresh	,
		CTLFLAG_RW	,
		&	sched_idlespinthresh,
		0	,
		"Threshold before we will permit idle thread spinning"
	)

◆ SYSCTL_INT() [4/6]

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		preempt_thresh	,
		CTLFLAG_RW	,
		&	preempt_thresh,
		0	,
		"Maximal (lowest) priority for preemption"
	)

◆ SYSCTL_INT() [5/6]

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		slice	,
		CTLFLAG_RW	,
		&	sched_slice,
		0	,
		"Quantum for timeshare threads in stathz ticks"
	)

◆ SYSCTL_INT() [6/6]

SYSCTL_INT	(	_kern_sched	,
		OID_AUTO	,
		static_boost	,
		CTLFLAG_RW	,
		&	static_boost,
		0	,
		"Assign static kernel priorities to sleeping threads"
	)

◆ sysctl_kern_quantum()

static int sysctl_kern_quantum ( SYSCTL_HANDLER_ARGS )

static

Definition at line 3198 of file sched_ule.c.

References hogticks, hz, realstathz, sched_slice, sched_slice_min, SCHED_SLICE_MIN_DIVISOR, and sysctl_handle_int().

Here is the call graph for this function:

◆ SYSCTL_NODE()

SYSCTL_NODE	(	_kern	,
		OID_AUTO	,
		sched	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"Scheduler"
	)

◆ SYSCTL_PROC()

SYSCTL_PROC	(	_kern_sched	,
		OID_AUTO	,
		quantum	,
		CTLTYPE_INT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		NULL	,
		0	,
		sysctl_kern_quantum	,
		"I"	,
		"Quantum for timeshare threads in microseconds"
	)