#include <sys/cdefs.h>
#include "opt_callout_profiling.h"
#include "opt_ddb.h"
#include "opt_rss.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/callout.h>
#include <sys/domainset.h>
#include <sys/file.h>
#include <sys/interrupt.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/kthread.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/random.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/sleepqueue.h>
#include <sys/sysctl.h>
#include <sys/smp.h>
#include <sys/unistd.h>

Include dependency graph for kern_timeout.c:

Data Structures
struct	cc_exec

struct	callout_cpu

Macros
#define	callout_migrating(c) ((c)->c_iflags & CALLOUT_DFRMIGRATION)

#define	cc_exec_curr(cc, dir) cc->cc_exec_entity[dir].cc_curr

#define	cc_exec_last_func(cc, dir) cc->cc_exec_entity[dir].cc_last_func

#define	cc_exec_last_arg(cc, dir) cc->cc_exec_entity[dir].cc_last_arg

#define	cc_exec_drain(cc, dir) cc->cc_exec_entity[dir].cc_drain

#define	cc_exec_next(cc) cc->cc_next

#define	cc_exec_cancel(cc, dir) cc->cc_exec_entity[dir].cc_cancel

#define	cc_exec_waiting(cc, dir) cc->cc_exec_entity[dir].cc_waiting

#define	CC_CPU(cpu) (&cc_cpu)

#define	CC_SELF() (&cc_cpu)

#define	CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)

#define	CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)

#define	CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED)

#define	CC_HASH_SHIFT 8

Functions
	__FBSDID ("$FreeBSD$")

	DPCPU_DECLARE (sbintime_t, hardclocktime)

	SDT_PROVIDER_DEFINE (callout_execute)

	SDT_PROBE_DEFINE1 (callout_execute,,, callout__start, "struct callout *")

	SDT_PROBE_DEFINE1 (callout_execute,,, callout__end, "struct callout *")

static void	softclock_thread (void *arg)

	SYSCTL_INT (_kern, OID_AUTO, ncallout, CTLFLAG_RDTUN\|CTLFLAG_NOFETCH, &ncallout, 0, "Number of entries in callwheel and size of timeout() preallocation")

	SYSCTL_INT (_kern, OID_AUTO, pin_default_swi, CTLFLAG_RDTUN\|CTLFLAG_NOFETCH, &pin_default_swi, 0, "Pin the default (non-per-cpu) swi (shared with PCPU 0 swi)")

	SYSCTL_INT (_kern, OID_AUTO, pin_pcpu_swi, CTLFLAG_RDTUN\|CTLFLAG_NOFETCH, &pin_pcpu_swi, 0, "Pin the per-CPU swis (except PCPU 0, which is also default)")

static void	callout_cpu_init (struct callout_cpu *cc, int cpu)

static void	softclock_call_cc (struct callout c, struct callout_cpu cc, int direct)

static	MALLOC_DEFINE (M_CALLOUT, "callout", "Callout datastructures")

static void	cc_cce_cleanup (struct callout_cpu *cc, int direct)

static int	cc_cce_migrating (struct callout_cpu *cc, int direct)

static void	callout_callwheel_init (void *dummy)

	SYSINIT (callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL)

static void	start_softclock (void *dummy)

	SYSINIT (start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL)

static u_int	callout_hash (sbintime_t sbt)

static u_int	callout_get_bucket (sbintime_t sbt)

void	callout_process (sbintime_t now)

static struct callout_cpu *	callout_lock (struct callout *c)

static void	callout_cc_add (struct callout c, struct callout_cpu cc, sbintime_t sbt, sbintime_t precision, void(func)(void ), void *arg, int cpu, int flags)

void	callout_when (sbintime_t sbt, sbintime_t precision, int flags, sbintime_t res, sbintime_t prec_res)

int	callout_reset_sbt_on (struct callout c, sbintime_t sbt, sbintime_t prec, callout_func_t ftn, void *arg, int cpu, int flags)

int	callout_schedule_on (struct callout *c, int to_ticks, int cpu)

int	callout_schedule (struct callout *c, int to_ticks)

int	_callout_stop_safe (struct callout c, int flags, callout_func_t drain)

void	callout_init (struct callout *c, int mpsafe)

void	_callout_init_lock (struct callout c, struct lock_object lock, int flags)

static int	flssbt (sbintime_t sbt)

static int	sysctl_kern_callout_stat (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern, OID_AUTO, callout_stat, CTLTYPE_INT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, 0, sysctl_kern_callout_stat, "I", "Dump immediate statistic snapshot of the scheduled callouts")

Variables
static int	ncallout

static int	pin_default_swi = 0

static int	pin_pcpu_swi = 0

static u_int __read_mostly	callwheelsize

static u_int __read_mostly	callwheelmask

static struct callout_cpu	cc_cpu

static int __read_mostly	cc_default_cpu

Macro Definition Documentation

◆ callout_migrating

#define callout_migrating ( c ) ((c)->c_iflags & CALLOUT_DFRMIGRATION)

Definition at line 183 of file kern_timeout.c.

◆ CC_CPU

#define CC_CPU ( cpu ) (&cc_cpu)

Definition at line 205 of file kern_timeout.c.

◆ cc_exec_cancel

#define cc_exec_cancel	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_cancel

Definition at line 190 of file kern_timeout.c.

◆ cc_exec_curr

#define cc_exec_curr	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_curr

Definition at line 185 of file kern_timeout.c.

◆ cc_exec_drain

#define cc_exec_drain	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_drain

Definition at line 188 of file kern_timeout.c.

◆ cc_exec_last_arg

#define cc_exec_last_arg	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_last_arg

Definition at line 187 of file kern_timeout.c.

◆ cc_exec_last_func

#define cc_exec_last_func	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_last_func

Definition at line 186 of file kern_timeout.c.

◆ cc_exec_next

#define cc_exec_next ( cc ) cc->cc_next

Definition at line 189 of file kern_timeout.c.

◆ cc_exec_waiting

#define cc_exec_waiting	(	cc,
		dir
	)	cc->cc_exec_entity[dir].cc_waiting

Definition at line 191 of file kern_timeout.c.

◆ CC_HASH_SHIFT

#define CC_HASH_SHIFT 8

Definition at line 415 of file kern_timeout.c.

◆ CC_LOCK

#define CC_LOCK ( cc ) mtx_lock_spin(&(cc)->cc_lock)

Definition at line 208 of file kern_timeout.c.

◆ CC_LOCK_ASSERT

#define CC_LOCK_ASSERT ( cc ) mtx_assert(&(cc)->cc_lock, MA_OWNED)

Definition at line 210 of file kern_timeout.c.

◆ CC_SELF

#define CC_SELF ( ) (&cc_cpu)

Definition at line 206 of file kern_timeout.c.

◆ CC_UNLOCK

#define CC_UNLOCK ( cc ) mtx_unlock_spin(&(cc)->cc_lock)

Definition at line 209 of file kern_timeout.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _callout_init_lock()

void _callout_init_lock	(	struct callout *	c,
		struct lock_object *	lock,
		int	flags
	)

Definition at line 1377 of file kern_timeout.c.

References cc_default_cpu, and flags.

◆ _callout_stop_safe()

int _callout_stop_safe	(	struct callout *	c,
		int	flags,
		callout_func_t *	drain
	)

Definition at line 1101 of file kern_timeout.c.

References callout_lock(), callout_migrating, cc_cce_migrating(), cc_exec_cancel, cc_exec_curr, cc_exec_drain, cc_exec_next, cc_exec_waiting, callout_cpu::cc_expireq, callout_cpu::cc_lock, CC_UNLOCK, flags, Giant, panic(), sleepq_add(), sleepq_lock(), sleepq_release(), and sleepq_wait().

Referenced by exit1().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_callwheel_init()

static void callout_callwheel_init ( void * dummy )

static

Definition at line 278 of file kern_timeout.c.

References callout_cpu_init(), callwheelmask, callwheelsize, CC_CPU, cc_default_cpu, maxfiles, maxproc, ncallout, pin_default_swi, and pin_pcpu_swi.

Here is the call graph for this function:

◆ callout_cc_add()

static void callout_cc_add	(	struct callout *	c,
		struct callout_cpu *	cc,
		sbintime_t	sbt,
		sbintime_t	precision,
		void()(void )	func,
		void *	arg,
		int	cpu,
		int	flags
	)

static

Definition at line 596 of file kern_timeout.c.

References callout_get_bucket(), callout_cpu::cc_bucket, callout_cpu::cc_callwheel, cc_exec_next, callout_cpu::cc_firstevent, callout_cpu::cc_lastscan, CC_LOCK_ASSERT, cpu_new_callout(), and flags.

Referenced by callout_reset_sbt_on(), and softclock_call_cc().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_cpu_init()

static void callout_cpu_init	(	struct callout_cpu *	cc,
		int	cpu
	)

static

Definition at line 321 of file kern_timeout.c.

References callwheelsize, callout_cpu::cc_callwheel, cc_cce_cleanup(), callout_cpu::cc_expireq, callout_cpu::cc_firstevent, callout_cpu::cc_inited, callout_cpu::cc_lock, malloc_domainset(), pcpu_find(), and snprintf().

Referenced by callout_callwheel_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_get_bucket()

static u_int callout_get_bucket ( sbintime_t sbt )

inlinestatic

Definition at line 425 of file kern_timeout.c.

References callout_hash(), and callwheelmask.

Referenced by callout_cc_add().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_hash()

static u_int callout_hash ( sbintime_t sbt )

inlinestatic

Definition at line 418 of file kern_timeout.c.

References CC_HASH_SHIFT.

Referenced by callout_get_bucket(), and callout_process().

Here is the caller graph for this function:

◆ callout_init()

void callout_init	(	struct callout *	c,
		int	mpsafe
	)

Definition at line 1363 of file kern_timeout.c.

References cc_default_cpu, and Giant.

Referenced by domaininit(), fail_point_alloc_callout(), filt_timerattach(), proc0_init(), start_periodic_resettodr(), synch_setup(), thread_link(), threadinit(), and vmem_start_callout().

Here is the caller graph for this function:

◆ callout_lock()

static struct callout_cpu * callout_lock ( struct callout * c )

static

Definition at line 572 of file kern_timeout.c.

References CC_CPU, CC_LOCK, and CC_UNLOCK.

Referenced by _callout_stop_safe(), and callout_reset_sbt_on().

Here is the caller graph for this function:

◆ callout_process()

void callout_process ( sbintime_t now )

Definition at line 432 of file kern_timeout.c.

References callout_hash(), callwheelmask, callwheelsize, callout_cpu::cc_bucket, callout_cpu::cc_callwheel, cc_exec_next, callout_cpu::cc_expireq, callout_cpu::cc_firstevent, CC_HASH_SHIFT, callout_cpu::cc_lastscan, callout_cpu::cc_lock, CC_SELF, callout_cpu::cc_thread, cpu_new_callout(), sched_add(), softclock_call_cc(), and thread_lock_block_wait().

Referenced by handleevents().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_reset_sbt_on()

int callout_reset_sbt_on	(	struct callout *	c,
		sbintime_t	sbt,
		sbintime_t	prec,
		callout_func_t *	ftn,
		void *	arg,
		int	cpu,
		int	flags
	)

Definition at line 943 of file kern_timeout.c.

References callout_cc_add(), callout_lock(), callout_migrating, callout_when(), CC_CPU, cc_exec_cancel, cc_exec_curr, cc_exec_drain, cc_exec_next, cc_exec_waiting, callout_cpu::cc_expireq, callout_cpu::cc_inited, CC_UNLOCK, flags, and panic().

Referenced by kqtimer_sched_callout(), and sleepq_set_timeout_sbt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ callout_schedule()

int callout_schedule	(	struct callout *	c,
		int	to_ticks
	)

Definition at line 1095 of file kern_timeout.c.

Referenced by periodic_resettodr().

Here is the caller graph for this function:

◆ callout_schedule_on()

int callout_schedule_on	(	struct callout *	c,
		int	to_ticks,
		int	cpu
	)

Definition at line 1089 of file kern_timeout.c.

◆ callout_when()

void callout_when	(	sbintime_t	sbt,
		sbintime_t	precision,
		int	flags,
		sbintime_t *	res,
		sbintime_t *	prec_res
	)

Definition at line 884 of file kern_timeout.c.

References flags, res, sbt_tickthreshold, tc_precexp, and tick_sbt.

Referenced by callout_reset_sbt_on(), and sleepq_set_timeout_sbt().

Here is the caller graph for this function:

◆ cc_cce_cleanup()

static void cc_cce_cleanup	(	struct callout_cpu *	cc,
		int	direct
	)

static

Locked by cc_lock: cc_curr - If a callout is in progress, it is cc_curr. If cc_curr is non-NULL, threads waiting in callout_drain() will be woken up as soon as the relevant callout completes. cc_cancel - Changing to 1 with both callout_lock and cc_lock held guarantees that the current callout will not run. The softclock_call_cc() function sets this to 0 before it drops callout_lock to acquire c_lock, and it calls the handler only if curr_cancelled is still 0 after cc_lock is successfully acquired. cc_waiting - If a thread is waiting in callout_drain(), then callout_wait is nonzero. Set only when cc_curr is non-NULL.

Definition at line 244 of file kern_timeout.c.

References cc_exec_cancel, cc_exec_curr, and cc_exec_waiting.

Referenced by callout_cpu_init(), and softclock_call_cc().

Here is the caller graph for this function:

◆ cc_cce_migrating()

static int cc_cce_migrating	(	struct callout_cpu *	cc,
		int	direct
	)

static

Definition at line 263 of file kern_timeout.c.

Referenced by _callout_stop_safe(), and softclock_call_cc().

Here is the caller graph for this function:

◆ DPCPU_DECLARE()

DPCPU_DECLARE	(	sbintime_t	,
		hardclocktime
	)

◆ flssbt()

static int flssbt ( sbintime_t sbt )

static

Definition at line 1392 of file kern_timeout.c.

Referenced by sysctl_kern_callout_stat().

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_CALLOUT	,
		"callout"	,
		"Callout datastructures"
	)

static

◆ SDT_PROBE_DEFINE1() [1/2]

SDT_PROBE_DEFINE1	(	callout_execute	,
		callout__end	,
		"struct callout *"
	)

◆ SDT_PROBE_DEFINE1() [2/2]

SDT_PROBE_DEFINE1	(	callout_execute	,
		callout__start	,
		"struct callout *"
	)

◆ SDT_PROVIDER_DEFINE()

SDT_PROVIDER_DEFINE ( callout_execute )

◆ softclock_call_cc()

static void softclock_call_cc	(	struct callout *	c,
		struct callout_cpu *	cc,
		int	direct
	)

static

Definition at line 636 of file kern_timeout.c.

References callout_cc_add(), callout_migrating, cc_cce_cleanup(), cc_cce_migrating(), cc_exec_cancel, cc_exec_curr, cc_exec_drain, cc_exec_last_arg, cc_exec_last_func, cc_exec_waiting, CC_LOCK, CC_UNLOCK, flags, Giant, lock_class_rm, panic(), printf(), and wakeup().

Referenced by callout_process(), and softclock_thread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ softclock_thread()

static void softclock_thread ( void * arg )

static

Definition at line 834 of file kern_timeout.c.

References callout_cpu::cc_expireq, callout_cpu::cc_lock, CC_LOCK, mi_switch(), mtx, softclock_call_cc(), and thread_lock_set().

Referenced by start_softclock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ start_softclock()

static void start_softclock ( void * dummy )

static

Definition at line 376 of file kern_timeout.c.

References CC_CPU, cc_default_cpu, callout_cpu::cc_lock, CC_LOCK, callout_cpu::cc_thread, cpuset_setithread(), kproc_kthread_add(), mtx, panic(), pin_default_swi, pin_pcpu_swi, printf(), sched_class(), sched_prio(), softclock_thread(), and thread_lock_set().

Here is the call graph for this function:

◆ SYSCTL_INT() [1/3]

SYSCTL_INT	(	_kern	,
		OID_AUTO	,
		ncallout	,
		CTLFLAG_RDTUN\|	CTLFLAG_NOFETCH,
		&	ncallout,
		0	,
		"Number of entries in callwheel and size of timeout() preallocation"
	)

◆ SYSCTL_INT() [2/3]

SYSCTL_INT	(	_kern	,
		OID_AUTO	,
		pin_default_swi	,
		CTLFLAG_RDTUN\|	CTLFLAG_NOFETCH,
		&	pin_default_swi,
		0	,
		"Pin the default (non-per-cpu) swi (shared with PCPU 0 swi)"
	)

◆ SYSCTL_INT() [3/3]

SYSCTL_INT	(	_kern	,
		OID_AUTO	,
		pin_pcpu_swi	,
		CTLFLAG_RDTUN\|	CTLFLAG_NOFETCH,
		&	pin_pcpu_swi,
		0	,
		"Pin the per-CPU swis (except PCPU 0, which is also default)"
	)

◆ sysctl_kern_callout_stat()

static int sysctl_kern_callout_stat ( SYSCTL_HANDLER_ARGS )

static

Definition at line 1407 of file kern_timeout.c.

References callwheelsize, callout_cpu::cc_callwheel, CC_CPU, CC_HASH_SHIFT, CC_LOCK, CC_UNLOCK, count, flssbt(), mp_ncpus, printf(), st, and sysctl_handle_int().

Here is the call graph for this function:

◆ SYSCTL_PROC()

SYSCTL_PROC	(	_kern	,
		OID_AUTO	,
		callout_stat	,
		CTLTYPE_INT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		0	,
		sysctl_kern_callout_stat	,
		"I"	,
		"Dump immediate statistic snapshot of the scheduled callouts"
	)

◆ SYSINIT() [1/2]

SYSINIT	(	callwheel_init	,
		SI_SUB_CPU	,
		SI_ORDER_ANY	,
		callout_callwheel_init	,
		NULL
	)

◆ SYSINIT() [2/2]

SYSINIT	(	start_softclock	,
		SI_SUB_SOFTINTR	,
		SI_ORDER_FIRST	,
		start_softclock	,
		NULL
	)

Variable Documentation

◆ callwheelmask

u_int __read_mostly callwheelmask

static

Definition at line 135 of file kern_timeout.c.

Referenced by callout_callwheel_init(), callout_get_bucket(), and callout_process().

◆ callwheelsize

u_int __read_mostly callwheelsize

static

Definition at line 134 of file kern_timeout.c.

Referenced by callout_callwheel_init(), callout_cpu_init(), callout_process(), and sysctl_kern_callout_stat().

◆ cc_cpu

struct callout_cpu cc_cpu

static

Definition at line 204 of file kern_timeout.c.

◆ cc_default_cpu

int __read_mostly cc_default_cpu

static

Definition at line 212 of file kern_timeout.c.

Referenced by _callout_init_lock(), callout_callwheel_init(), callout_init(), and start_softclock().

◆ ncallout

int ncallout

static

Definition at line 113 of file kern_timeout.c.

Referenced by callout_callwheel_init().

◆ pin_default_swi

int pin_default_swi = 0

static

Definition at line 121 of file kern_timeout.c.

Referenced by callout_callwheel_init(), and start_softclock().

◆ pin_pcpu_swi

int pin_pcpu_swi = 0

static

Definition at line 122 of file kern_timeout.c.

Referenced by callout_callwheel_init(), and start_softclock().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ callout_migrating

◆ CC_CPU

◆ cc_exec_cancel

◆ cc_exec_curr

◆ cc_exec_drain

◆ cc_exec_last_arg

◆ cc_exec_last_func

◆ cc_exec_next

◆ cc_exec_waiting

◆ CC_HASH_SHIFT

◆ CC_LOCK

◆ CC_LOCK_ASSERT

◆ CC_SELF

◆ CC_UNLOCK

Function Documentation

◆ __FBSDID()

◆ _callout_init_lock()

◆ _callout_stop_safe()

◆ callout_callwheel_init()

◆ callout_cc_add()

◆ callout_cpu_init()

◆ callout_get_bucket()

◆ callout_hash()

◆ callout_init()

◆ callout_lock()

◆ callout_process()

◆ callout_reset_sbt_on()

◆ callout_schedule()

◆ callout_schedule_on()

◆ callout_when()

◆ cc_cce_cleanup()

◆ cc_cce_migrating()

◆ DPCPU_DECLARE()

◆ flssbt()

◆ MALLOC_DEFINE()

◆ SDT_PROBE_DEFINE1() [1/2]

◆ SDT_PROBE_DEFINE1() [2/2]

◆ SDT_PROVIDER_DEFINE()

◆ softclock_call_cc()

◆ softclock_thread()

◆ start_softclock()

◆ SYSCTL_INT() [1/3]

◆ SYSCTL_INT() [2/3]

◆ SYSCTL_INT() [3/3]

◆ sysctl_kern_callout_stat()

◆ SYSCTL_PROC()

◆ SYSINIT() [1/2]

◆ SYSINIT() [2/2]

Variable Documentation

◆ callwheelmask

◆ callwheelsize

◆ cc_cpu

◆ cc_default_cpu

◆ ncallout

◆ pin_default_swi

◆ pin_pcpu_swi