#include <sys/cdefs.h>
#include "opt_ddb.h"
#include "opt_turnstile_profiling.h"
#include "opt_sched.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/queue.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <vm/uma.h>

Include dependency graph for subr_turnstile.c:

Data Structures
struct	turnstile

struct	turnstile_chain

Macros
#define	TC_TABLESIZE 128 /* Must be power of 2. */

#define	TC_MASK (TC_TABLESIZE - 1)

#define	TC_SHIFT 8

#define	TC_HASH(lock) (((uintptr_t)(lock) >> TC_SHIFT) & TC_MASK)

#define	TC_LOOKUP(lock) &turnstile_chains[TC_HASH(lock)]

Functions
	__FBSDID ("$FreeBSD$")

static void	init_turnstile0 (void *dummy)

static void	propagate_priority (struct thread *td)

static int	turnstile_adjust_thread (struct turnstile ts, struct thread td)

static struct thread *	turnstile_first_waiter (struct turnstile *ts)

static void	turnstile_setowner (struct turnstile ts, struct thread owner)

static int	turnstile_init (void *mem, int size, int flags)

static void	turnstile_fini (void *mem, int size)

	SDT_PROVIDER_DECLARE (sched)

	SDT_PROBE_DEFINE (sched,,, sleep)

	SDT_PROBE_DEFINE2 (sched,,, wakeup, "struct thread ", "struct proc ")

static void	propagate_unlock_ts (struct turnstile top, struct turnstile ts)

static void	propagate_unlock_td (struct turnstile top, struct thread td)

void	init_turnstiles (void)

	SYSINIT (turnstile0, SI_SUB_LOCK, SI_ORDER_ANY, init_turnstile0, NULL)

void	turnstile_adjust (struct thread *td, u_char oldpri)

struct turnstile *	turnstile_alloc (void)

void	turnstile_free (struct turnstile *ts)

void	turnstile_chain_lock (struct lock_object *lock)

struct turnstile *	turnstile_trywait (struct lock_object *lock)

bool	turnstile_lock (struct turnstile ts, struct lock_object lockp, struct thread *tdp)

void	turnstile_unlock (struct turnstile ts, struct lock_object lock)

void	turnstile_assert (struct turnstile *ts)

void	turnstile_cancel (struct turnstile *ts)

struct turnstile *	turnstile_lookup (struct lock_object *lock)

void	turnstile_chain_unlock (struct lock_object *lock)

void	turnstile_claim (struct turnstile *ts)

void	turnstile_wait (struct turnstile ts, struct thread owner, int queue)

int	turnstile_signal (struct turnstile *ts, int queue)

void	turnstile_broadcast (struct turnstile *ts, int queue)

static u_char	turnstile_calc_unlend_prio_locked (struct thread *td)

void	turnstile_unpend (struct turnstile *ts)

void	turnstile_disown (struct turnstile *ts)

struct thread *	turnstile_head (struct turnstile *ts, int queue)

int	turnstile_empty (struct turnstile *ts, int queue)

Variables
static struct mtx	td_contested_lock

static struct turnstile_chain	turnstile_chains [TC_TABLESIZE]

static uma_zone_t	turnstile_zone

Macro Definition Documentation

◆ TC_HASH

#define TC_HASH ( lock ) (((uintptr_t)(lock) >> TC_SHIFT) & TC_MASK)

Definition at line 99 of file subr_turnstile.c.

◆ TC_LOOKUP

#define TC_LOOKUP ( lock ) &turnstile_chains[TC_HASH(lock)]

Definition at line 100 of file subr_turnstile.c.

◆ TC_MASK

#define TC_MASK (TC_TABLESIZE - 1)

Definition at line 97 of file subr_turnstile.c.

◆ TC_SHIFT

#define TC_SHIFT 8

Definition at line 98 of file subr_turnstile.c.

◆ TC_TABLESIZE

#define TC_TABLESIZE 128 /* Must be power of 2. */

Definition at line 96 of file subr_turnstile.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ init_turnstile0()

static void init_turnstile0 ( void * dummy )

static

Definition at line 424 of file subr_turnstile.c.

References turnstile_alloc(), turnstile_fini(), turnstile_init(), and turnstile_zone.

Here is the call graph for this function:

◆ init_turnstiles()

void init_turnstiles ( void )

Definition at line 383 of file subr_turnstile.c.

References tc_lock, TC_TABLESIZE, td_contested_lock, and turnstile_chains.

Referenced by mutex_init().

Here is the caller graph for this function:

◆ propagate_priority()

static void propagate_priority ( struct thread * td )

static

Definition at line 201 of file subr_turnstile.c.

References kdb_backtrace_thread(), panic(), printf(), propagate_unlock_td(), propagate_unlock_ts(), sched_lend_prio(), thread_lock_block_wait(), ts, and turnstile_adjust_thread().

Referenced by turnstile_adjust(), and turnstile_wait().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ propagate_unlock_td()

static void propagate_unlock_td	(	struct turnstile *	top,
		struct thread *	td
	)

inlinestatic

Definition at line 188 of file subr_turnstile.c.

References turnstile::ts_lock.

Referenced by propagate_priority().

Here is the caller graph for this function:

◆ propagate_unlock_ts()

static void propagate_unlock_ts	(	struct turnstile *	top,
		struct turnstile *	ts
	)

inlinestatic

Definition at line 180 of file subr_turnstile.c.

References ts.

Referenced by propagate_priority().

Here is the caller graph for this function:

◆ SDT_PROBE_DEFINE()

SDT_PROBE_DEFINE	(	sched	,
		sleep
	)

◆ SDT_PROBE_DEFINE2()

SDT_PROBE_DEFINE2	(	sched	,
		wakeup	,
		"struct thread *"	,
		"struct proc *"
	)

◆ SDT_PROVIDER_DECLARE()

SDT_PROVIDER_DECLARE ( sched )

◆ SYSINIT()

SYSINIT	(	turnstile0	,
		SI_SUB_LOCK	,
		SI_ORDER_ANY	,
		init_turnstile0	,
		NULL
	)

◆ turnstile_adjust()

void turnstile_adjust	(	struct thread *	td,
		u_char	oldpri
	)

Definition at line 444 of file subr_turnstile.c.

References propagate_priority(), ts, and turnstile_adjust_thread().

Referenced by sched_prio().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ turnstile_adjust_thread()

static int turnstile_adjust_thread	(	struct turnstile *	ts,
		struct thread *	td
	)

static

Definition at line 314 of file subr_turnstile.c.

References td_contested_lock, and ts.

Referenced by propagate_priority(), and turnstile_adjust().

Here is the caller graph for this function:

◆ turnstile_alloc()

struct turnstile * turnstile_alloc ( void )

Definition at line 541 of file subr_turnstile.c.

References turnstile_zone.

Referenced by init_turnstile0(), and thread_init().

Here is the caller graph for this function:

◆ turnstile_assert()

void turnstile_assert ( struct turnstile * ts )

Definition at line 628 of file subr_turnstile.c.

References ts.

◆ turnstile_broadcast()

void turnstile_broadcast	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 880 of file subr_turnstile.c.

References TC_LOOKUP, td_contested_lock, and ts.

Referenced by __mtx_unlock_sleep(), __rw_downgrade_int(), __rw_runlock_hard(), and __rw_wunlock_hard().

Here is the caller graph for this function:

◆ turnstile_calc_unlend_prio_locked()

static u_char turnstile_calc_unlend_prio_locked ( struct thread * td )

static

Definition at line 925 of file subr_turnstile.c.

References td_contested_lock, and turnstile_first_waiter().

Referenced by turnstile_disown(), and turnstile_unpend().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ turnstile_cancel()

void turnstile_cancel ( struct turnstile * ts )

Definition at line 634 of file subr_turnstile.c.

References TC_LOOKUP, and ts.

Referenced by __mtx_lock_sleep(), __rw_rlock_hard(), __rw_try_upgrade_int(), and __rw_wlock_hard().

Here is the caller graph for this function:

◆ turnstile_chain_lock()

void turnstile_chain_lock ( struct lock_object * lock )

Definition at line 561 of file subr_turnstile.c.

References TC_LOOKUP.

Referenced by __mtx_unlock_sleep(), __rw_downgrade_int(), __rw_runlock_hard(), __rw_wunlock_hard(), and _rm_unlock_hard().

Here is the caller graph for this function:

◆ turnstile_chain_unlock()

void turnstile_chain_unlock ( struct lock_object * lock )

Definition at line 674 of file subr_turnstile.c.

References TC_LOOKUP.

Referenced by __mtx_unlock_sleep(), __rw_downgrade_int(), __rw_runlock_hard(), __rw_wunlock_hard(), and _rm_unlock_hard().

Here is the caller graph for this function:

◆ turnstile_claim()

void turnstile_claim ( struct turnstile * ts )

Definition at line 703 of file subr_turnstile.c.

References sched_lend_prio(), TC_LOOKUP, td_contested_lock, ts, turnstile_first_waiter(), and turnstile_setowner().

Referenced by __rw_try_upgrade_int(), and __rw_wlock_hard().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ turnstile_disown()

void turnstile_disown ( struct turnstile * ts )

Definition at line 1026 of file subr_turnstile.c.

References sched_unlend_prio(), td_contested_lock, ts, and turnstile_calc_unlend_prio_locked().

Referenced by __rw_downgrade_int().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ turnstile_empty()

int turnstile_empty	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 1083 of file subr_turnstile.c.

References ts.

◆ turnstile_fini()

static void turnstile_fini	(	void *	mem,
		int	size
	)

static

Definition at line 529 of file subr_turnstile.c.

References ts.

Referenced by init_turnstile0().

Here is the caller graph for this function:

◆ turnstile_first_waiter()

static struct thread * turnstile_first_waiter ( struct turnstile * ts )

static

Definition at line 687 of file subr_turnstile.c.

References ts.

Referenced by turnstile_calc_unlend_prio_locked(), and turnstile_claim().

Here is the caller graph for this function:

◆ turnstile_free()

void turnstile_free ( struct turnstile * ts )

Definition at line 551 of file subr_turnstile.c.

References ts, and turnstile_zone.

Referenced by thread_fini().

Here is the caller graph for this function:

◆ turnstile_head()

struct thread * turnstile_head	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 1068 of file subr_turnstile.c.

References ts.

◆ turnstile_init()

static int turnstile_init	(	void *	mem,
		int	size,
		int	flags
	)

static

Definition at line 514 of file subr_turnstile.c.

References ts.

Referenced by init_turnstile0().

Here is the caller graph for this function:

◆ turnstile_lock()

bool turnstile_lock	(	struct turnstile *	ts,
		struct lock_object **	lockp,
		struct thread **	tdp
	)

Definition at line 593 of file subr_turnstile.c.

References TC_LOOKUP, and ts.

Referenced by epoch_block_handler_preempt().

Here is the caller graph for this function:

◆ turnstile_lookup()

struct turnstile * turnstile_lookup ( struct lock_object * lock )

Definition at line 655 of file subr_turnstile.c.

References TC_LOOKUP, and ts.

Referenced by __mtx_unlock_sleep(), __rw_downgrade_int(), __rw_runlock_hard(), __rw_wunlock_hard(), and _rm_unlock_hard().

Here is the caller graph for this function:

◆ turnstile_setowner()

static void turnstile_setowner	(	struct turnstile *	ts,
		struct thread *	owner
	)

static

Definition at line 479 of file subr_turnstile.c.

References td_contested_lock, and ts.

Referenced by turnstile_claim(), and turnstile_wait().

Here is the caller graph for this function:

◆ turnstile_signal()

int turnstile_signal	(	struct turnstile *	ts,
		int	queue
	)

Definition at line 829 of file subr_turnstile.c.

References empty, TC_LOOKUP, td_contested_lock, and ts.

Referenced by _rm_unlock_hard().

Here is the caller graph for this function:

◆ turnstile_trywait()

struct turnstile * turnstile_trywait ( struct lock_object * lock )

Definition at line 570 of file subr_turnstile.c.

References TC_LOOKUP, and ts.

Referenced by __mtx_lock_sleep(), __rw_rlock_hard(), __rw_try_upgrade_int(), __rw_wlock_hard(), and _rm_wlock().

Here is the caller graph for this function:

◆ turnstile_unlock()

void turnstile_unlock	(	struct turnstile *	ts,
		struct lock_object *	lock
	)

Definition at line 615 of file subr_turnstile.c.

References TC_LOOKUP, and ts.

Referenced by epoch_block_handler_preempt().

Here is the caller graph for this function:

◆ turnstile_unpend()

void turnstile_unpend ( struct turnstile * ts )

Definition at line 948 of file subr_turnstile.c.

References sched_add(), sched_unlend_prio(), TAILQ_HEAD(), td_contested_lock, thread_lock_block_wait(), ts, turnstile_calc_unlend_prio_locked(), and wakeup().

Referenced by __mtx_unlock_sleep(), __rw_downgrade_int(), __rw_runlock_hard(), __rw_wunlock_hard(), and _rm_unlock_hard().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ turnstile_wait()

void turnstile_wait	(	struct turnstile *	ts,
		struct thread *	owner,
		int	queue
	)

Definition at line 740 of file subr_turnstile.c.

References mi_switch(), propagate_priority(), TC_LOOKUP, td_contested_lock, thread_lock_set(), ticks, ts, and turnstile_setowner().

Referenced by __mtx_lock_sleep(), __rw_rlock_hard(), __rw_wlock_hard(), _rm_wlock(), and epoch_block_handler_preempt().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ td_contested_lock

struct mtx td_contested_lock

static

Definition at line 152 of file subr_turnstile.c.

Referenced by init_turnstiles(), turnstile_adjust_thread(), turnstile_broadcast(), turnstile_calc_unlend_prio_locked(), turnstile_claim(), turnstile_disown(), turnstile_setowner(), turnstile_signal(), turnstile_unpend(), and turnstile_wait().

◆ turnstile_chains

struct turnstile_chain turnstile_chains[TC_TABLESIZE]

static

Definition at line 153 of file subr_turnstile.c.

Referenced by init_turnstiles().

◆ turnstile_zone

uma_zone_t turnstile_zone

static

Definition at line 154 of file subr_turnstile.c.

Referenced by init_turnstile0(), turnstile_alloc(), and turnstile_free().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ TC_HASH

◆ TC_LOOKUP

◆ TC_MASK

◆ TC_SHIFT

◆ TC_TABLESIZE

Function Documentation

◆ __FBSDID()

◆ init_turnstile0()

◆ init_turnstiles()

◆ propagate_priority()

◆ propagate_unlock_td()

◆ propagate_unlock_ts()

◆ SDT_PROBE_DEFINE()

◆ SDT_PROBE_DEFINE2()

◆ SDT_PROVIDER_DECLARE()

◆ SYSINIT()

◆ turnstile_adjust()

◆ turnstile_adjust_thread()

◆ turnstile_alloc()

◆ turnstile_assert()

◆ turnstile_broadcast()

◆ turnstile_calc_unlend_prio_locked()

◆ turnstile_cancel()

◆ turnstile_chain_lock()

◆ turnstile_chain_unlock()

◆ turnstile_claim()

◆ turnstile_disown()

◆ turnstile_empty()

◆ turnstile_fini()

◆ turnstile_first_waiter()

◆ turnstile_free()

◆ turnstile_head()

◆ turnstile_init()

◆ turnstile_lock()

◆ turnstile_lookup()

◆ turnstile_setowner()

◆ turnstile_signal()

◆ turnstile_trywait()

◆ turnstile_unlock()

◆ turnstile_unpend()

◆ turnstile_wait()

Variable Documentation

◆ td_contested_lock

◆ turnstile_chains

◆ turnstile_zone