#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_no_adaptive_sx.h"
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kdb.h>
#include <sys/kernel.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/sx.h>
#include <sys/smp.h>
#include <sys/sysctl.h>

Include dependency graph for kern_sx.c:

Macros
#define	SQ_EXCLUSIVE_QUEUE 0

#define	SQ_SHARED_QUEUE 1

#define	GIANT_DECLARE

#define	GIANT_SAVE(work)

#define	GIANT_RESTORE()

#define	sx_recursed(sx) ((sx)->sx_recurse != 0)

#define	_sx_assert(sx, what, file, line)

#define	sx_drop_critical(x, in_critical, extra_work) do { } while (0)

Functions
	__FBSDID ("$FreeBSD$")

static void	assert_sx (const struct lock_object *lock, int what)

static void	lock_sx (struct lock_object *lock, uintptr_t how)

static uintptr_t	unlock_sx (struct lock_object *lock)

void	sx_sysinit (void *arg)

void	sx_init_flags (struct sx sx, const char description, int opts)

void	sx_destroy (struct sx *sx)

int	sx_try_slock_int (struct sx *sx LOCK_FILE_LINE_ARG_DEF)

int	sx_try_slock_ (struct sx sx, const char file, int line)

int	_sx_xlock (struct sx sx, int opts, const char file, int line)

int	sx_try_xlock_int (struct sx *sx LOCK_FILE_LINE_ARG_DEF)

int	sx_try_xlock_ (struct sx sx, const char file, int line)

void	_sx_xunlock (struct sx sx, const char file, int line)

int	sx_try_upgrade_int (struct sx *sx LOCK_FILE_LINE_ARG_DEF)

int	sx_try_upgrade_ (struct sx sx, const char file, int line)

void	sx_downgrade_int (struct sx *sx LOCK_FILE_LINE_ARG_DEF)

void	sx_downgrade_ (struct sx sx, const char file, int line)

int	_sx_xlock_hard (struct sx *sx, uintptr_t x, int opts LOCK_FILE_LINE_ARG_DEF)

void	_sx_xunlock_hard (struct sx *sx, uintptr_t x LOCK_FILE_LINE_ARG_DEF)

static bool __always_inline	__sx_can_read (struct thread *td, uintptr_t x, bool fp)

static bool __always_inline	__sx_slock_try (struct sx sx, struct thread td, uintptr_t *xp, bool fp LOCK_FILE_LINE_ARG_DEF)

static int __noinline	_sx_slock_hard (struct sx *sx, int opts, uintptr_t x LOCK_FILE_LINE_ARG_DEF)

int	_sx_slock_int (struct sx *sx, int opts LOCK_FILE_LINE_ARG_DEF)

int	_sx_slock (struct sx sx, int opts, const char file, int line)

static bool __always_inline	_sx_sunlock_try (struct sx sx, struct thread td, uintptr_t *xp)

static void __noinline	_sx_sunlock_hard (struct sx sx, struct thread td, uintptr_t x LOCK_FILE_LINE_ARG_DEF)

void	_sx_sunlock_int (struct sx *sx LOCK_FILE_LINE_ARG_DEF)

void	_sx_sunlock (struct sx sx, const char file, int line)

Variables
struct lock_class	lock_class_sx

Macro Definition Documentation

◆ _sx_assert

#define _sx_assert	(	sx,
		what,
		file,
		line
	)

Definition at line 142 of file kern_sx.c.

◆ GIANT_DECLARE

#define GIANT_DECLARE

Value:

int _giantcnt = 0; \

WITNESS_SAVE_DECL(Giant) \

Giant

struct mtx __exclusive_cache_line Giant

Definition: kern_mutex.c:181

Definition at line 87 of file kern_sx.c.

◆ GIANT_RESTORE

#define GIANT_RESTORE ( )

Value:

        do {                                            \
        if (_giantcnt > 0) {                                            \
                mtx_assert(&Giant, MA_NOTOWNED);                        \
                while (_giantcnt--)                                     \
                        mtx_lock(&Giant);                               \
                WITNESS_RESTORE(&Giant.lock_object, Giant);             \
        }                                                               \
} while (0)

Definition at line 102 of file kern_sx.c.

◆ GIANT_SAVE

#define GIANT_SAVE ( work )

Value:

        do {                                            \
        if (__predict_false(mtx_owned(&Giant))) {                       \
                work++;                                                 \
                WITNESS_SAVE(&Giant.lock_object, Giant);                \
                while (mtx_owned(&Giant)) {                             \
                        _giantcnt++;                                    \
                        mtx_unlock(&Giant);                             \
                }                                                       \
        }                                                               \
} while (0)

Definition at line 91 of file kern_sx.c.

◆ SQ_EXCLUSIVE_QUEUE

#define SQ_EXCLUSIVE_QUEUE 0

Definition at line 80 of file kern_sx.c.

◆ SQ_SHARED_QUEUE

#define SQ_SHARED_QUEUE 1

Definition at line 81 of file kern_sx.c.

◆ sx_drop_critical

#define sx_drop_critical	(	x,
		in_critical,
		extra_work
	)	do { } while (0)

Definition at line 561 of file kern_sx.c.

◆ sx_recursed

#define sx_recursed ( sx ) ((sx)->sx_recurse != 0)

Definition at line 115 of file kern_sx.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ __sx_can_read()

static bool __always_inline __sx_can_read	(	struct thread *	td,
		uintptr_t	x,
		bool	fp
	)

static

Definition at line 981 of file kern_sx.c.

Referenced by __sx_slock_try(), and _sx_slock_hard().

Here is the caller graph for this function:

◆ __sx_slock_try()

static bool __always_inline __sx_slock_try	(	struct sx *	sx,
		struct thread *	td,
		uintptr_t *	xp,
		bool fp	LOCK_FILE_LINE_ARG_DEF
	)

static

Definition at line 993 of file kern_sx.c.

References __sx_can_read().

Referenced by _sx_slock_hard(), and _sx_slock_int().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_slock()

int _sx_slock	(	struct sx *	sx,
		int	opts,
		const char *	file,
		int	line
	)

Definition at line 1286 of file kern_sx.c.

References _sx_slock_int().

Here is the call graph for this function:

◆ _sx_slock_hard()

static int __noinline _sx_slock_hard	(	struct sx *	sx,
		int	opts,
		uintptr_t x	LOCK_FILE_LINE_ARG_DEF
	)

static

Definition at line 1018 of file kern_sx.c.

References __sx_can_read(), __sx_slock_try(), GIANT_DECLARE, GIANT_RESTORE, GIANT_SAVE, lock_delay(), lockstat_nsecs(), sched_tdname(), sleepq_add(), sleepq_lock(), sleepq_release(), sleepq_wait(), sleepq_wait_sig(), and SQ_SHARED_QUEUE.

Referenced by _sx_slock_int().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_slock_int()

int _sx_slock_int	(	struct sx *	sx,
		int opts	LOCK_FILE_LINE_ARG_DEF
	)

Definition at line 1254 of file kern_sx.c.

References __sx_slock_try(), _sx_slock_hard(), and kdb_active.

Referenced by _sx_slock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_sunlock()

void _sx_sunlock	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 1388 of file kern_sx.c.

References _sx_sunlock_int().

Here is the call graph for this function:

◆ _sx_sunlock_hard()

static void __noinline _sx_sunlock_hard	(	struct sx *	sx,
		struct thread *	td,
		uintptr_t x	LOCK_FILE_LINE_ARG_DEF
	)

static

Definition at line 1316 of file kern_sx.c.

References _sx_sunlock_try(), sleepq_broadcast(), sleepq_lock(), sleepq_release(), SQ_EXCLUSIVE_QUEUE, and SQ_SHARED_QUEUE.

Referenced by _sx_sunlock_int().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_sunlock_int()

void _sx_sunlock_int ( struct sx *sx LOCK_FILE_LINE_ARG_DEF )

Definition at line 1365 of file kern_sx.c.

References _sx_assert, _sx_sunlock_hard(), and _sx_sunlock_try().

Referenced by _sx_sunlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_sunlock_try()

static bool __always_inline _sx_sunlock_try	(	struct sx *	sx,
		struct thread *	td,
		uintptr_t *	xp
	)

static

Definition at line 1293 of file kern_sx.c.

Referenced by _sx_sunlock_hard(), and _sx_sunlock_int().

Here is the caller graph for this function:

◆ _sx_xlock()

int _sx_xlock	(	struct sx *	sx,
		int	opts,
		const char *	file,
		int	line
	)

Definition at line 316 of file kern_sx.c.

References _sx_xlock_hard(), and kdb_active.

Here is the call graph for this function:

◆ _sx_xlock_hard()

int _sx_xlock_hard	(	struct sx *	sx,
		uintptr_t	x,
		int opts	LOCK_FILE_LINE_ARG_DEF
	)

Definition at line 571 of file kern_sx.c.

References GIANT_DECLARE, GIANT_RESTORE, GIANT_SAVE, lock_delay(), lockstat_nsecs(), sched_tdname(), sleepq_add(), sleepq_lock(), sleepq_release(), sleepq_wait(), sleepq_wait_sig(), SQ_EXCLUSIVE_QUEUE, and sx_drop_critical.

Referenced by _sx_xlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _sx_xunlock()

void _sx_xunlock	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 403 of file kern_sx.c.

References _sx_assert, and _sx_xunlock_hard().

Here is the call graph for this function:

◆ _sx_xunlock_hard()

void _sx_xunlock_hard	(	struct sx *	sx,
		uintptr_t x	LOCK_FILE_LINE_ARG_DEF
	)

Definition at line 913 of file kern_sx.c.

References sleepq_broadcast(), sleepq_lock(), sleepq_release(), sleepq_sleepcnt(), SQ_EXCLUSIVE_QUEUE, and SQ_SHARED_QUEUE.

Referenced by _sx_xunlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ assert_sx()

void assert_sx	(	const struct lock_object *	lock,
		int	what
	)

static

Definition at line 178 of file kern_sx.c.

◆ lock_sx()

void lock_sx	(	struct lock_object *	lock,
		uintptr_t	how
	)

static

Definition at line 185 of file kern_sx.c.

◆ sx_destroy()

void sx_destroy ( struct sx * sx )

Definition at line 266 of file kern_sx.c.

References lock_destroy().

Referenced by epoch_free(), ksem_module_destroy(), lf_advlockasync(), lf_purgelocks(), mqfs_uninit(), rm_destroy(), shmunload(), and sodealloc().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sx_downgrade_()

void sx_downgrade_	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 541 of file kern_sx.c.

References sx_downgrade_int().

Here is the call graph for this function:

◆ sx_downgrade_int()

void sx_downgrade_int ( struct sx *sx LOCK_FILE_LINE_ARG_DEF )

Definition at line 477 of file kern_sx.c.

References _sx_assert, panic(), sleepq_broadcast(), sleepq_lock(), sleepq_release(), SQ_SHARED_QUEUE, and sx_recursed.

Referenced by sx_downgrade_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sx_init_flags()

void sx_init_flags	(	struct sx *	sx,
		const char *	description,
		int	opts
	)

Definition at line 236 of file kern_sx.c.

References flags, lock_class_sx, and lock_init().

Referenced by procinit(), rm_init_flags(), and sx_sysinit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sx_sysinit()

void sx_sysinit ( void * arg )

Definition at line 228 of file kern_sx.c.

References sx_init_flags().

Here is the call graph for this function:

◆ sx_try_slock_()

int sx_try_slock_	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 309 of file kern_sx.c.

References sx_try_slock_int().

Here is the call graph for this function:

◆ sx_try_slock_int()

int sx_try_slock_int ( struct sx *sx LOCK_FILE_LINE_ARG_DEF )

Definition at line 276 of file kern_sx.c.

References kdb_active.

Referenced by sx_try_slock_().

Here is the caller graph for this function:

◆ sx_try_upgrade_()

int sx_try_upgrade_	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 467 of file kern_sx.c.

References sx_try_upgrade_int().

Here is the call graph for this function:

◆ sx_try_upgrade_int()

int sx_try_upgrade_int ( struct sx *sx LOCK_FILE_LINE_ARG_DEF )

Definition at line 426 of file kern_sx.c.

References _sx_assert.

Referenced by sx_try_upgrade_().

Here is the caller graph for this function:

◆ sx_try_xlock_()

int sx_try_xlock_	(	struct sx *	sx,
		const char *	file,
		int	line
	)

Definition at line 396 of file kern_sx.c.

References sx_try_xlock_int().

Here is the call graph for this function:

◆ sx_try_xlock_int()

int sx_try_xlock_int ( struct sx *sx LOCK_FILE_LINE_ARG_DEF )

Definition at line 347 of file kern_sx.c.

References kdb_active.

Referenced by sx_try_xlock_().

Here is the caller graph for this function:

◆ unlock_sx()

uintptr_t unlock_sx ( struct lock_object * lock )

static

Definition at line 197 of file kern_sx.c.

Variable Documentation

◆ lock_class_sx

struct lock_class lock_class_sx

Initial value:

= {
        .lc_name = "sx",
        .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE | LC_UPGRADABLE,
        .lc_assert = assert_sx,
 
        .lc_lock = lock_sx,
        .lc_unlock = unlock_sx,
 
}

Definition at line 127 of file kern_sx.c.

Referenced by sx_init_flags().

Macros

Functions

Variables

Macro Definition Documentation

◆ _sx_assert

◆ GIANT_DECLARE

◆ GIANT_RESTORE

◆ GIANT_SAVE

◆ SQ_EXCLUSIVE_QUEUE

◆ SQ_SHARED_QUEUE

◆ sx_drop_critical

◆ sx_recursed

Function Documentation

◆ __FBSDID()

◆ __sx_can_read()

◆ __sx_slock_try()

◆ _sx_slock()

◆ _sx_slock_hard()

◆ _sx_slock_int()

◆ _sx_sunlock()

◆ _sx_sunlock_hard()

◆ _sx_sunlock_int()

◆ _sx_sunlock_try()

◆ _sx_xlock()

◆ _sx_xlock_hard()

◆ _sx_xunlock()

◆ _sx_xunlock_hard()

◆ assert_sx()

◆ lock_sx()

◆ sx_destroy()

◆ sx_downgrade_()

◆ sx_downgrade_int()

◆ sx_init_flags()

◆ sx_sysinit()

◆ sx_try_slock_()

◆ sx_try_slock_int()

◆ sx_try_upgrade_()

◆ sx_try_upgrade_int()

◆ sx_try_xlock_()

◆ sx_try_xlock_int()

◆ unlock_sx()

Variable Documentation

◆ lock_class_sx