FreeBSD kernel kern code
|
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/lock_profile.h>
#include <sys/lockmgr.h>
#include <sys/lockstat.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sleepqueue.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <machine/cpu.h>
Go to the source code of this file.
Data Structures | |
struct | lockmgr_wait |
Macros | |
#define | SQ_EXCLUSIVE_QUEUE 0 |
#define | SQ_SHARED_QUEUE 1 |
#define | _lockmgr_assert(lk, what, file, line) |
#define | TD_SLOCKS_INC(td) ((td)->td_lk_slocks++) |
#define | TD_SLOCKS_DEC(td) ((td)->td_lk_slocks--) |
#define | STACK_PRINT(lk) |
#define | STACK_SAVE(lk) |
#define | STACK_ZERO(lk) |
#define | LOCK_LOG2(lk, string, arg1, arg2) |
#define | LOCK_LOG3(lk, string, arg1, arg2, arg3) |
#define | GIANT_DECLARE |
#define | GIANT_RESTORE() |
#define | GIANT_SAVE() |
#define | LK_TRYOP(x) ((x) & LK_NOWAIT) |
#define | LK_CAN_WITNESS(x) (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x)) |
#define | LK_TRYWIT(x) (LK_TRYOP(x) ? LOP_TRYLOCK : 0) |
#define | lockmgr_disowned(lk) (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) |
#define | lockmgr_xlocked_v(v) (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) |
#define | lockmgr_xlocked(lk) lockmgr_xlocked_v(lockmgr_read_value(lk)) |
#define | lockmgr_delay locks_delay |
Functions | |
__FBSDID ("$FreeBSD$") | |
_Static_assert ((PRILASTFLAG *2) - 1<=USHRT_MAX, "prio flags wont fit in u_short pri in struct lock") | |
CTASSERT (LK_UNLOCKED==(LK_UNLOCKED &~(LK_ALL_WAITERS|LK_EXCLUSIVE_SPINNERS))) | |
static bool __always_inline | LK_CAN_SHARE (uintptr_t x, int flags, bool fp) |
static void | assert_lockmgr (const struct lock_object *lock, int how) |
static void | lock_lockmgr (struct lock_object *lock, uintptr_t how) |
static uintptr_t | unlock_lockmgr (struct lock_object *lock) |
static | SYSCTL_NODE (_debug, OID_AUTO, lockmgr, CTLFLAG_RD, NULL, "lockmgr debugging") |
SYSCTL_BOOL (_debug_lockmgr, OID_AUTO, adaptive_spinning, CTLFLAG_RW, &lk_adaptive, 0, "") | |
static bool __always_inline | lockmgr_slock_try (struct lock *lk, uintptr_t *xp, int flags, bool fp) |
static bool __always_inline | lockmgr_sunlock_try (struct lock *lk, uintptr_t *xp) |
static void | lockmgr_exit (u_int flags, struct lock_object *ilk, int wakeup_swapper) |
static void | lockmgr_note_shared_acquire (struct lock *lk, int contested, uint64_t waittime, const char *file, int line, int flags) |
static void | lockmgr_note_shared_release (struct lock *lk, const char *file, int line) |
static void | lockmgr_note_exclusive_acquire (struct lock *lk, int contested, uint64_t waittime, const char *file, int line, int flags) |
static void | lockmgr_note_exclusive_release (struct lock *lk, const char *file, int line) |
static __inline struct thread * | lockmgr_xholder (const struct lock *lk) |
static __inline int | sleeplk (struct lock *lk, u_int flags, struct lock_object *ilk, const char *wmesg, int pri, int timo, int queue) |
static __inline int | wakeupshlk (struct lock *lk, const char *file, int line) |
void | lockinit (struct lock *lk, int pri, const char *wmesg, int timo, int flags) |
void | lockallowshare (struct lock *lk) |
void | lockdisableshare (struct lock *lk) |
void | lockallowrecurse (struct lock *lk) |
void | lockdisablerecurse (struct lock *lk) |
void | lockdestroy (struct lock *lk) |
static bool | lockmgr_slock_adaptive (struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp, int flags) |
static __noinline int | lockmgr_slock_hard (struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) |
static bool | lockmgr_xlock_adaptive (struct lock_delay_arg *lda, struct lock *lk, uintptr_t *xp) |
static __noinline int | lockmgr_xlock_hard (struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) |
static __noinline int | lockmgr_upgrade (struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line, struct lockmgr_wait *lwa) |
int | lockmgr_lock_flags (struct lock *lk, u_int flags, struct lock_object *ilk, const char *file, int line) |
static __noinline int | lockmgr_sunlock_hard (struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, const char *file, int line) |
static __noinline int | lockmgr_xunlock_hard (struct lock *lk, uintptr_t x, u_int flags, struct lock_object *ilk, const char *file, int line) |
int | lockmgr_slock (struct lock *lk, u_int flags, const char *file, int line) |
int | lockmgr_xlock (struct lock *lk, u_int flags, const char *file, int line) |
int | lockmgr_unlock (struct lock *lk) |
int | __lockmgr_args (struct lock *lk, u_int flags, struct lock_object *ilk, const char *wmesg, int pri, int timo, const char *file, int line) |
void | _lockmgr_disown (struct lock *lk, const char *file, int line) |
void | lockmgr_printinfo (const struct lock *lk) |
int | lockstatus (const struct lock *lk) |
Variables | |
struct lock_class | lock_class_lockmgr |
static __read_mostly bool | lk_adaptive = true |
#define _lockmgr_assert | ( | lk, | |
what, | |||
file, | |||
line | |||
) |
Definition at line 78 of file kern_lock.c.
#define GIANT_DECLARE |
Definition at line 101 of file kern_lock.c.
#define GIANT_RESTORE | ( | ) |
Definition at line 104 of file kern_lock.c.
#define GIANT_SAVE | ( | ) |
#define LK_CAN_WITNESS | ( | x | ) | (((x) & LK_NOWITNESS) == 0 && !LK_TRYOP(x)) |
Definition at line 139 of file kern_lock.c.
#define LK_TRYOP | ( | x | ) | ((x) & LK_NOWAIT) |
Definition at line 136 of file kern_lock.c.
#define LK_TRYWIT | ( | x | ) | (LK_TRYOP(x) ? LOP_TRYLOCK : 0) |
Definition at line 141 of file kern_lock.c.
#define LOCK_LOG2 | ( | lk, | |
string, | |||
arg1, | |||
arg2 | |||
) |
Definition at line 94 of file kern_lock.c.
#define LOCK_LOG3 | ( | lk, | |
string, | |||
arg1, | |||
arg2, | |||
arg3 | |||
) |
Definition at line 97 of file kern_lock.c.
#define lockmgr_delay locks_delay |
Definition at line 181 of file kern_lock.c.
#define lockmgr_disowned | ( | lk | ) | (((lk)->lk_lock & ~(LK_FLAGMASK & ~LK_SHARE)) == LK_KERNPROC) |
Definition at line 144 of file kern_lock.c.
#define lockmgr_xlocked | ( | lk | ) | lockmgr_xlocked_v(lockmgr_read_value(lk)) |
Definition at line 150 of file kern_lock.c.
#define lockmgr_xlocked_v | ( | v | ) | (((v) & ~(LK_FLAGMASK & ~LK_SHARE)) == (uintptr_t)curthread) |
Definition at line 147 of file kern_lock.c.
#define SQ_EXCLUSIVE_QUEUE 0 |
Definition at line 74 of file kern_lock.c.
#define SQ_SHARED_QUEUE 1 |
Definition at line 75 of file kern_lock.c.
#define STACK_PRINT | ( | lk | ) |
Definition at line 85 of file kern_lock.c.
#define STACK_SAVE | ( | lk | ) |
Definition at line 86 of file kern_lock.c.
#define STACK_ZERO | ( | lk | ) |
Definition at line 87 of file kern_lock.c.
#define TD_SLOCKS_DEC | ( | td | ) | ((td)->td_lk_slocks--) |
Definition at line 82 of file kern_lock.c.
#define TD_SLOCKS_INC | ( | td | ) | ((td)->td_lk_slocks++) |
Definition at line 81 of file kern_lock.c.
__FBSDID | ( | "$FreeBSD$" | ) |
int __lockmgr_args | ( | struct lock * | lk, |
u_int | flags, | ||
struct lock_object * | ilk, | ||
const char * | wmesg, | ||
int | pri, | ||
int | timo, | ||
const char * | file, | ||
int | line | ||
) |
Definition at line 1308 of file kern_lock.c.
References _lockmgr_assert, flags, GIANT_DECLARE, GIANT_RESTORE, GIANT_SAVE, lockmgr_wait::ipri, lockmgr_wait::itimo, lockmgr_wait::iwmesg, kdb_active, LK_CAN_WITNESS, LK_TRYOP, LK_TRYWIT, LOCK_LOG2, LOCK_LOG3, lockmgr_note_exclusive_release(), lockmgr_note_shared_release(), lockmgr_slock_hard(), lockmgr_sunlock_hard(), lockmgr_upgrade(), lockmgr_xlock_hard(), lockmgr_xlocked, lockmgr_xunlock_hard(), panic(), sleepq_add(), sleepq_broadcast(), sleepq_lock(), sleepq_release(), sleepq_sleepcnt(), sleepq_wait(), SQ_EXCLUSIVE_QUEUE, SQ_SHARED_QUEUE, STACK_SAVE, and TD_SLOCKS_INC.
Referenced by lockmgr_lock_flags().
void _lockmgr_disown | ( | struct lock * | lk, |
const char * | file, | ||
int | line | ||
) |
Definition at line 1619 of file kern_lock.c.
References _lockmgr_assert, lockmgr_xlocked, panic(), and STACK_SAVE.
_Static_assert | ( | (PRILASTFLAG *2) - 1<= | USHRT_MAX, |
"prio flags wont fit in u_short pri in struct lock" | |||
) |
|
static |
Definition at line 409 of file kern_lock.c.
References panic().
CTASSERT | ( | LK_UNLOCKED | = =(LK_UNLOCKED &~(LK_ALL_WAITERS|LK_EXCLUSIVE_SPINNERS)) | ) |
|
static |
Definition at line 122 of file kern_lock.c.
References flags.
Referenced by lockmgr_slock_adaptive(), lockmgr_slock_hard(), and lockmgr_slock_try().
|
static |
Definition at line 416 of file kern_lock.c.
References panic().
void lockallowrecurse | ( | struct lock * | lk | ) |
Definition at line 495 of file kern_lock.c.
void lockallowshare | ( | struct lock * | lk | ) |
Definition at line 479 of file kern_lock.c.
void lockdestroy | ( | struct lock * | lk | ) |
Definition at line 511 of file kern_lock.c.
References lock_destroy().
Referenced by mount_fini(), and vnode_fini().
void lockdisablerecurse | ( | struct lock * | lk | ) |
Definition at line 503 of file kern_lock.c.
void lockdisableshare | ( | struct lock * | lk | ) |
Definition at line 487 of file kern_lock.c.
void lockinit | ( | struct lock * | lk, |
int | pri, | ||
const char * | wmesg, | ||
int | timo, | ||
int | flags | ||
) |
Definition at line 439 of file kern_lock.c.
References flags, lock_class_lockmgr, lock_init(), and STACK_ZERO.
Referenced by mount_init(), and vnode_init().
|
static |
Definition at line 194 of file kern_lock.c.
References flags.
Referenced by lockmgr_slock_hard(), lockmgr_sunlock_hard(), lockmgr_upgrade(), lockmgr_xlock_hard(), and lockmgr_xunlock_hard().
int lockmgr_lock_flags | ( | struct lock * | lk, |
u_int | flags, | ||
struct lock_object * | ilk, | ||
const char * | file, | ||
int | line | ||
) |
Definition at line 1041 of file kern_lock.c.
References __lockmgr_args(), flags, LK_CAN_WITNESS, lockmgr_note_exclusive_acquire(), lockmgr_note_shared_acquire(), lockmgr_slock_hard(), lockmgr_slock_try(), lockmgr_upgrade(), and lockmgr_xlock_hard().
Referenced by vop_lock(), and vop_stdlock().
|
static |
Definition at line 232 of file kern_lock.c.
References flags, LK_TRYWIT, and STACK_SAVE.
Referenced by lockmgr_lock_flags(), lockmgr_xlock(), and lockmgr_xlock_hard().
|
static |
Definition at line 246 of file kern_lock.c.
Referenced by __lockmgr_args(), and lockmgr_unlock().
|
static |
Definition at line 208 of file kern_lock.c.
References flags, LK_TRYWIT, STACK_SAVE, and TD_SLOCKS_INC.
Referenced by lockmgr_lock_flags(), lockmgr_slock(), and lockmgr_slock_hard().
|
static |
Definition at line 222 of file kern_lock.c.
References TD_SLOCKS_DEC.
Referenced by __lockmgr_args(), lockmgr_unlock(), and lockmgr_upgrade().
void lockmgr_printinfo | ( | const struct lock * | lk | ) |
Definition at line 1663 of file kern_lock.c.
References lockmgr_xholder(), printf(), and STACK_PRINT.
Referenced by vn_printf().
int lockmgr_slock | ( | struct lock * | lk, |
u_int | flags, | ||
const char * | file, | ||
int | line | ||
) |
Definition at line 1235 of file kern_lock.c.
References flags, LK_CAN_WITNESS, lockmgr_note_shared_acquire(), lockmgr_slock_hard(), and lockmgr_slock_try().
Referenced by vop_lock().
|
static |
Definition at line 557 of file kern_lock.c.
References flags, LK_CAN_SHARE(), and lock_delay().
Referenced by lockmgr_slock_hard().
|
static |
Definition at line 589 of file kern_lock.c.
References flags, lockmgr_wait::ipri, lockmgr_wait::itimo, lockmgr_wait::iwmesg, lk_adaptive, LK_CAN_SHARE(), LK_CAN_WITNESS, LK_TRYOP, LOCK_LOG2, LOCK_LOG3, lockmgr_delay, lockmgr_exit(), lockmgr_note_shared_acquire(), lockmgr_slock_adaptive(), lockmgr_slock_try(), lockstat_nsecs(), sleeplk(), sleepq_lock(), sleepq_release(), and SQ_SHARED_QUEUE.
Referenced by __lockmgr_args(), lockmgr_lock_flags(), and lockmgr_slock().
|
static |
Definition at line 521 of file kern_lock.c.
References flags, and LK_CAN_SHARE().
Referenced by lockmgr_lock_flags(), lockmgr_slock(), and lockmgr_slock_hard().
|
static |
Definition at line 1106 of file kern_lock.c.
References flags, lockmgr_exit(), and wakeupshlk().
Referenced by __lockmgr_args(), and lockmgr_unlock().
|
static |
Definition at line 541 of file kern_lock.c.
Referenced by lockmgr_unlock(), and wakeupshlk().
int lockmgr_unlock | ( | struct lock * | lk | ) |
Definition at line 1277 of file kern_lock.c.
References _lockmgr_assert, lockmgr_note_exclusive_release(), lockmgr_note_shared_release(), lockmgr_sunlock_hard(), lockmgr_sunlock_try(), and lockmgr_xunlock_hard().
Referenced by vop_stdunlock(), and vop_unlock().
|
static |
Definition at line 981 of file kern_lock.c.
References _lockmgr_assert, flags, LK_TRYWIT, LOCK_LOG2, lockmgr_exit(), lockmgr_note_shared_release(), lockmgr_xlock_hard(), and TD_SLOCKS_DEC.
Referenced by __lockmgr_args(), and lockmgr_lock_flags().
|
static |
Definition at line 258 of file kern_lock.c.
Referenced by lockmgr_printinfo().
int lockmgr_xlock | ( | struct lock * | lk, |
u_int | flags, | ||
const char * | file, | ||
int | line | ||
) |
Definition at line 1256 of file kern_lock.c.
References flags, LK_CAN_WITNESS, lockmgr_note_exclusive_acquire(), and lockmgr_xlock_hard().
Referenced by vop_lock().
|
static |
Definition at line 743 of file kern_lock.c.
References lock_delay().
Referenced by lockmgr_xlock_hard().
|
static |
Definition at line 774 of file kern_lock.c.
References flags, lockmgr_wait::ipri, lockmgr_wait::itimo, lockmgr_wait::iwmesg, lk_adaptive, LK_CAN_WITNESS, LK_TRYOP, LK_TRYWIT, LOCK_LOG2, LOCK_LOG3, lockmgr_delay, lockmgr_exit(), lockmgr_note_exclusive_acquire(), lockmgr_xlock_adaptive(), lockmgr_xlocked, lockstat_nsecs(), panic(), sleeplk(), sleepq_lock(), sleepq_release(), SQ_EXCLUSIVE_QUEUE, and STACK_PRINT.
Referenced by __lockmgr_args(), lockmgr_lock_flags(), lockmgr_upgrade(), and lockmgr_xlock().
|
static |
Definition at line 1123 of file kern_lock.c.
References flags, LOCK_LOG2, LOCK_LOG3, lockmgr_exit(), sleepq_broadcast(), sleepq_lock(), sleepq_release(), sleepq_sleepcnt(), SQ_EXCLUSIVE_QUEUE, and SQ_SHARED_QUEUE.
Referenced by __lockmgr_args(), and lockmgr_unlock().
int lockstatus | ( | const struct lock * | lk | ) |
Definition at line 1698 of file kern_lock.c.
Referenced by vop_islocked(), and vop_stdislocked().
|
static |
Definition at line 273 of file kern_lock.c.
References flags, GIANT_DECLARE, GIANT_RESTORE, GIANT_SAVE, LOCK_LOG3, sleepq_add(), sleepq_timedwait(), sleepq_timedwait_sig(), sleepq_wait(), sleepq_wait_sig(), and SQ_EXCLUSIVE_QUEUE.
Referenced by lockmgr_slock_hard(), and lockmgr_xlock_hard().
SYSCTL_BOOL | ( | _debug_lockmgr | , |
OID_AUTO | , | ||
adaptive_spinning | , | ||
CTLFLAG_RW | , | ||
& | lk_adaptive, | ||
0 | , | ||
"" | |||
) |
|
static |
|
static |
Definition at line 423 of file kern_lock.c.
References panic().
|
static |
Definition at line 319 of file kern_lock.c.
References LOCK_LOG2, LOCK_LOG3, lockmgr_sunlock_try(), sleepq_broadcast(), sleepq_lock(), sleepq_release(), sleepq_sleepcnt(), SQ_EXCLUSIVE_QUEUE, and SQ_SHARED_QUEUE.
Referenced by lockmgr_sunlock_hard().
|
static |
Definition at line 177 of file kern_lock.c.
Referenced by lockmgr_slock_hard(), and lockmgr_xlock_hard().
struct lock_class lock_class_lockmgr |
Definition at line 163 of file kern_lock.c.
Referenced by lockinit().