kern_rwlock.c File Reference

#include <sys/cdefs.h>
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_no_adaptive_rwlocks.h"
#include <sys/param.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/turnstile.h>
#include <machine/cpu.h>

Include dependency graph for kern_rwlock.c:

Go to the source code of this file.

Macros
#define	rwlock2rw(c)   (__containerof(c, struct rwlock, rw_lock))
#define	lv_rw_wowner(v)
#define	rw_wowner(rw)   lv_rw_wowner(RW_READ_VALUE(rw))
#define	rw_recursed(rw)   ((rw)->rw_recurse != 0)
#define	rw_wlocked(rw)   (rw_wowner((rw)) == curthread)
#define	rw_owner(rw)   rw_wowner(rw)
#define	__rw_assert(c, what, file, line)
#define	rw_drop_critical(v, in_critical, extra_work)   do { } while (0)

Functions
	__FBSDID ("$FreeBSD$")
static void	assert_rw (const struct lock_object *lock, int what)
static void	lock_rw (struct lock_object *lock, uintptr_t how)
static uintptr_t	unlock_rw (struct lock_object *lock)
void	_rw_init_flags (volatile uintptr_t *c, const char *name, int opts)
void	_rw_destroy (volatile uintptr_t *c)
void	rw_sysinit (void *arg)
int	_rw_wowned (const volatile uintptr_t *c)
void	_rw_wlock_cookie (volatile uintptr_t *c, const char *file, int line)
int	__rw_try_wlock_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
int	__rw_try_wlock (volatile uintptr_t *c, const char *file, int line)
void	_rw_wunlock_cookie (volatile uintptr_t *c, const char *file, int line)
static bool __always_inline	__rw_can_read (struct thread *td, uintptr_t v, bool fp)
static bool __always_inline	__rw_rlock_try (struct rwlock *rw, struct thread *td, uintptr_t *vp, bool fp LOCK_FILE_LINE_ARG_DEF)
static void __noinline	__rw_rlock_hard (struct rwlock *rw, struct thread *td, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
void	__rw_rlock_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
void	__rw_rlock (volatile uintptr_t *c, const char *file, int line)
int	__rw_try_rlock_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
int	__rw_try_rlock (volatile uintptr_t *c, const char *file, int line)
static bool __always_inline	__rw_runlock_try (struct rwlock *rw, struct thread *td, uintptr_t *vp)
static void __noinline	__rw_runlock_hard (struct rwlock *rw, struct thread *td, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
void	_rw_runlock_cookie_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
void	_rw_runlock_cookie (volatile uintptr_t *c, const char *file, int line)
void	__rw_wlock_hard (volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
void	__rw_wunlock_hard (volatile uintptr_t *c, uintptr_t v LOCK_FILE_LINE_ARG_DEF)
int	__rw_try_upgrade_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
int	__rw_try_upgrade (volatile uintptr_t *c, const char *file, int line)
void	__rw_downgrade_int (struct rwlock *rw LOCK_FILE_LINE_ARG_DEF)
void	__rw_downgrade (volatile uintptr_t *c, const char *file, int line)

Variables
struct lock_class	lock_class_rw

Macro Definition Documentation

__rw_assert

#define __rw_assert	(		c,
			what,
			file,
			line
	)

Definition at line 159 of file kern_rwlock.c.

lv_rw_wowner

#define lv_rw_wowner

(

v

)

Value:

        ((v) & RW_LOCK_READ ? NULL :                                    \
         (struct thread *)RW_OWNER((v)))

Definition at line 134 of file kern_rwlock.c.

rw_drop_critical

#define rw_drop_critical	(		v,
			in_critical,
			extra_work
	)		do { } while (0)

Definition at line 887 of file kern_rwlock.c.

rw_owner

#define rw_owner

(

rw

)

rw_wowner(rw)

Definition at line 156 of file kern_rwlock.c.

rw_recursed

#define rw_recursed

(

rw

)

((rw)->rw_recurse != 0)

Definition at line 144 of file kern_rwlock.c.

rw_wlocked

#define rw_wlocked

(

rw

)

(rw_wowner((rw)) == curthread)

Definition at line 149 of file kern_rwlock.c.

rw_wowner

#define rw_wowner

(

rw

)

lv_rw_wowner(RW_READ_VALUE(rw))

Definition at line 138 of file kern_rwlock.c.

rwlock2rw

#define rwlock2rw

(

c

)

(__containerof(c, struct rwlock, rw_lock))

Definition at line 68 of file kern_rwlock.c.

Function Documentation

__FBSDID()

__FBSDID

(

"$FreeBSD$"

)

__rw_can_read()

static bool __always_inline __rw_can_read ( struct thread *  td, uintptr_t  v, bool  fp  )

static

Definition at line 390 of file kern_rwlock.c.

Referenced by __rw_rlock_hard(), and __rw_rlock_try().

Here is the caller graph for this function:

__rw_downgrade()

void __rw_downgrade	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 1426 of file kern_rwlock.c.

References __rw_downgrade_int(), and rwlock2rw.

Here is the call graph for this function:

__rw_downgrade_int()

void __rw_downgrade_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 1362 of file kern_rwlock.c.

References __rw_assert, panic(), rw_recursed, ts, turnstile_broadcast(), turnstile_chain_lock(), turnstile_chain_unlock(), turnstile_disown(), turnstile_lookup(), and turnstile_unpend().

Referenced by __rw_downgrade().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_rlock()

void __rw_rlock	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 693 of file kern_rwlock.c.

References __rw_rlock_int(), and rwlock2rw.

Here is the call graph for this function:

__rw_rlock_hard()

static void __noinline __rw_rlock_hard ( struct rwlock *  rw, struct thread *  td, uintptr_t v  LOCK_FILE_LINE_ARG_DEF  )

static

Definition at line 432 of file kern_rwlock.c.

References __rw_can_read(), __rw_rlock_try(), lock_delay(), lockstat_nsecs(), lv_rw_wowner, rw_owner, sched_tdname(), ts, turnstile_cancel(), turnstile_trywait(), and turnstile_wait().

Referenced by __rw_rlock_int().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_rlock_int()

void __rw_rlock_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 661 of file kern_rwlock.c.

References __rw_rlock_hard(), __rw_rlock_try(), kdb_active, and rw_wowner.

Referenced by __rw_rlock().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_rlock_try()

static bool __always_inline __rw_rlock_try ( struct rwlock *  rw, struct thread *  td, uintptr_t *  vp, bool fp  LOCK_FILE_LINE_ARG_DEF  )

static

Definition at line 402 of file kern_rwlock.c.

References __rw_can_read().

Referenced by __rw_rlock_hard(), and __rw_rlock_int().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_runlock_hard()

static void __noinline __rw_runlock_hard ( struct rwlock *  rw, struct thread *  td, uintptr_t v  LOCK_FILE_LINE_ARG_DEF  )

static

Definition at line 768 of file kern_rwlock.c.

References __rw_runlock_try(), ts, turnstile_broadcast(), turnstile_chain_lock(), turnstile_chain_unlock(), turnstile_lookup(), and turnstile_unpend().

Referenced by _rw_runlock_cookie_int().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_runlock_try()

static bool __always_inline __rw_runlock_try ( struct rwlock *  rw, struct thread *  td, uintptr_t *  vp  )

static

Definition at line 745 of file kern_rwlock.c.

Referenced by __rw_runlock_hard(), and _rw_runlock_cookie_int().

Here is the caller graph for this function:

__rw_try_rlock()

int __rw_try_rlock	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 736 of file kern_rwlock.c.

References __rw_try_rlock_int(), and rwlock2rw.

Here is the call graph for this function:

__rw_try_rlock_int()

int __rw_try_rlock_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 702 of file kern_rwlock.c.

References kdb_active.

Referenced by __rw_try_rlock().

Here is the caller graph for this function:

__rw_try_upgrade()

int __rw_try_upgrade	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 1350 of file kern_rwlock.c.

References __rw_try_upgrade_int(), and rwlock2rw.

Here is the call graph for this function:

__rw_try_upgrade_int()

int __rw_try_upgrade_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 1279 of file kern_rwlock.c.

References __rw_assert, ts, turnstile_cancel(), turnstile_claim(), and turnstile_trywait().

Referenced by __rw_try_upgrade().

Here is the call graph for this function:

Here is the caller graph for this function:

__rw_try_wlock()

int __rw_try_wlock	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 351 of file kern_rwlock.c.

References __rw_try_wlock_int(), and rwlock2rw.

Here is the call graph for this function:

__rw_try_wlock_int()

int __rw_try_wlock_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 303 of file kern_rwlock.c.

References kdb_active.

Referenced by __rw_try_wlock().

Here is the caller graph for this function:

__rw_wlock_hard()

void __rw_wlock_hard	(	volatile uintptr_t *	c,
		uintptr_t v	LOCK_FILE_LINE_ARG_DEF
	)

Definition at line 896 of file kern_rwlock.c.

References lock_delay(), lockstat_nsecs(), lv_rw_wowner, rw_drop_critical, rw_owner, rwlock2rw, sched_tdname(), ts, turnstile_cancel(), turnstile_claim(), turnstile_trywait(), and turnstile_wait().

Here is the call graph for this function:

__rw_wunlock_hard()

void __rw_wunlock_hard	(	volatile uintptr_t *	c,
		uintptr_t v	LOCK_FILE_LINE_ARG_DEF
	)

Definition at line 1201 of file kern_rwlock.c.

References rwlock2rw, ts, turnstile_broadcast(), turnstile_chain_lock(), turnstile_chain_unlock(), turnstile_lookup(), and turnstile_unpend().

Here is the call graph for this function:

_rw_destroy()

void _rw_destroy

(

volatile uintptr_t *

c

)

Definition at line 244 of file kern_rwlock.c.

References lock_destroy(), and rwlock2rw.

Here is the call graph for this function:

_rw_init_flags()

void _rw_init_flags	(	volatile uintptr_t *	c,
		const char *	name,
		int	opts
	)

Definition at line 211 of file kern_rwlock.c.

References flags, lock_class_rw, lock_init(), name, and rwlock2rw.

Here is the call graph for this function:

_rw_runlock_cookie()

void _rw_runlock_cookie	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 865 of file kern_rwlock.c.

References _rw_runlock_cookie_int(), and rwlock2rw.

Here is the call graph for this function:

_rw_runlock_cookie_int()

void _rw_runlock_cookie_int

(

struct rwlock *rw

LOCK_FILE_LINE_ARG_DEF

)

Definition at line 841 of file kern_rwlock.c.

References __rw_assert, __rw_runlock_hard(), and __rw_runlock_try().

Referenced by _rw_runlock_cookie().

Here is the call graph for this function:

Here is the caller graph for this function:

_rw_wlock_cookie()

void _rw_wlock_cookie	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 274 of file kern_rwlock.c.

References kdb_active, and rwlock2rw.

_rw_wowned()

int _rw_wowned

(

const volatile uintptr_t *

c

)

Definition at line 267 of file kern_rwlock.c.

References rw_wowner, and rwlock2rw.

_rw_wunlock_cookie()

void _rw_wunlock_cookie	(	volatile uintptr_t *	c,
		const char *	file,
		int	line
	)

Definition at line 360 of file kern_rwlock.c.

References __rw_assert, and rwlock2rw.

assert_rw()

void assert_rw ( const struct lock_object *  lock, int  what  )

static

Definition at line 163 of file kern_rwlock.c.

lock_rw()

void lock_rw ( struct lock_object *  lock, uintptr_t  how  )

static

Definition at line 170 of file kern_rwlock.c.

rw_sysinit()

void rw_sysinit

(

void *

arg

)

Definition at line 257 of file kern_rwlock.c.

unlock_rw()

uintptr_t unlock_rw ( struct lock_object *  lock )

static

Definition at line 182 of file kern_rwlock.c.

Variable Documentation

lock_class_rw

struct lock_class lock_class_rw

Initial value:

= {
        .lc_name = "rw",
        .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE | LC_UPGRADABLE,
        .lc_assert = assert_rw,
 
 
 
        .lc_lock = lock_rw,
        .lc_unlock = unlock_rw,
 
 
 
}

Definition at line 82 of file kern_rwlock.c.

Referenced by _rw_init_flags().