#include <sys/cdefs.h>
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kdb.h>
#include <sys/ktr.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/rmlock.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/turnstile.h>
#include <sys/lock_profile.h>
#include <machine/cpu.h>
#include <vm/uma.h>

Include dependency graph for kern_rmlock.c:

Data Structures
struct	rmslock_pcpu

struct	rmslock_ipi

Macros
#define	RM_DESTROYED ((void *)0xdead)

#define	rm_destroyed(rm) (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)

#define	RMPF_ONQUEUE 1

#define	RMPF_SIGNAL 2

#define	_rm_assert(c, what, file, line)

#define	RMS_NOOWNER ((void *)0x1)

#define	RMS_TRANSIENT ((void *)0x2)

#define	RMS_FLAGMASK 0xf

Functions
	__FBSDID ("$FreeBSD$")

static void	assert_rm (const struct lock_object *lock, int what)

static void	lock_rm (struct lock_object *lock, uintptr_t how)

static uintptr_t	unlock_rm (struct lock_object *lock)

	MTX_SYSINIT (rm_spinlock, &rm_spinlock, "rm_spinlock", MTX_SPIN)

static void	rm_tracker_add (struct pcpu pc, struct rm_priotracker tracker)

static int	rm_trackers_present (const struct pcpu pc, const struct rmlock rm, const struct thread *td)

static void	rm_tracker_remove (struct pcpu pc, struct rm_priotracker tracker)

static void	rm_cleanIPI (void *arg)

void	rm_init_flags (struct rmlock rm, const char name, int opts)

void	rm_init (struct rmlock rm, const char name)

void	rm_destroy (struct rmlock *rm)

int	rm_wowned (const struct rmlock *rm)

void	rm_sysinit (void *arg)

static __noinline int	_rm_rlock_hard (struct rmlock rm, struct rm_priotracker tracker, int trylock)

int	_rm_rlock (struct rmlock rm, struct rm_priotracker tracker, int trylock)

static __noinline void	_rm_unlock_hard (struct thread td, struct rm_priotracker tracker)

void	_rm_runlock (struct rmlock rm, struct rm_priotracker tracker)

void	_rm_wlock (struct rmlock *rm)

void	_rm_wunlock (struct rmlock *rm)

void	_rm_wlock_debug (struct rmlock rm, const char file, int line)

void	_rm_wunlock_debug (struct rmlock rm, const char file, int line)

int	_rm_rlock_debug (struct rmlock rm, struct rm_priotracker tracker, int trylock, const char *file, int line)

void	_rm_runlock_debug (struct rmlock rm, struct rm_priotracker tracker, const char *file, int line)

	_Static_assert (sizeof(struct rmslock_pcpu)==8, "bad size")

static struct rmslock_pcpu *	rms_int_pcpu (struct rmslock *rms)

static struct rmslock_pcpu *	rms_int_remote_pcpu (struct rmslock *rms, int cpu)

static void	rms_int_influx_enter (struct rmslock rms, struct rmslock_pcpu pcpu)

static void	rms_int_influx_exit (struct rmslock rms, struct rmslock_pcpu pcpu)

static void	rms_int_debug_readers_inc (struct rmslock *rms)

static void	rms_int_debug_readers_dec (struct rmslock *rms)

static void	rms_int_readers_inc (struct rmslock rms, struct rmslock_pcpu pcpu)

static void	rms_int_readers_dec (struct rmslock rms, struct rmslock_pcpu pcpu)

void	rms_init (struct rmslock rms, const char name)

void	rms_destroy (struct rmslock *rms)

static void __noinline	rms_rlock_fallback (struct rmslock *rms)

void	rms_rlock (struct rmslock *rms)

int	rms_try_rlock (struct rmslock *rms)

static void __noinline	rms_runlock_fallback (struct rmslock *rms)

void	rms_runlock (struct rmslock *rms)

static void	rms_action_func (void *arg)

static void	rms_wait_func (void *arg, int cpu)

static void	rms_assert_no_pcpu_readers (struct rmslock *rms)

static void	rms_wlock_switch (struct rmslock *rms)

void	rms_wlock (struct rmslock *rms)

void	rms_wunlock (struct rmslock *rms)

void	rms_unlock (struct rmslock *rms)

Variables
struct lock_class	lock_class_rm

struct lock_class	lock_class_rm_sleepable

static struct mtx	rm_spinlock

Macro Definition Documentation

◆ _rm_assert

#define _rm_assert	(	c,
		what,
		file,
		line
	)

Definition at line 75 of file kern_rmlock.c.

◆ RM_DESTROYED

#define RM_DESTROYED ((void *)0xdead)

Definition at line 66 of file kern_rmlock.c.

◆ rm_destroyed

#define rm_destroyed ( rm ) (LIST_FIRST(&(rm)->rm_activeReaders) == RM_DESTROYED)

Definition at line 68 of file kern_rmlock.c.

◆ RMPF_ONQUEUE

#define RMPF_ONQUEUE 1

Definition at line 71 of file kern_rmlock.c.

◆ RMPF_SIGNAL

#define RMPF_SIGNAL 2

Definition at line 72 of file kern_rmlock.c.

◆ RMS_FLAGMASK

#define RMS_FLAGMASK 0xf

Definition at line 889 of file kern_rmlock.c.

◆ RMS_NOOWNER

#define RMS_NOOWNER ((void *)0x1)

Definition at line 887 of file kern_rmlock.c.

◆ RMS_TRANSIENT

#define RMS_TRANSIENT ((void *)0x2)

Definition at line 888 of file kern_rmlock.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _rm_rlock()

int _rm_rlock	(	struct rmlock *	rm,
		struct rm_priotracker *	tracker,
		int	trylock
	)

Definition at line 436 of file kern_rmlock.c.

References _rm_rlock_hard(), and rm_tracker_add().

Referenced by _rm_rlock_debug().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _rm_rlock_debug()

int _rm_rlock_debug	(	struct rmlock *	rm,
		struct rm_priotracker *	tracker,
		int	trylock,
		const char *	file,
		int	line
	)

Definition at line 720 of file kern_rmlock.c.

References _rm_rlock().

Here is the call graph for this function:

◆ _rm_rlock_hard()

static __noinline int _rm_rlock_hard	(	struct rmlock *	rm,
		struct rm_priotracker *	tracker,
		int	trylock
	)

static

Definition at line 351 of file kern_rmlock.c.

References rm_spinlock, rm_tracker_add(), rm_tracker_remove(), rm_trackers_present(), and RMPF_ONQUEUE.

Referenced by _rm_rlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _rm_runlock()

void _rm_runlock	(	struct rmlock *	rm,
		struct rm_priotracker *	tracker
	)

Definition at line 507 of file kern_rmlock.c.

References _rm_unlock_hard(), and rm_tracker_remove().

Referenced by _rm_runlock_debug().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _rm_runlock_debug()

void _rm_runlock_debug	(	struct rmlock *	rm,
		struct rm_priotracker *	tracker,
		const char *	file,
		int	line
	)

Definition at line 728 of file kern_rmlock.c.

References _rm_runlock().

Here is the call graph for this function:

◆ _rm_unlock_hard()

static __noinline void _rm_unlock_hard	(	struct thread *	td,
		struct rm_priotracker *	tracker
	)

static

Definition at line 474 of file kern_rmlock.c.

References rm_spinlock, RMPF_SIGNAL, ts, turnstile_chain_lock(), turnstile_chain_unlock(), turnstile_lookup(), turnstile_signal(), and turnstile_unpend().

Referenced by _rm_runlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _rm_wlock()

void _rm_wlock ( struct rmlock * rm )

Definition at line 535 of file kern_rmlock.c.

References all_cpus, rm_cleanIPI(), rm_spinlock, RMPF_ONQUEUE, RMPF_SIGNAL, smp_no_rendezvous_barrier(), smp_rendezvous_cpus(), ts, turnstile_trywait(), and turnstile_wait().

Referenced by _rm_wlock_debug().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _rm_wlock_debug()

void _rm_wlock_debug	(	struct rmlock *	rm,
		const char *	file,
		int	line
	)

Definition at line 706 of file kern_rmlock.c.

References _rm_wlock().

Here is the call graph for this function:

◆ _rm_wunlock()

void _rm_wunlock ( struct rmlock * rm )

Definition at line 584 of file kern_rmlock.c.

Referenced by _rm_wunlock_debug().

Here is the caller graph for this function:

◆ _rm_wunlock_debug()

void _rm_wunlock_debug	(	struct rmlock *	rm,
		const char *	file,
		int	line
	)

Definition at line 713 of file kern_rmlock.c.

References _rm_wunlock().

Here is the call graph for this function:

◆ _Static_assert()

_Static_assert	(	sizeof(struct rmslock_pcpu)	= `=8`,
		"bad size"
	)

◆ assert_rm()

static void assert_rm	(	const struct lock_object *	lock,
		int	what
	)

static

Definition at line 117 of file kern_rmlock.c.

◆ lock_rm()

static void lock_rm	(	struct lock_object *	lock,
		uintptr_t	how
	)

static

Definition at line 124 of file kern_rmlock.c.

◆ MTX_SYSINIT()

MTX_SYSINIT	(	rm_spinlock	,
		&	rm_spinlock,
		"rm_spinlock"	,
		MTX_SPIN
	)

◆ rm_cleanIPI()

static void rm_cleanIPI ( void * arg )

static

Definition at line 258 of file kern_rmlock.c.

References rm_spinlock, and RMPF_ONQUEUE.

Referenced by _rm_wlock().

Here is the caller graph for this function:

◆ rm_destroy()

void rm_destroy ( struct rmlock * rm )

Definition at line 319 of file kern_rmlock.c.

References lock_destroy(), RM_DESTROYED, and sx_destroy().

Here is the call graph for this function:

◆ rm_init()

void rm_init	(	struct rmlock *	rm,
		const char *	name
	)

Definition at line 312 of file kern_rmlock.c.

References name, and rm_init_flags().

Referenced by osd_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rm_init_flags()

void rm_init_flags	(	struct rmlock *	rm,
		const char *	name,
		int	opts
	)

Definition at line 280 of file kern_rmlock.c.

References all_cpus, lc, lock_class_rm, lock_class_rm_sleepable, lock_init(), name, and sx_init_flags().

Referenced by rm_init(), and rm_sysinit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rm_sysinit()

void rm_sysinit ( void * arg )

Definition at line 342 of file kern_rmlock.c.

References rm_init_flags().

Here is the call graph for this function:

◆ rm_tracker_add()

static void rm_tracker_add	(	struct pcpu *	pc,
		struct rm_priotracker *	tracker
	)

inlinestatic

Definition at line 204 of file kern_rmlock.c.

Referenced by _rm_rlock(), and _rm_rlock_hard().

Here is the caller graph for this function:

◆ rm_tracker_remove()

static void rm_tracker_remove	(	struct pcpu *	pc,
		struct rm_priotracker *	tracker
	)

inlinestatic

Definition at line 243 of file kern_rmlock.c.

Referenced by _rm_rlock_hard(), and _rm_runlock().

Here is the caller graph for this function:

◆ rm_trackers_present()

static int rm_trackers_present	(	const struct pcpu *	pc,
		const struct rmlock *	rm,
		const struct thread *	td
	)

static

Definition at line 225 of file kern_rmlock.c.

References count.

Referenced by _rm_rlock_hard().

Here is the caller graph for this function:

◆ rm_wowned()

int rm_wowned ( const struct rmlock * rm )

Definition at line 332 of file kern_rmlock.c.

Referenced by unlock_rm().

Here is the caller graph for this function:

◆ rms_action_func()

static void rms_action_func ( void * arg )

static

Definition at line 1118 of file kern_rmlock.c.

References rmslock_pcpu::influx, rmslock_pcpu::readers, rmslock_ipi::rms, rms_int_pcpu(), and smp_rendezvous_cpus_done().

Referenced by rms_wlock_switch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_assert_no_pcpu_readers()

static void rms_assert_no_pcpu_readers ( struct rmslock * rms )

static

Definition at line 1169 of file kern_rmlock.c.

Referenced by rms_wlock().

Here is the caller graph for this function:

◆ rms_destroy()

void rms_destroy ( struct rmslock * rms )

Definition at line 997 of file kern_rmlock.c.

References pcpu_zone_8.

◆ rms_init()

void rms_init	(	struct rmslock *	rms,
		const char *	name
	)

Definition at line 985 of file kern_rmlock.c.

References name, pcpu_zone_8, and RMS_NOOWNER.

◆ rms_int_debug_readers_dec()

static void rms_int_debug_readers_dec ( struct rmslock * rms )

static

Definition at line 958 of file kern_rmlock.c.

Referenced by rms_int_readers_dec(), and rms_runlock_fallback().

Here is the caller graph for this function:

◆ rms_int_debug_readers_inc()

static void rms_int_debug_readers_inc ( struct rmslock * rms )

static

Definition at line 953 of file kern_rmlock.c.

Referenced by rms_int_readers_inc().

Here is the caller graph for this function:

◆ rms_int_influx_enter()

static void rms_int_influx_enter	(	struct rmslock *	rms,
		struct rmslock_pcpu *	pcpu
	)

static

Definition at line 917 of file kern_rmlock.c.

References rmslock_pcpu::influx.

Referenced by rms_rlock(), rms_runlock(), and rms_try_rlock().

Here is the caller graph for this function:

◆ rms_int_influx_exit()

static void rms_int_influx_exit	(	struct rmslock *	rms,
		struct rmslock_pcpu *	pcpu
	)

static

Definition at line 926 of file kern_rmlock.c.

References rmslock_pcpu::influx.

Referenced by rms_rlock(), rms_rlock_fallback(), rms_runlock(), rms_runlock_fallback(), and rms_try_rlock().

Here is the caller graph for this function:

◆ rms_int_pcpu()

static struct rmslock_pcpu * rms_int_pcpu ( struct rmslock * rms )

static

Definition at line 902 of file kern_rmlock.c.

Referenced by rms_action_func(), rms_rlock(), rms_rlock_fallback(), rms_runlock(), rms_runlock_fallback(), and rms_try_rlock().

Here is the caller graph for this function:

◆ rms_int_readers_dec()

static void rms_int_readers_dec	(	struct rmslock *	rms,
		struct rmslock_pcpu *	pcpu
	)

static

Definition at line 973 of file kern_rmlock.c.

References rmslock_pcpu::readers, and rms_int_debug_readers_dec().

Referenced by rms_runlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_int_readers_inc()

static void rms_int_readers_inc	(	struct rmslock *	rms,
		struct rmslock_pcpu *	pcpu
	)

static

Definition at line 964 of file kern_rmlock.c.

References rmslock_pcpu::readers, and rms_int_debug_readers_inc().

Referenced by rms_rlock(), rms_rlock_fallback(), and rms_try_rlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_int_remote_pcpu()

static struct rmslock_pcpu * rms_int_remote_pcpu	(	struct rmslock *	rms,
		int	cpu
	)

static

Definition at line 910 of file kern_rmlock.c.

Referenced by rms_wait_func().

Here is the caller graph for this function:

◆ rms_rlock()

void rms_rlock ( struct rmslock * rms )

Definition at line 1024 of file kern_rmlock.c.

References rms_int_influx_enter(), rms_int_influx_exit(), rms_int_pcpu(), rms_int_readers_inc(), and rms_rlock_fallback().

Here is the call graph for this function:

◆ rms_rlock_fallback()

static void __noinline rms_rlock_fallback ( struct rmslock * rms )

static

Definition at line 1007 of file kern_rmlock.c.

References rms_int_influx_exit(), rms_int_pcpu(), and rms_int_readers_inc().

Referenced by rms_rlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_runlock()

void rms_runlock ( struct rmslock * rms )

Definition at line 1092 of file kern_rmlock.c.

References rms_int_influx_enter(), rms_int_influx_exit(), rms_int_pcpu(), rms_int_readers_dec(), and rms_runlock_fallback().

Referenced by rms_unlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_runlock_fallback()

static void __noinline rms_runlock_fallback ( struct rmslock * rms )

static

Definition at line 1073 of file kern_rmlock.c.

References rms_int_debug_readers_dec(), rms_int_influx_exit(), rms_int_pcpu(), and wakeup_one().

Referenced by rms_runlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_try_rlock()

int rms_try_rlock ( struct rmslock * rms )

Definition at line 1048 of file kern_rmlock.c.

References rms_int_influx_enter(), rms_int_influx_exit(), rms_int_pcpu(), and rms_int_readers_inc().

Here is the call graph for this function:

◆ rms_unlock()

void rms_unlock ( struct rmslock * rms )

Definition at line 1252 of file kern_rmlock.c.

References rmslock_ipi::rms, rms_runlock(), and rms_wunlock().

Here is the call graph for this function:

◆ rms_wait_func()

static void rms_wait_func	(	void *	arg,
		int	cpu
	)

static

Definition at line 1138 of file kern_rmlock.c.

References rmslock_pcpu::influx, rmslock_ipi::rms, and rms_int_remote_pcpu().

Referenced by rms_wlock_switch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_wlock()

void rms_wlock ( struct rmslock * rms )

Definition at line 1193 of file kern_rmlock.c.

References rmslock_ipi::rms, rms_assert_no_pcpu_readers(), RMS_NOOWNER, RMS_TRANSIENT, and rms_wlock_switch().

Here is the call graph for this function:

◆ rms_wlock_switch()

static void rms_wlock_switch ( struct rmslock * rms )

static

Definition at line 1175 of file kern_rmlock.c.

References all_cpus, rmslock_ipi::rms, rms_action_func(), rms_wait_func(), smp_no_rendezvous_barrier(), smp_rendezvous_cpus_retry(), and rmslock_ipi::srcra.

Referenced by rms_wlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ rms_wunlock()

void rms_wunlock ( struct rmslock * rms )

Definition at line 1231 of file kern_rmlock.c.

References rmslock_ipi::rms, RMS_NOOWNER, RMS_TRANSIENT, wakeup(), and wakeup_one().

Referenced by rms_unlock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ unlock_rm()

static uintptr_t unlock_rm ( struct lock_object * lock )

static

Definition at line 139 of file kern_rmlock.c.

References rm_wowned().

Here is the call graph for this function:

Variable Documentation

◆ lock_class_rm

struct lock_class lock_class_rm

Initial value:

= {
        .lc_name = "rm",
        .lc_flags = LC_SLEEPLOCK | LC_RECURSABLE,
        .lc_assert = assert_rm,
 
        .lc_lock = lock_rm,
        .lc_unlock = unlock_rm,
 
}

Definition at line 88 of file kern_rmlock.c.

Referenced by rm_init_flags(), and softclock_call_cc().

◆ lock_class_rm_sleepable

struct lock_class lock_class_rm_sleepable

Initial value:

= {
        .lc_name = "sleepable rm",
        .lc_flags = LC_SLEEPLOCK | LC_SLEEPABLE | LC_RECURSABLE,
        .lc_assert = assert_rm,
 
        .lc_lock = lock_rm,
        .lc_unlock = unlock_rm,
 
}

Definition at line 102 of file kern_rmlock.c.

Referenced by rm_init_flags().

◆ rm_spinlock

struct mtx rm_spinlock

static

Definition at line 193 of file kern_rmlock.c.

Referenced by _rm_rlock_hard(), _rm_unlock_hard(), _rm_wlock(), and rm_cleanIPI().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ _rm_assert

◆ RM_DESTROYED

◆ rm_destroyed

◆ RMPF_ONQUEUE

◆ RMPF_SIGNAL

◆ RMS_FLAGMASK

◆ RMS_NOOWNER

◆ RMS_TRANSIENT

Function Documentation

◆ __FBSDID()

◆ _rm_rlock()

◆ _rm_rlock_debug()

◆ _rm_rlock_hard()

◆ _rm_runlock()

◆ _rm_runlock_debug()

◆ _rm_unlock_hard()

◆ _rm_wlock()

◆ _rm_wlock_debug()

◆ _rm_wunlock()

◆ _rm_wunlock_debug()

◆ _Static_assert()

◆ assert_rm()

◆ lock_rm()

◆ MTX_SYSINIT()

◆ rm_cleanIPI()

◆ rm_destroy()

◆ rm_init()

◆ rm_init_flags()

◆ rm_sysinit()

◆ rm_tracker_add()

◆ rm_tracker_remove()

◆ rm_trackers_present()

◆ rm_wowned()

◆ rms_action_func()

◆ rms_assert_no_pcpu_readers()

◆ rms_destroy()

◆ rms_init()

◆ rms_int_debug_readers_dec()

◆ rms_int_debug_readers_inc()

◆ rms_int_influx_enter()

◆ rms_int_influx_exit()

◆ rms_int_pcpu()

◆ rms_int_readers_dec()

◆ rms_int_readers_inc()

◆ rms_int_remote_pcpu()

◆ rms_rlock()

◆ rms_rlock_fallback()

◆ rms_runlock()

◆ rms_runlock_fallback()

◆ rms_try_rlock()

◆ rms_unlock()

◆ rms_wait_func()

◆ rms_wlock()

◆ rms_wlock_switch()

◆ rms_wunlock()

◆ unlock_rm()

Variable Documentation

◆ lock_class_rm

◆ lock_class_rm_sleepable

◆ rm_spinlock