#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/counter.h>
#include <sys/epoch.h>
#include <sys/gtaskqueue.h>
#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/sx.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/turnstile.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/uma.h>
#include <ck_epoch.h>

Include dependency graph for subr_epoch.c:

Data Structures
struct	epoch_record

Macros
#define	EPOCH_ALIGN CACHE_LINE_SIZE

#define	MAX_ADAPTIVE_SPIN 100

#define	MAX_EPOCHS 64

#define	EPOCH_LOCK() sx_xlock(&epoch_sx)

#define	EPOCH_UNLOCK() sx_xunlock(&epoch_sx)

#define	INIT_CHECK(epoch)

#define	epoch_assert_nocpu(e, td) do {} while (0)

Functions
	__FBSDID ("$FreeBSD$")

	TAILQ_HEAD (epoch_tdlist, epoch_tracker)

struct epoch_record	__aligned (EPOCH_ALIGN)

	CTASSERT (sizeof(ck_epoch_entry_t)==sizeof(struct epoch_context))

	SYSCTL_NODE (_kern, OID_AUTO, epoch, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "epoch information")

	SYSCTL_NODE (_kern_epoch, OID_AUTO, stats, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "epoch stats")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, nblocked, CTLFLAG_RW, &block_count, "# of times a thread was in an epoch when epoch_wait was called")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, migrations, CTLFLAG_RW, &migrate_count, "# of times thread was migrated to another CPU in epoch_wait")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, ncontended, CTLFLAG_RW, &turnstile_count, "# of times a thread was blocked on a lock in an epoch during an epoch_wait")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, switches, CTLFLAG_RW, &switch_count, "# of times a thread voluntarily context switched in epoch_wait")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, epoch_calls, CTLFLAG_RW, &epoch_call_count, "# of times a callback was deferred")

	SYSCTL_COUNTER_U64 (_kern_epoch_stats, OID_AUTO, epoch_call_tasks, CTLFLAG_RW, &epoch_call_task_count, "# of times a callback task was run")

	TAILQ_HEAD (threadlist, thread)

	CK_STACK_CONTAINER (struct ck_epoch_entry, stack_entry, ck_epoch_entry_container)

static void	epoch_init (void *arg __unused)

	SYSINIT (epoch, SI_SUB_EPOCH, SI_ORDER_FIRST, epoch_init, NULL)

static void	epoch_init_smp (void *dummy __unused)

	SYSINIT (epoch_smp, SI_SUB_SMP+1, SI_ORDER_FIRST, epoch_init_smp, NULL)

static void	epoch_ctor (epoch_t epoch)

static void	epoch_adjust_prio (struct thread *td, u_char prio)

epoch_t	epoch_alloc (const char *name, int flags)

void	epoch_free (epoch_t epoch)

void	_epoch_enter_preempt (epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE)

void	epoch_enter (epoch_t epoch)

void	_epoch_exit_preempt (epoch_t epoch, epoch_tracker_t et EPOCH_FILE_LINE)

void	epoch_exit (epoch_t epoch)

static void	epoch_block_handler_preempt (struct ck_epoch global __unused, ck_epoch_record_t cr, void *arg __unused)

void	epoch_wait_preempt (epoch_t epoch)

static void	epoch_block_handler (struct ck_epoch g __unused, ck_epoch_record_t c __unused, void *arg __unused)

void	epoch_wait (epoch_t epoch)

void	epoch_call (epoch_t epoch, epoch_callback_t callback, epoch_context_t ctx)

static void	epoch_call_task (void *arg __unused)

static int	in_epoch_verbose_preempt (epoch_t epoch, int dump_onfail)

int	in_epoch_verbose (epoch_t epoch, int dump_onfail)

int	in_epoch (epoch_t epoch)

static void	epoch_drain_cb (struct epoch_context *ctx)

void	epoch_drain_callbacks (epoch_t epoch)

Variables
ck_epoch_record_t	er_record

struct epoch_context	er_drain_ctx

struct epoch *	er_parent

volatile struct epoch_tdlist	er_tdlist

volatile uint32_t	er_gen

uint32_t	er_cpuid

static counter_u64_t	block_count

static counter_u64_t	migrate_count

static counter_u64_t	turnstile_count

static counter_u64_t	switch_count

static counter_u64_t	epoch_call_count

static counter_u64_t	epoch_call_task_count

Macro Definition Documentation

◆ EPOCH_ALIGN

#define EPOCH_ALIGN CACHE_LINE_SIZE

Definition at line 64 of file subr_epoch.c.

◆ epoch_assert_nocpu

#define epoch_assert_nocpu	(	e,
		td
	)	do {} while (0)

Definition at line 900 of file subr_epoch.c.

◆ EPOCH_LOCK

#define EPOCH_LOCK ( ) sx_xlock(&epoch_sx)

◆ EPOCH_UNLOCK

#define EPOCH_UNLOCK ( ) sx_xunlock(&epoch_sx)

◆ INIT_CHECK

#define INIT_CHECK ( epoch )

Value:

        do {                                                    \
                if (__predict_false((epoch) == NULL))           \
                        return;                                 \
        } while (0)

Definition at line 458 of file subr_epoch.c.

◆ MAX_ADAPTIVE_SPIN

#define MAX_ADAPTIVE_SPIN 100

Definition at line 93 of file subr_epoch.c.

◆ MAX_EPOCHS

#define MAX_EPOCHS 64

Definition at line 94 of file subr_epoch.c.

Function Documentation

◆ __aligned()

struct epoch_record __aligned ( EPOCH_ALIGN )

Definition at line 67 of file subr_epoch.c.

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _epoch_enter_preempt()

void _epoch_enter_preempt	(	epoch_t	epoch,
		epoch_tracker_t et	EPOCH_FILE_LINE
	)

Definition at line 465 of file subr_epoch.c.

References epoch_record::er_record, epoch_record::er_tdlist, and INIT_CHECK.

◆ _epoch_exit_preempt()

void _epoch_exit_preempt	(	epoch_t	epoch,
		epoch_tracker_t et	EPOCH_FILE_LINE
	)

Definition at line 517 of file subr_epoch.c.

References epoch_adjust_prio(), epoch_record::er_gen, epoch_record::er_record, epoch_record::er_tdlist, and INIT_CHECK.

Here is the call graph for this function:

◆ CK_STACK_CONTAINER()

CK_STACK_CONTAINER	(	struct ck_epoch_entry	,
		stack_entry	,
		ck_epoch_entry_container
	)

Definition at line 130 of file subr_epoch.c.

◆ CTASSERT()

CTASSERT ( sizeof(ck_epoch_entry_t) = =sizeof(struct epoch_context) )

◆ epoch_adjust_prio()

static void epoch_adjust_prio	(	struct thread *	td,
		u_char	prio
	)

static

Definition at line 359 of file subr_epoch.c.

References sched_prio().

Referenced by _epoch_exit_preempt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_alloc()

epoch_t epoch_alloc	(	const char *	name,
		int	flags
	)

Definition at line 368 of file subr_epoch.c.

References epoch_ctor(), EPOCH_LOCK, EPOCH_UNLOCK, flags, MAX_EPOCHS, name, and panic().

Referenced by epoch_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_block_handler()

static void epoch_block_handler	(	struct ck_epoch *g	__unused,
		ck_epoch_record_t *c	__unused,
		void *arg	__unused
	)

static

Definition at line 762 of file subr_epoch.c.

Referenced by epoch_wait().

Here is the caller graph for this function:

◆ epoch_block_handler_preempt()

static void epoch_block_handler_preempt	(	struct ck_epoch *global	__unused,
		ck_epoch_record_t *	cr,
		void *arg	__unused
	)

static

Definition at line 568 of file subr_epoch.c.

References block_count, er_record, MAX_ADAPTIVE_SPIN, mi_switch(), migrate_count, sched_bind(), sched_prio(), switch_count, ts, turnstile_count, turnstile_lock(), turnstile_unlock(), and turnstile_wait().

Referenced by epoch_wait_preempt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_call()

void epoch_call	(	epoch_t	epoch,
		epoch_callback_t	callback,
		epoch_context_t	ctx
	)

Definition at line 781 of file subr_epoch.c.

References callback.

Referenced by epoch_drain_callbacks().

Here is the caller graph for this function:

◆ epoch_call_task()

static void epoch_call_task ( void *arg __unused )

static

Definition at line 808 of file subr_epoch.c.

References epoch_call_count, epoch_call_task_count, epoch_enter(), epoch_exit(), and MAX_EPOCHS.

Referenced by epoch_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_ctor()

static void epoch_ctor ( epoch_t epoch )

static

Definition at line 342 of file subr_epoch.c.

Referenced by epoch_alloc().

Here is the caller graph for this function:

◆ epoch_drain_callbacks()

void epoch_drain_callbacks ( epoch_t epoch )

Definition at line 961 of file subr_epoch.c.

References epoch_call(), epoch_drain_cb(), sched_bind(), sched_is_bound(), and sched_unbind().

Referenced by epoch_free().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_drain_cb()

static void epoch_drain_cb ( struct epoch_context * ctx )

static

Definition at line 948 of file subr_epoch.c.

References er_drain_ctx, and wakeup().

Referenced by epoch_drain_callbacks().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_enter()

void epoch_enter ( epoch_t epoch )

Definition at line 496 of file subr_epoch.c.

References INIT_CHECK.

Referenced by epoch_call_task().

Here is the caller graph for this function:

◆ epoch_exit()

void epoch_exit ( epoch_t epoch )

Definition at line 548 of file subr_epoch.c.

References INIT_CHECK.

Referenced by epoch_call_task().

Here is the caller graph for this function:

◆ epoch_free()

void epoch_free ( epoch_t epoch )

Definition at line 417 of file subr_epoch.c.

References epoch_drain_callbacks(), EPOCH_LOCK, EPOCH_UNLOCK, epoch_wait(), and sx_destroy().

Here is the call graph for this function:

◆ epoch_init()

static void epoch_init ( void *arg __unused )

static

Definition at line 301 of file subr_epoch.c.

References block_count, counter_u64_alloc(), epoch_alloc(), epoch_call_count, epoch_call_task(), epoch_call_task_count, migrate_count, switch_count, taskqgroup_attach_cpu(), and turnstile_count.

Here is the call graph for this function:

◆ epoch_init_smp()

static void epoch_init_smp ( void *dummy __unused )

static

Definition at line 334 of file subr_epoch.c.

◆ epoch_wait()

void epoch_wait ( epoch_t epoch )

Definition at line 769 of file subr_epoch.c.

References epoch_block_handler(), and INIT_CHECK.

Referenced by epoch_free().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ epoch_wait_preempt()

void epoch_wait_preempt ( epoch_t epoch )

Definition at line 706 of file subr_epoch.c.

References epoch_block_handler_preempt(), in_epoch(), INIT_CHECK, sched_bind(), sched_is_bound(), sched_prio(), and sched_unbind().

Here is the call graph for this function:

◆ in_epoch()

int in_epoch ( epoch_t epoch )

Definition at line 942 of file subr_epoch.c.

References in_epoch_verbose().

Referenced by epoch_wait_preempt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ in_epoch_verbose()

int in_epoch_verbose	(	epoch_t	epoch,
		int	dump_onfail
	)

Definition at line 904 of file subr_epoch.c.

References epoch_assert_nocpu, and in_epoch_verbose_preempt().

Referenced by in_epoch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ in_epoch_verbose_preempt()

static int in_epoch_verbose_preempt	(	epoch_t	epoch,
		int	dump_onfail
	)

static

Definition at line 850 of file subr_epoch.c.

References printf().

Referenced by in_epoch_verbose().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ SYSCTL_COUNTER_U64() [1/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		epoch_call_tasks	,
		CTLFLAG_RW	,
		&	epoch_call_task_count,
		"# of times a callback task was run"
	)

◆ SYSCTL_COUNTER_U64() [2/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		epoch_calls	,
		CTLFLAG_RW	,
		&	epoch_call_count,
		"# of times a callback was deferred"
	)

◆ SYSCTL_COUNTER_U64() [3/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		migrations	,
		CTLFLAG_RW	,
		&	migrate_count,
		"# of times thread was migrated to another CPU in epoch_wait"
	)

◆ SYSCTL_COUNTER_U64() [4/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		nblocked	,
		CTLFLAG_RW	,
		&	block_count,
		"# of times a thread was in an epoch when epoch_wait was called"
	)

◆ SYSCTL_COUNTER_U64() [5/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		ncontended	,
		CTLFLAG_RW	,
		&	turnstile_count,
		"# of times a thread was blocked on a lock in an epoch during an epoch_wait"
	)

◆ SYSCTL_COUNTER_U64() [6/6]

SYSCTL_COUNTER_U64	(	_kern_epoch_stats	,
		OID_AUTO	,
		switches	,
		CTLFLAG_RW	,
		&	switch_count,
		"# of times a thread voluntarily context switched in epoch_wait"
	)

◆ SYSCTL_NODE() [1/2]

SYSCTL_NODE	(	_kern	,
		OID_AUTO	,
		epoch	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"epoch information"
	)

◆ SYSCTL_NODE() [2/2]

SYSCTL_NODE	(	_kern_epoch	,
		OID_AUTO	,
		stats	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"epoch stats"
	)

◆ SYSINIT() [1/2]

SYSINIT	(	epoch	,
		SI_SUB_EPOCH	,
		SI_ORDER_FIRST	,
		epoch_init	,
		NULL
	)

◆ SYSINIT() [2/2]

SYSINIT	(	epoch_smp	,
		SI_SUB_SMP+	1,
		SI_ORDER_FIRST	,
		epoch_init_smp	,
		NULL
	)

◆ TAILQ_HEAD() [1/2]

TAILQ_HEAD	(	epoch_tdlist	,
		epoch_tracker
	)

◆ TAILQ_HEAD() [2/2]

TAILQ_HEAD	(	threadlist	,
		thread
	)

Variable Documentation

◆ block_count

counter_u64_t block_count

static

Definition at line 103 of file subr_epoch.c.

Referenced by epoch_block_handler_preempt(), and epoch_init().

◆ epoch_call_count

counter_u64_t epoch_call_count

static

Definition at line 119 of file subr_epoch.c.

Referenced by epoch_call_task(), and epoch_init().

◆ epoch_call_task_count

counter_u64_t epoch_call_task_count

static

Definition at line 123 of file subr_epoch.c.

Referenced by epoch_call_task(), and epoch_init().

◆ er_cpuid

uint32_t er_cpuid

Definition at line 85 of file subr_epoch.c.

◆ er_drain_ctx

struct epoch_context er_drain_ctx

Definition at line 81 of file subr_epoch.c.

Referenced by epoch_drain_cb().

◆ er_gen

volatile uint32_t er_gen

Definition at line 84 of file subr_epoch.c.

◆ er_parent

struct epoch* er_parent

Definition at line 82 of file subr_epoch.c.

◆ er_record

ck_epoch_record_t er_record

Definition at line 80 of file subr_epoch.c.

Referenced by epoch_block_handler_preempt().

◆ er_tdlist

volatile struct epoch_tdlist er_tdlist

Definition at line 83 of file subr_epoch.c.

◆ migrate_count

counter_u64_t migrate_count

static

Definition at line 107 of file subr_epoch.c.

Referenced by epoch_block_handler_preempt(), and epoch_init().

◆ switch_count

counter_u64_t switch_count

static

Definition at line 115 of file subr_epoch.c.

Referenced by epoch_block_handler_preempt(), and epoch_init().

◆ turnstile_count

counter_u64_t turnstile_count

static

Definition at line 111 of file subr_epoch.c.

Referenced by epoch_block_handler_preempt(), and epoch_init().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ EPOCH_ALIGN

◆ epoch_assert_nocpu

◆ EPOCH_LOCK

◆ EPOCH_UNLOCK

◆ INIT_CHECK

◆ MAX_ADAPTIVE_SPIN

◆ MAX_EPOCHS

Function Documentation

◆ __aligned()

◆ __FBSDID()

◆ _epoch_enter_preempt()

◆ _epoch_exit_preempt()

◆ CK_STACK_CONTAINER()

◆ CTASSERT()

◆ epoch_adjust_prio()

◆ epoch_alloc()

◆ epoch_block_handler()

◆ epoch_block_handler_preempt()

◆ epoch_call()

◆ epoch_call_task()

◆ epoch_ctor()

◆ epoch_drain_callbacks()

◆ epoch_drain_cb()

◆ epoch_enter()

◆ epoch_exit()

◆ epoch_free()

◆ epoch_init()

◆ epoch_init_smp()

◆ epoch_wait()

◆ epoch_wait_preempt()

◆ in_epoch()

◆ in_epoch_verbose()

◆ in_epoch_verbose_preempt()

◆ SYSCTL_COUNTER_U64() [1/6]

◆ SYSCTL_COUNTER_U64() [2/6]

◆ SYSCTL_COUNTER_U64() [3/6]

◆ SYSCTL_COUNTER_U64() [4/6]

◆ SYSCTL_COUNTER_U64() [5/6]

◆ SYSCTL_COUNTER_U64() [6/6]

◆ SYSCTL_NODE() [1/2]

◆ SYSCTL_NODE() [2/2]

◆ SYSINIT() [1/2]

◆ SYSINIT() [2/2]

◆ TAILQ_HEAD() [1/2]

◆ TAILQ_HEAD() [2/2]

Variable Documentation

◆ block_count

◆ epoch_call_count

◆ epoch_call_task_count

◆ er_cpuid

◆ er_drain_ctx

◆ er_gen

◆ er_parent

◆ er_record

◆ er_tdlist

◆ migrate_count

◆ switch_count

◆ turnstile_count