#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/cpuset.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/libkern.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/epoch.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/gtaskqueue.h>
#include <sys/unistd.h>
#include <machine/stdarg.h>

Include dependency graph for subr_gtaskqueue.c:

Data Structures
struct	gtaskqueue_busy

struct	gtaskqueue

struct	taskqgroup_cpu

struct	taskqgroup

struct	taskq_bind_task

Macros
#define	TQ_FLAGS_ACTIVE (1 << 0)

#define	TQ_FLAGS_BLOCKED (1 << 1)

#define	TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2)

#define	DT_CALLOUT_ARMED (1 << 0)

#define	TQ_LOCK(tq)

#define	TQ_ASSERT_LOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_OWNED)

#define	TQ_UNLOCK(tq)

#define	TQ_ASSERT_UNLOCKED(tq) mtx_assert(&(tq)->tq_mutex, MA_NOTOWNED)

Typedefs
typedef void(*	gtaskqueue_enqueue_fn) (void *context)

Functions
	__FBSDID ("$FreeBSD$")

static	MALLOC_DEFINE (M_GTASKQUEUE, "gtaskqueue", "Group Task Queues")

static void	gtaskqueue_thread_enqueue (void *)

static void	gtaskqueue_thread_loop (void *arg)

static int	task_is_running (struct gtaskqueue queue, struct gtask gtask)

static void	gtaskqueue_drain_locked (struct gtaskqueue queue, struct gtask gtask)

	TASKQGROUP_DEFINE (softirq, mp_ncpus, 1)

static __inline int	TQ_SLEEP (struct gtaskqueue tq, void p, const char *wm)

static struct gtaskqueue *	_gtaskqueue_create (const char name, int mflags, taskqueue_enqueue_fn enqueue, void context, int mtxflags, const char *mtxname __unused)

static void	gtaskqueue_terminate (struct thread *pp, struct gtaskqueue tq)

static void __unused	gtaskqueue_free (struct gtaskqueue *queue)

void	grouptask_block (struct grouptask *grouptask)

void	grouptask_unblock (struct grouptask *grouptask)

int	grouptaskqueue_enqueue (struct gtaskqueue queue, struct gtask gtask)

static void	gtaskqueue_task_nop_fn (void *context)

static void	gtaskqueue_drain_tq_queue (struct gtaskqueue *queue)

static void	gtaskqueue_drain_tq_active (struct gtaskqueue *queue)

void	gtaskqueue_block (struct gtaskqueue *queue)

void	gtaskqueue_unblock (struct gtaskqueue *queue)

static void	gtaskqueue_run_locked (struct gtaskqueue *queue)

static int	gtaskqueue_cancel_locked (struct gtaskqueue queue, struct gtask gtask)

int	gtaskqueue_cancel (struct gtaskqueue queue, struct gtask gtask)

void	gtaskqueue_drain (struct gtaskqueue queue, struct gtask gtask)

void	gtaskqueue_drain_all (struct gtaskqueue *queue)

static int	_gtaskqueue_start_threads (struct gtaskqueue *tqp, int count, int pri, cpuset_t mask, const char *name, va_list ap)

static int	gtaskqueue_start_threads (struct gtaskqueue *tqp, int count, int pri, const char name,...)

static void	gtaskqueue_run_callback (struct gtaskqueue *tq, enum taskqueue_callback_type cb_type)

static struct gtaskqueue *	gtaskqueue_create_fast (const char name, int mflags, taskqueue_enqueue_fn enqueue, void context)

static void	taskqgroup_cpu_create (struct taskqgroup *qgroup, int idx, int cpu)

static int	taskqgroup_find (struct taskqgroup qgroup, void uniq)

void	taskqgroup_attach (struct taskqgroup qgroup, struct grouptask gtask, void uniq, device_t dev, struct resource irq, const char *name)

int	taskqgroup_attach_cpu (struct taskqgroup qgroup, struct grouptask gtask, void uniq, int cpu, device_t dev, struct resource irq, const char *name)

void	taskqgroup_detach (struct taskqgroup qgroup, struct grouptask gtask)

static void	taskqgroup_binder (void *ctx)

void	taskqgroup_bind (struct taskqgroup *qgroup)

struct taskqgroup *	taskqgroup_create (const char *name, int cnt, int stride)

void	taskqgroup_destroy (struct taskqgroup *qgroup)

void	taskqgroup_drain_all (struct taskqgroup *tqg)

Macro Definition Documentation

◆ DT_CALLOUT_ARMED

#define DT_CALLOUT_ARMED (1 << 0)

Definition at line 88 of file subr_gtaskqueue.c.

◆ TQ_ASSERT_LOCKED

#define TQ_ASSERT_LOCKED ( tq ) mtx_assert(&(tq)->tq_mutex, MA_OWNED)

Definition at line 97 of file subr_gtaskqueue.c.

◆ TQ_ASSERT_UNLOCKED

#define TQ_ASSERT_UNLOCKED ( tq ) mtx_assert(&(tq)->tq_mutex, MA_NOTOWNED)

Definition at line 106 of file subr_gtaskqueue.c.

◆ TQ_FLAGS_ACTIVE

#define TQ_FLAGS_ACTIVE (1 << 0)

Definition at line 84 of file subr_gtaskqueue.c.

◆ TQ_FLAGS_BLOCKED

#define TQ_FLAGS_BLOCKED (1 << 1)

Definition at line 85 of file subr_gtaskqueue.c.

◆ TQ_FLAGS_UNLOCKED_ENQUEUE

#define TQ_FLAGS_UNLOCKED_ENQUEUE (1 << 2)

Definition at line 86 of file subr_gtaskqueue.c.

◆ TQ_LOCK

#define TQ_LOCK ( tq )

Value:

        do {                                                            \
                if ((tq)->tq_spin)                                      \
                        mtx_lock_spin(&(tq)->tq_mutex);                 \
                else                                                    \
                        mtx_lock(&(tq)->tq_mutex);                      \
        } while (0)

Definition at line 90 of file subr_gtaskqueue.c.

◆ TQ_UNLOCK

#define TQ_UNLOCK ( tq )

Value:

        do {                                                            \
                if ((tq)->tq_spin)                                      \
                        mtx_unlock_spin(&(tq)->tq_mutex);               \
                else                                                    \
                        mtx_unlock(&(tq)->tq_mutex);                    \
        } while (0)

Definition at line 99 of file subr_gtaskqueue.c.

Typedef Documentation

◆ gtaskqueue_enqueue_fn

typedef void(* gtaskqueue_enqueue_fn) (void *context)

Definition at line 65 of file subr_gtaskqueue.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _gtaskqueue_create()

static struct gtaskqueue * _gtaskqueue_create	(	const char *	name,
		int	mflags,
		taskqueue_enqueue_fn	enqueue,
		void *	context,
		int	mtxflags,
		const char *mtxname	__unused
	)

static

Definition at line 126 of file subr_gtaskqueue.c.

References free(), gtaskqueue_thread_enqueue(), malloc(), name, snprintf(), TQ_FLAGS_ACTIVE, and TQ_FLAGS_UNLOCKED_ENQUEUE.

Referenced by gtaskqueue_create_fast().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ _gtaskqueue_start_threads()

static int _gtaskqueue_start_threads	(	struct gtaskqueue **	tqp,
		int	count,
		int	pri,
		cpuset_t *	mask,
		const char *	name,
		va_list	ap
	)

static

Definition at line 449 of file subr_gtaskqueue.c.

References count, cpuset_setthread(), gtaskqueue_thread_loop(), kthread_add(), malloc(), mask, name, printf(), sched_add(), sched_prio(), and vsnprintf().

Referenced by gtaskqueue_start_threads().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ grouptask_block()

void grouptask_block ( struct grouptask * grouptask )

Definition at line 191 of file subr_gtaskqueue.c.

References gtaskqueue_drain_locked(), panic(), TQ_LOCK, and TQ_UNLOCK.

Referenced by taskqgroup_detach().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ grouptask_unblock()

void grouptask_unblock ( struct grouptask * grouptask )

Definition at line 209 of file subr_gtaskqueue.c.

References panic(), TQ_LOCK, and TQ_UNLOCK.

Here is the call graph for this function:

◆ grouptaskqueue_enqueue()

int grouptaskqueue_enqueue	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

Definition at line 226 of file subr_gtaskqueue.c.

References panic(), TQ_FLAGS_BLOCKED, TQ_LOCK, and TQ_UNLOCK.

Referenced by taskqgroup_bind().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_block()

void gtaskqueue_block ( struct gtaskqueue * queue )

Definition at line 323 of file subr_gtaskqueue.c.

References TQ_FLAGS_BLOCKED, TQ_LOCK, and TQ_UNLOCK.

◆ gtaskqueue_cancel()

int gtaskqueue_cancel	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

Definition at line 405 of file subr_gtaskqueue.c.

References gtaskqueue_cancel_locked(), TQ_LOCK, and TQ_UNLOCK.

Here is the call graph for this function:

◆ gtaskqueue_cancel_locked()

static int gtaskqueue_cancel_locked	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

static

Definition at line 395 of file subr_gtaskqueue.c.

References task_is_running().

Referenced by gtaskqueue_cancel().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_create_fast()

static struct gtaskqueue * gtaskqueue_create_fast	(	const char *	name,
		int	mflags,
		taskqueue_enqueue_fn	enqueue,
		void *	context
	)

static

Definition at line 585 of file subr_gtaskqueue.c.

References _gtaskqueue_create(), and name.

Referenced by taskqgroup_cpu_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_drain()

void gtaskqueue_drain	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

Definition at line 424 of file subr_gtaskqueue.c.

References gtaskqueue_drain_locked(), TQ_LOCK, and TQ_UNLOCK.

Here is the call graph for this function:

◆ gtaskqueue_drain_all()

void gtaskqueue_drain_all ( struct gtaskqueue * queue )

Definition at line 436 of file subr_gtaskqueue.c.

References gtaskqueue_drain_tq_active(), gtaskqueue_drain_tq_queue(), TQ_LOCK, and TQ_UNLOCK.

Referenced by taskqgroup_drain_all().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_drain_locked()

static void gtaskqueue_drain_locked	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

static

Definition at line 417 of file subr_gtaskqueue.c.

References task_is_running(), and TQ_SLEEP().

Referenced by grouptask_block(), and gtaskqueue_drain().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_drain_tq_active()

static void gtaskqueue_drain_tq_active ( struct gtaskqueue * queue )

static

Definition at line 295 of file subr_gtaskqueue.c.

References gtaskqueue_busy::tb_running, gtaskqueue_busy::tb_seq, TQ_FLAGS_ACTIVE, TQ_SLEEP(), and wakeup_one().

Referenced by gtaskqueue_drain_all().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_drain_tq_queue()

static void gtaskqueue_drain_tq_queue ( struct gtaskqueue * queue )

static

Definition at line 262 of file subr_gtaskqueue.c.

References gtaskqueue_task_nop_fn(), and TQ_SLEEP().

Referenced by gtaskqueue_drain_all().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_free()

static void __unused gtaskqueue_free ( struct gtaskqueue * queue )

static

Definition at line 173 of file subr_gtaskqueue.c.

References free(), gtaskqueue_terminate(), and TQ_LOCK.

Here is the call graph for this function:

◆ gtaskqueue_run_callback()

static void gtaskqueue_run_callback	(	struct gtaskqueue *	tq,
		enum taskqueue_callback_type	cb_type
	)

inlinestatic

Definition at line 525 of file subr_gtaskqueue.c.

References TQ_ASSERT_UNLOCKED.

Referenced by gtaskqueue_thread_loop().

Here is the caller graph for this function:

◆ gtaskqueue_run_locked()

static void gtaskqueue_run_locked ( struct gtaskqueue * queue )

static

Definition at line 343 of file subr_gtaskqueue.c.

References gtaskqueue_busy::tb_running, gtaskqueue_busy::tb_seq, TQ_ASSERT_LOCKED, TQ_LOCK, TQ_UNLOCK, and wakeup().

Referenced by gtaskqueue_thread_loop().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_start_threads()

static int gtaskqueue_start_threads	(	struct gtaskqueue **	tqp,
		int	count,
		int	pri,
		const char *	name,
			...
	)

static

Definition at line 512 of file subr_gtaskqueue.c.

References _gtaskqueue_start_threads(), count, and name.

Referenced by taskqgroup_cpu_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_task_nop_fn()

static void gtaskqueue_task_nop_fn ( void * context )

static

Definition at line 252 of file subr_gtaskqueue.c.

Referenced by gtaskqueue_drain_tq_queue().

Here is the caller graph for this function:

◆ gtaskqueue_terminate()

static void gtaskqueue_terminate	(	struct thread **	pp,
		struct gtaskqueue *	tq
	)

static

Definition at line 163 of file subr_gtaskqueue.c.

References TQ_SLEEP(), and wakeup().

Referenced by gtaskqueue_free().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_thread_enqueue()

static void gtaskqueue_thread_enqueue ( void * context )

static

Definition at line 575 of file subr_gtaskqueue.c.

References wakeup_any().

Referenced by _gtaskqueue_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_thread_loop()

static void gtaskqueue_thread_loop ( void * arg )

static

Definition at line 537 of file subr_gtaskqueue.c.

References gtaskqueue_run_callback(), gtaskqueue_run_locked(), kthread_exit(), TQ_FLAGS_ACTIVE, TQ_LOCK, TQ_SLEEP(), TQ_UNLOCK, and wakeup_one().

Referenced by _gtaskqueue_start_threads().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ gtaskqueue_unblock()

void gtaskqueue_unblock ( struct gtaskqueue * queue )

Definition at line 332 of file subr_gtaskqueue.c.

References TQ_LOCK, and TQ_UNLOCK.

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_GTASKQUEUE	,
		"gtaskqueue"	,
		"Group Task Queues"
	)

static

◆ task_is_running()

static int task_is_running	(	struct gtaskqueue *	queue,
		struct gtask *	gtask
	)

static

Definition at line 382 of file subr_gtaskqueue.c.

References gtaskqueue_busy::tb_running, and TQ_ASSERT_LOCKED.

Referenced by gtaskqueue_cancel_locked(), and gtaskqueue_drain_locked().

Here is the caller graph for this function:

◆ taskqgroup_attach()

void taskqgroup_attach	(	struct taskqgroup *	qgroup,
		struct grouptask *	gtask,
		void *	uniq,
		device_t	dev,
		struct resource *	irq,
		const char *	name
	)

Definition at line 669 of file subr_gtaskqueue.c.

References bus_bind_intr(), name, printf(), snprintf(), taskqgroup_find(), taskqgroup::tqg_cnt, taskqgroup::tqg_lock, taskqgroup::tqg_name, and taskqgroup::tqg_queue.

Here is the call graph for this function:

◆ taskqgroup_attach_cpu()

int taskqgroup_attach_cpu	(	struct taskqgroup *	qgroup,
		struct grouptask *	gtask,
		void *	uniq,
		int	cpu,
		device_t	dev,
		struct resource *	irq,
		const char *	name
	)