#include <sys/cdefs.h>
#include "opt_ddb.h"
#include "opt_hwpmc_hooks.h"
#include "opt_kstack_usage_prof.h"
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/cpuset.h>
#include <sys/rtprio.h>
#include <sys/systm.h>
#include <sys/interrupt.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/priv.h>
#include <sys/proc.h>
#include <sys/epoch.h>
#include <sys/random.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <sys/unistd.h>
#include <sys/vmmeter.h>
#include <machine/atomic.h>
#include <machine/cpu.h>
#include <machine/md_var.h>
#include <machine/smp.h>
#include <machine/stdarg.h>

Include dependency graph for kern_intr.c:

Data Structures
struct	intr_thread

struct	intr_entropy

Macros
#define	IT_DEAD 0x000001 /* Thread is waiting to exit. */

#define	IT_WAIT 0x000002 /* Thread is waiting for completion. */

Functions
	__FBSDID ("$FreeBSD$")

static	MALLOC_DEFINE (M_ITHREAD, "ithread", "Interrupt Threads")

	SYSCTL_INT (_hw, OID_AUTO, intr_storm_threshold, CTLFLAG_RWTUN, &intr_storm_threshold, 0, "Number of consecutive interrupts before storm protection is enabled")

	SYSCTL_INT (_hw, OID_AUTO, intr_epoch_batch, CTLFLAG_RWTUN, &intr_epoch_batch, 0, "Maximum interrupt handler executions without re-entering epoch(9)")

static	TAILQ_HEAD (intr_event)

static void	ithread_update (struct intr_thread *ithd)

static void	intr_event_update (struct intr_event *ie)

int	intr_event_create (struct intr_event *event, void source, int flags, int irq, void(pre_ithread)(void ), void(post_ithread)(void ), void(post_filter)(void ), int(assign_cpu)(void , int), const char *fmt,...)

static int	_intr_event_bind (struct intr_event *ie, int cpu, bool bindirq, bool bindithread)

int	intr_event_bind (struct intr_event *ie, int cpu)

int	intr_event_bind_irqonly (struct intr_event *ie, int cpu)

int	intr_event_bind_ithread (struct intr_event *ie, int cpu)

int	intr_event_bind_ithread_cpuset (struct intr_event ie, cpuset_t cs)

static struct intr_event *	intr_lookup (int irq)

int	intr_setaffinity (int irq, int mode, void *m)

int	intr_getaffinity (int irq, int mode, void *m)

int	intr_event_destroy (struct intr_event *ie)

static struct intr_thread *	ithread_create (const char *name)

static void	ithread_destroy (struct intr_thread *ithread)

int	intr_event_add_handler (struct intr_event ie, const char name, driver_filter_t filter, driver_intr_t handler, void arg, u_char pri, enum intr_type flags, void *cookiep)

int	intr_event_describe_handler (struct intr_event ie, void cookie, const char *descr)

void *	intr_handler_source (void *cookie)

static void	intr_event_barrier (struct intr_event *ie)

static void	intr_handler_barrier (struct intr_handler *handler)

void	_intr_drain (int irq)

int	intr_event_remove_handler (void *cookie)

int	intr_event_suspend_handler (void *cookie)

int	intr_event_resume_handler (void *cookie)

static int	intr_event_schedule_thread (struct intr_event ie, struct trapframe frame)

static int	swi_assign_cpu (void *arg, int cpu)

int	swi_add (struct intr_event *eventp, const char name, driver_intr_t handler, void arg, int pri, enum intr_type flags, void *cookiep)

void	swi_sched (void *cookie, int flags)

int	swi_remove (void *cookie)

static void	intr_event_execute_handlers (struct proc p, struct intr_event ie)

static void	ithread_execute_handlers (struct proc p, struct intr_event ie)

static void	ithread_loop (void *arg)

int	intr_event_handle (struct intr_event ie, struct trapframe frame)

static void	start_softintr (void *dummy)

	SYSINIT (start_softintr, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softintr, NULL)

static int	sysctl_intrnames (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_hw, OID_AUTO, intrnames, CTLTYPE_OPAQUE\|CTLFLAG_RD\|CTLFLAG_MPSAFE, NULL, 0, sysctl_intrnames, "", "Interrupt Names")

static int	sysctl_intrcnt (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_hw, OID_AUTO, intrcnt, CTLTYPE_OPAQUE\|CTLFLAG_RD\|CTLFLAG_MPSAFE, NULL, 0, sysctl_intrcnt, "", "Interrupt Counts")

Variables
struct intr_event *	clk_intr_event

struct intr_event *	tty_intr_event

struct proc *	intrproc

static int	intr_storm_threshold = 0

static int	intr_epoch_batch = 1000

Macro Definition Documentation

◆ IT_DEAD

#define IT_DEAD 0x000001 /* Thread is waiting to exit. */

Definition at line 83 of file kern_intr.c.

◆ IT_WAIT

#define IT_WAIT 0x000002 /* Thread is waiting for completion. */

Definition at line 84 of file kern_intr.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _intr_drain()

void _intr_drain ( int irq )

Definition at line 816 of file kern_intr.c.

References intr_lookup(), intr_thread::it_flags, intr_thread::it_thread, and IT_WAIT.

Here is the call graph for this function:

◆ _intr_event_bind()

static int _intr_event_bind	(	struct intr_event *	ie,
		int	cpu,
		bool	bindirq,
		bool	bindithread
	)

static

Definition at line 325 of file kern_intr.c.

References cpuset_setithread(), and priv_check().

Referenced by intr_event_bind(), intr_event_bind_irqonly(), and intr_event_bind_ithread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_add_handler()

int intr_event_add_handler	(	struct intr_event *	ie,
		const char *	name,
		driver_filter_t	filter,
		driver_intr_t	handler,
		void *	arg,
		u_char	pri,
		enum intr_type	flags,
		void **	cookiep
	)

Definition at line 598 of file kern_intr.c.

References flags, free(), intr_event_update(), intr_thread::it_event, ithread_create(), ithread_update(), malloc(), name, and wakeup().

Referenced by isrc_add_handler(), and swi_add().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_barrier()

static void intr_event_barrier ( struct intr_event * ie )

static

Definition at line 752 of file kern_intr.c.

References phase.

Referenced by intr_event_remove_handler(), and intr_handler_barrier().

Here is the caller graph for this function:

◆ intr_event_bind()

int intr_event_bind	(	struct intr_event *	ie,
		int	cpu
	)

Definition at line 387 of file kern_intr.c.

References _intr_event_bind().

Referenced by intr_setaffinity().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_bind_irqonly()

int intr_event_bind_irqonly	(	struct intr_event *	ie,
		int	cpu
	)

Definition at line 398 of file kern_intr.c.

References _intr_event_bind().

Referenced by intr_setaffinity().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_bind_ithread()

int intr_event_bind_ithread	(	struct intr_event *	ie,
		int	cpu
	)

Definition at line 408 of file kern_intr.c.

References _intr_event_bind().

Referenced by intr_setaffinity().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_bind_ithread_cpuset()

int intr_event_bind_ithread_cpuset	(	struct intr_event *	ie,
		cpuset_t *	cs
	)

Definition at line 418 of file kern_intr.c.

References cpuset_setthread().

Here is the call graph for this function:

◆ intr_event_create()

int intr_event_create	(	struct intr_event **	event,
		void *	source,
		int	flags,
		int	irq,
		void()(void )	pre_ithread,
		void()(void )	post_ithread,
		void()(void )	post_filter,
		int()(void , int)	assign_cpu,
		const char *	fmt,
			...
	)

Definition at line 282 of file kern_intr.c.

References flags, malloc(), post_filter, post_ithread, pre_ithread, and vsnprintf().

Referenced by isrc_event_create(), and swi_add().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_describe_handler()

int intr_event_describe_handler	(	struct intr_event *	ie,
		void *	cookie,
		const char *	descr
	)

Definition at line 677 of file kern_intr.c.

References intr_event_update(), panic(), and start.

Referenced by intr_describe_irq().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_destroy()

int intr_event_destroy ( struct intr_event * ie )

Definition at line 533 of file kern_intr.c.

References free(), and ithread_destroy().

Referenced by isrc_event_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_execute_handlers()

static void intr_event_execute_handlers	(	struct proc *	p,
		struct intr_event *	ie
	)

static

Definition at line 1142 of file kern_intr.c.

References Giant, and wakeup().

Referenced by ithread_execute_handlers().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_handle()

int intr_event_handle	(	struct intr_event *	ie,
		struct trapframe *	frame
	)

Definition at line 1356 of file kern_intr.c.

References intr_event_schedule_thread(), and phase.

Referenced by hardclock(), and intr_isrc_dispatch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_remove_handler()

int intr_event_remove_handler ( void * cookie )

Definition at line 848 of file kern_intr.c.

References free(), intr_event_barrier(), intr_event_schedule_thread(), intr_event_update(), ithread_destroy(), and panic().

Referenced by intr_setup_irq(), intr_teardown_irq(), and swi_remove().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_resume_handler()

int intr_event_resume_handler ( void * cookie )

Definition at line 949 of file kern_intr.c.

References intr_handler_barrier().

Here is the call graph for this function:

◆ intr_event_schedule_thread()

static int intr_event_schedule_thread	(	struct intr_event *	ie,
		struct trapframe *	frame
	)

static

Definition at line 973 of file kern_intr.c.

References intr_entropy::event, intr_thread::it_need, intr_thread::it_thread, intr_thread::it_waiting, sched_add(), and intr_entropy::td.

Referenced by intr_event_handle(), intr_event_remove_handler(), intr_handler_barrier(), and swi_sched().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_event_suspend_handler()

int intr_event_suspend_handler ( void * cookie )

Definition at line 930 of file kern_intr.c.

References intr_handler_barrier().

Here is the call graph for this function:

◆ intr_event_update()

static void intr_event_update ( struct intr_event * ie )

static

Definition at line 216 of file kern_intr.c.

References flags, and ithread_update().

Referenced by intr_event_add_handler(), intr_event_describe_handler(), and intr_event_remove_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_getaffinity()

int intr_getaffinity	(	int	irq,
		int	mode,
		void *	m
	)

Definition at line 486 of file kern_intr.c.

References cpuset_root, cpuset_which(), intr_lookup(), mask, and mode.

Referenced by kern_cpuset_getaffinity().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_handler_barrier()

static void intr_handler_barrier ( struct intr_handler * handler )

static

Definition at line 787 of file kern_intr.c.

References intr_event_barrier(), and intr_event_schedule_thread().

Referenced by intr_event_resume_handler(), and intr_event_suspend_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ intr_handler_source()

void * intr_handler_source ( void * cookie )

Definition at line 732 of file kern_intr.c.

Referenced by intr_teardown_irq().

Here is the caller graph for this function:

◆ intr_lookup()

static struct intr_event * intr_lookup ( int irq )

static

Definition at line 434 of file kern_intr.c.

Referenced by _intr_drain(), intr_getaffinity(), and intr_setaffinity().

Here is the caller graph for this function:

◆ intr_setaffinity()

int intr_setaffinity	(	int	irq,
		int	mode,
		void *	m
	)

Definition at line 449 of file kern_intr.c.

References cpuset_root, intr_event_bind(), intr_event_bind_irqonly(), intr_event_bind_ithread(), intr_lookup(), mask, and mode.

Referenced by kern_cpuset_setaffinity().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ithread_create()

static struct intr_thread * ithread_create ( const char * name )

static

Definition at line 558 of file kern_intr.c.

References intrproc, intr_thread::it_thread, ithread_loop(), kproc_kthread_add(), malloc(), name, panic(), and sched_class().

Referenced by intr_event_add_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ithread_destroy()

static void ithread_destroy ( struct intr_thread * ithread )

static

Definition at line 582 of file kern_intr.c.

References IT_DEAD, intr_thread::it_event, intr_thread::it_flags, intr_thread::it_thread, and sched_add().

Referenced by intr_event_destroy(), and intr_event_remove_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ithread_execute_handlers()

static void ithread_execute_handlers	(	struct proc *	p,
		struct intr_event *	ie
	)

static

Definition at line 1212 of file kern_intr.c.

References intr_event_execute_handlers(), intr_storm_threshold, ppsratecheck(), and printf().

Referenced by ithread_loop().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ithread_loop()

static void ithread_loop ( void * arg )

static

Definition at line 1256 of file kern_intr.c.

References free(), Giant, intr_epoch_batch, IT_DEAD, intr_thread::it_event, intr_thread::it_flags, intr_thread::it_need, intr_thread::it_thread, IT_WAIT, ithread_execute_handlers(), kthread_exit(), mi_switch(), and wakeup().

Referenced by ithread_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ithread_update()

static void ithread_update ( struct intr_thread * ithd )

static

Definition at line 186 of file kern_intr.c.

References intr_thread::it_event, intr_thread::it_thread, and sched_prio().

Referenced by intr_event_add_handler(), and intr_event_update().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_ITHREAD	,
		"ithread"	,
		"Interrupt Threads"
	)

static

◆ start_softintr()

static void start_softintr ( void * dummy )

static

Definition at line 1635 of file kern_intr.c.

References clk_intr_event, panic(), and swi_add().

Here is the call graph for this function:

◆ swi_add()

int swi_add	(	struct intr_event **	eventp,
		const char *	name,
		driver_intr_t	handler,
		void *	arg,
		int	pri,
		enum intr_type	flags,
		void **	cookiep
	)

Definition at line 1056 of file kern_intr.c.

References flags, intr_event_add_handler(), intr_event_create(), name, and swi_assign_cpu().

Referenced by start_busdma_swi(), and start_softintr().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ swi_assign_cpu()

static int swi_assign_cpu	(	void *	arg,
		int	cpu
	)

static

Definition at line 1045 of file kern_intr.c.

Referenced by swi_add().

Here is the caller graph for this function:

◆ swi_remove()

int swi_remove ( void * cookie )

Definition at line 1135 of file kern_intr.c.

References intr_event_remove_handler().

Here is the call graph for this function:

◆ swi_sched()

void swi_sched	(	void *	cookie,
		int	flags
	)

Definition at line 1089 of file kern_intr.c.

References clk_intr_event, intr_entropy::event, flags, intr_event_schedule_thread(), and intr_entropy::td.

Referenced by free_bounce_page(), taskqueue_fast_enqueue(), taskqueue_swi_enqueue(), and taskqueue_swi_giant_enqueue().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ SYSCTL_INT() [1/2]

SYSCTL_INT	(	_hw	,
		OID_AUTO	,
		intr_epoch_batch	,
		CTLFLAG_RWTUN	,
		&	intr_epoch_batch,
		0	,
		"Maximum interrupt handler executions without re-entering epoch(9)"
	)

◆ SYSCTL_INT() [2/2]

SYSCTL_INT	(	_hw	,
		OID_AUTO	,
		intr_storm_threshold	,
		CTLFLAG_RWTUN	,
		&	intr_storm_threshold,
		0	,
		"Number of consecutive interrupts before storm protection is enabled"
	)

◆ sysctl_intrcnt()

static int sysctl_intrcnt ( SYSCTL_HANDLER_ARGS )

static

Definition at line 1666 of file kern_intr.c.

References free(), malloc(), and sysctl_handle_opaque().

Here is the call graph for this function:

◆ sysctl_intrnames()

static int sysctl_intrnames ( SYSCTL_HANDLER_ARGS )

static

Definition at line 1655 of file kern_intr.c.

References sysctl_handle_opaque().

Here is the call graph for this function:

◆ SYSCTL_PROC() [1/2]

SYSCTL_PROC	(	_hw	,
		OID_AUTO	,
		intrcnt	,
		CTLTYPE_OPAQUE\|CTLFLAG_RD\|	CTLFLAG_MPSAFE,
		NULL	,
		0	,
		sysctl_intrcnt	,
		""	,
		"Interrupt Counts"
	)

◆ SYSCTL_PROC() [2/2]

SYSCTL_PROC	(	_hw	,
		OID_AUTO	,
		intrnames	,
		CTLTYPE_OPAQUE\|CTLFLAG_RD\|	CTLFLAG_MPSAFE,
		NULL	,
		0	,
		sysctl_intrnames	,
		""	,
		"Interrupt Names"
	)

◆ SYSINIT()

SYSINIT	(	start_softintr	,
		SI_SUB_SOFTINTR	,
		SI_ORDER_FIRST	,
		start_softintr	,
		NULL
	)

◆ TAILQ_HEAD()

static TAILQ_HEAD ( intr_event )

static

Definition at line 111 of file kern_intr.c.

Variable Documentation

◆ clk_intr_event

struct intr_event* clk_intr_event

Definition at line 91 of file kern_intr.c.

Referenced by hardclock(), start_softintr(), and swi_sched().

◆ intr_epoch_batch

int intr_epoch_batch = 1000

static

Definition at line 101 of file kern_intr.c.

Referenced by ithread_loop().

◆ intr_storm_threshold

int intr_storm_threshold = 0

static

Definition at line 97 of file kern_intr.c.

Referenced by ithread_execute_handlers().

◆ intrproc

struct proc* intrproc

Definition at line 93 of file kern_intr.c.

Referenced by ithread_create().

◆ tty_intr_event

struct intr_event* tty_intr_event

Definition at line 92 of file kern_intr.c.

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ IT_DEAD

◆ IT_WAIT

Function Documentation

◆ __FBSDID()

◆ _intr_drain()

◆ _intr_event_bind()

◆ intr_event_add_handler()

◆ intr_event_barrier()

◆ intr_event_bind()

◆ intr_event_bind_irqonly()

◆ intr_event_bind_ithread()

◆ intr_event_bind_ithread_cpuset()

◆ intr_event_create()

◆ intr_event_describe_handler()

◆ intr_event_destroy()

◆ intr_event_execute_handlers()

◆ intr_event_handle()

◆ intr_event_remove_handler()

◆ intr_event_resume_handler()

◆ intr_event_schedule_thread()

◆ intr_event_suspend_handler()

◆ intr_event_update()

◆ intr_getaffinity()

◆ intr_handler_barrier()

◆ intr_handler_source()

◆ intr_lookup()

◆ intr_setaffinity()

◆ ithread_create()

◆ ithread_destroy()

◆ ithread_execute_handlers()

◆ ithread_loop()

◆ ithread_update()

◆ MALLOC_DEFINE()

◆ start_softintr()

◆ swi_add()

◆ swi_assign_cpu()

◆ swi_remove()

◆ swi_sched()

◆ SYSCTL_INT() [1/2]

◆ SYSCTL_INT() [2/2]

◆ sysctl_intrcnt()

◆ sysctl_intrnames()

◆ SYSCTL_PROC() [1/2]

◆ SYSCTL_PROC() [2/2]

◆ SYSINIT()

◆ TAILQ_HEAD()

Variable Documentation

◆ clk_intr_event

◆ intr_epoch_batch

◆ intr_storm_threshold

◆ intrproc

◆ tty_intr_event