#include <sys/cdefs.h>
#include "opt_device_polling.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/proc.h>
#include <sys/epoch.h>
#include <sys/eventhandler.h>
#include <sys/resourcevar.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/netisr.h>
#include <net/vnet.h>

Include dependency graph for kern_poll.c:

Data Structures
struct	pollrec

Macros
#define	MIN_POLL_BURST_MAX 10

#define	MAX_POLL_BURST_MAX 20000

#define	POLL_LIST_LEN 128

Functions
	__FBSDID ("$FreeBSD$")

void	hardclock_device_poll (void)

static	SYSCTL_NODE (_kern, OID_AUTO, polling, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "Device polling parameters")

	SYSCTL_UINT (_kern_polling, OID_AUTO, burst, CTLFLAG_RD, &poll_burst, 0, "Current polling burst size")

static int	poll_burst_max_sysctl (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern_polling, OID_AUTO, burst_max, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, sizeof(uint32_t), poll_burst_max_sysctl, "I", "Max Polling burst size")

static int	poll_each_burst_sysctl (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern_polling, OID_AUTO, each_burst, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, sizeof(uint32_t), poll_each_burst_sysctl, "I", "Max size of each burst")

	SYSCTL_UINT (_kern_polling, OID_AUTO, idle_poll, CTLFLAG_RW, &poll_in_idle_loop, 0, "Enable device polling in idle loop")

static int	user_frac_sysctl (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern_polling, OID_AUTO, user_frac, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, sizeof(uint32_t), user_frac_sysctl, "I", "Desired user fraction of cpu time")

static int	reg_frac_sysctl (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern_polling, OID_AUTO, reg_frac, CTLTYPE_UINT\|CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, sizeof(uint32_t), reg_frac_sysctl, "I", "Every this many cycles check registers")

	SYSCTL_UINT (_kern_polling, OID_AUTO, short_ticks, CTLFLAG_RD, &short_ticks, 0, "Hardclock ticks shorter than they should be")

	SYSCTL_UINT (_kern_polling, OID_AUTO, lost_polls, CTLFLAG_RD, &lost_polls, 0, "How many times we would have lost a poll tick")

	SYSCTL_UINT (_kern_polling, OID_AUTO, pending_polls, CTLFLAG_RD, &pending_polls, 0, "Do we need to poll again")

	SYSCTL_INT (_kern_polling, OID_AUTO, residual_burst, CTLFLAG_RD, &residual_burst, 0, "# of residual cycles in burst")

	SYSCTL_UINT (_kern_polling, OID_AUTO, handlers, CTLFLAG_RD, &poll_handlers, 0, "Number of registered poll handlers")

	SYSCTL_UINT (_kern_polling, OID_AUTO, phase, CTLFLAG_RD, &phase, 0, "Polling phase")

	SYSCTL_UINT (_kern_polling, OID_AUTO, suspect, CTLFLAG_RD, &suspect, 0, "suspect event")

	SYSCTL_UINT (_kern_polling, OID_AUTO, stalled, CTLFLAG_RD, &stalled, 0, "potential stalls")

	SYSCTL_UINT (_kern_polling, OID_AUTO, idlepoll_sleeping, CTLFLAG_RD, &idlepoll_sleeping, 0, "idlepoll is sleeping")

static void	poll_shutdown (void *arg, int howto)

static void	init_device_poll (void)

	SYSINIT (device_poll, SI_SUB_SOFTINTR, SI_ORDER_MIDDLE, init_device_poll, NULL)

static void	ether_poll (int count)

void	netisr_pollmore ()

void	netisr_poll (void)

int	ether_poll_register (poll_handler_t *h, if_t ifp)

int	ether_poll_deregister (if_t ifp)

static void	poll_idle (void)

	SYSINIT (idlepoll, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, kproc_start, &idlepoll_kp)

Variables
static struct mtx	poll_mtx

static uint32_t	poll_burst = 5

static uint32_t	poll_burst_max = 150

static uint32_t	poll_each_burst = 5

static int	netisr_poll_scheduled

static int	netisr_pollmore_scheduled

static int	poll_shutting_down

static uint32_t	poll_in_idle_loop =0

static uint32_t	user_frac = 50

static uint32_t	reg_frac_count = 0

static uint32_t	reg_frac = 20

static uint32_t	short_ticks

static uint32_t	lost_polls

static uint32_t	pending_polls

static int	residual_burst = 0

static uint32_t	poll_handlers

static uint32_t	phase

static uint32_t	suspect

static uint32_t	stalled

static uint32_t	idlepoll_sleeping

static struct pollrec	pr [POLL_LIST_LEN]

static struct timeval	poll_start_t

static struct proc *	idlepoll

static struct kproc_desc	idlepoll_kp

Macro Definition Documentation

◆ MAX_POLL_BURST_MAX

#define MAX_POLL_BURST_MAX 20000

Definition at line 98 of file kern_poll.c.

◆ MIN_POLL_BURST_MAX

#define MIN_POLL_BURST_MAX 10

Definition at line 97 of file kern_poll.c.

◆ POLL_LIST_LEN

#define POLL_LIST_LEN 128

Definition at line 255 of file kern_poll.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ ether_poll()

static void ether_poll ( int count )

static

Definition at line 337 of file kern_poll.c.

References count, poll_each_burst, poll_handlers, poll_mtx, and pr.

Referenced by poll_idle().

Here is the caller graph for this function:

◆ ether_poll_deregister()

int ether_poll_deregister ( if_t ifp )

Definition at line 526 of file kern_poll.c.

References pollrec::handler, pollrec::ifp, log(), poll_handlers, poll_mtx, and pr.

Here is the call graph for this function:

◆ ether_poll_register()

int ether_poll_register	(	poll_handler_t *	h,
		if_t	ifp
	)

Definition at line 479 of file kern_poll.c.

References pollrec::handler, idlepoll_sleeping, pollrec::ifp, log(), poll_handlers, POLL_LIST_LEN, poll_mtx, pr, and wakeup().

Here is the call graph for this function:

◆ hardclock_device_poll()

void hardclock_device_poll ( void )

Definition at line 294 of file kern_poll.c.

References hz, lost_polls, microuptime(), netisr_poll_scheduled, netisr_pollmore_scheduled, pending_polls, phase, poll_handlers, poll_shutting_down, short_ticks, stalled, and suspect.

Referenced by hardclock().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ init_device_poll()

static void init_device_poll ( void )

static

Definition at line 271 of file kern_poll.c.

References poll_mtx, and poll_shutdown().

Here is the call graph for this function:

◆ netisr_poll()

void netisr_poll ( void )

Definition at line 434 of file kern_poll.c.

References microuptime(), netisr_poll_scheduled, phase, poll_burst, poll_each_burst, poll_handlers, poll_mtx, poll_start_t, pr, reg_frac, reg_frac_count, and residual_burst.

Here is the call graph for this function:

◆ netisr_pollmore()

void netisr_pollmore ( )

Definition at line 374 of file kern_poll.c.

References hz, microuptime(), netisr_poll_scheduled, netisr_pollmore_scheduled, pending_polls, phase, poll_burst, poll_burst_max, poll_handlers, poll_mtx, poll_start_t, residual_burst, and user_frac.

Here is the call graph for this function:

◆ poll_burst_max_sysctl()

static int poll_burst_max_sysctl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 114 of file kern_poll.c.

References MAX_POLL_BURST_MAX, MIN_POLL_BURST_MAX, poll_burst, poll_burst_max, poll_each_burst, poll_mtx, and sysctl_handle_int().

Here is the call graph for this function:

◆ poll_each_burst_sysctl()

static int poll_each_burst_sysctl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 140 of file kern_poll.c.

References poll_burst_max, poll_each_burst, poll_mtx, and sysctl_handle_int().

Here is the call graph for this function:

◆ poll_idle()

static void poll_idle ( void )

static

Definition at line 553 of file kern_poll.c.

References ether_poll(), hz, idlepoll_sleeping, mi_switch(), poll_each_burst, poll_handlers, poll_in_idle_loop, and rtp_to_pri().

Here is the call graph for this function:

◆ poll_shutdown()

static void poll_shutdown	(	void *	arg,
		int	howto
	)

static

Definition at line 264 of file kern_poll.c.

References poll_shutting_down.

Referenced by init_device_poll().

Here is the caller graph for this function:

◆ reg_frac_sysctl()

static int reg_frac_sysctl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 195 of file kern_poll.c.

References hz, poll_mtx, reg_frac, reg_frac_count, and sysctl_handle_int().

Here is the call graph for this function:

◆ SYSCTL_INT()

SYSCTL_INT	(	_kern_polling	,
		OID_AUTO	,
		residual_burst	,
		CTLFLAG_RD	,
		&	residual_burst,
		0	,
		"# of residual cycles in burst"
	)

◆ SYSCTL_NODE()

static SYSCTL_NODE	(	_kern	,
		OID_AUTO	,
		polling	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"Device polling parameters"
	)

static

◆ SYSCTL_PROC() [1/4]

SYSCTL_PROC	(	_kern_polling	,
		OID_AUTO	,
		burst_max	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		sizeof(uint32_t)	,
		poll_burst_max_sysctl	,
		"I"	,
		"Max Polling burst size"
	)

◆ SYSCTL_PROC() [2/4]

SYSCTL_PROC	(	_kern_polling	,
		OID_AUTO	,
		each_burst	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		sizeof(uint32_t)	,
		poll_each_burst_sysctl	,
		"I"	,
		"Max size of each burst"
	)

◆ SYSCTL_PROC() [3/4]

SYSCTL_PROC	(	_kern_polling	,
		OID_AUTO	,
		reg_frac	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		sizeof(uint32_t)	,
		reg_frac_sysctl	,
		"I"	,
		"Every this many cycles check registers"
	)

◆ SYSCTL_PROC() [4/4]

SYSCTL_PROC	(	_kern_polling	,
		OID_AUTO	,
		user_frac	,
		CTLTYPE_UINT\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		sizeof(uint32_t)	,
		user_frac_sysctl	,
		"I"	,
		"Desired user fraction of cpu time"
	)

◆ SYSCTL_UINT() [1/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		burst	,
		CTLFLAG_RD	,
		&	poll_burst,
		0	,
		"Current polling burst size"
	)

◆ SYSCTL_UINT() [2/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		handlers	,
		CTLFLAG_RD	,
		&	poll_handlers,
		0	,
		"Number of registered poll handlers"
	)

◆ SYSCTL_UINT() [3/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		idle_poll	,
		CTLFLAG_RW	,
		&	poll_in_idle_loop,
		0	,
		"Enable device polling in idle loop"
	)

◆ SYSCTL_UINT() [4/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		idlepoll_sleeping	,
		CTLFLAG_RD	,
		&	idlepoll_sleeping,
		0	,
		"idlepoll is sleeping"
	)

◆ SYSCTL_UINT() [5/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		lost_polls	,
		CTLFLAG_RD	,
		&	lost_polls,
		0	,
		"How many times we would have lost a poll tick"
	)

◆ SYSCTL_UINT() [6/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		pending_polls	,
		CTLFLAG_RD	,
		&	pending_polls,
		0	,
		"Do we need to poll again"
	)

◆ SYSCTL_UINT() [7/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		phase	,
		CTLFLAG_RD	,
		&	phase,
		0	,
		"Polling phase"
	)

◆ SYSCTL_UINT() [8/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		short_ticks	,
		CTLFLAG_RD	,
		&	short_ticks,
		0	,
		"Hardclock ticks shorter than they should be"
	)

◆ SYSCTL_UINT() [9/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		stalled	,
		CTLFLAG_RD	,
		&	stalled,
		0	,
		"potential stalls"
	)

◆ SYSCTL_UINT() [10/10]

SYSCTL_UINT	(	_kern_polling	,
		OID_AUTO	,
		suspect	,
		CTLFLAG_RD	,
		&	suspect,
		0	,
		"suspect event"
	)

◆ SYSINIT() [1/2]

SYSINIT	(	device_poll	,
		SI_SUB_SOFTINTR	,
		SI_ORDER_MIDDLE	,
		init_device_poll	,
		NULL
	)

◆ SYSINIT() [2/2]

SYSINIT	(	idlepoll	,
		SI_SUB_KTHREAD_VM	,
		SI_ORDER_ANY	,
		kproc_start	,
		&	idlepoll_kp
	)

◆ user_frac_sysctl()

static int user_frac_sysctl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 171 of file kern_poll.c.

References poll_mtx, sysctl_handle_int(), and user_frac.

Here is the call graph for this function:

Variable Documentation

◆ idlepoll

struct proc* idlepoll

static

Definition at line 577 of file kern_poll.c.

◆ idlepoll_kp

struct kproc_desc idlepoll_kp

static

Initial value:

= {
         "idlepoll",
         poll_idle,
         &idlepoll
}

Definition at line 578 of file kern_poll.c.

◆ idlepoll_sleeping

uint32_t idlepoll_sleeping

static

Definition at line 251 of file kern_poll.c.

Referenced by ether_poll_register(), and poll_idle().

◆ lost_polls

uint32_t lost_polls

static

Definition at line 223 of file kern_poll.c.

Referenced by hardclock_device_poll().

◆ netisr_poll_scheduled

int netisr_poll_scheduled

static

Definition at line 110 of file kern_poll.c.

Referenced by hardclock_device_poll(), netisr_poll(), and netisr_pollmore().

◆ netisr_pollmore_scheduled

int netisr_pollmore_scheduled

static

Definition at line 111 of file kern_poll.c.

Referenced by hardclock_device_poll(), and netisr_pollmore().

◆ pending_polls

uint32_t pending_polls

static

Definition at line 227 of file kern_poll.c.

Referenced by hardclock_device_poll(), and netisr_pollmore().

◆ phase

uint32_t phase

static

Definition at line 239 of file kern_poll.c.

Referenced by hardclock_device_poll(), intr_event_barrier(), intr_event_handle(), netisr_poll(), netisr_pollmore(), vmem_fit(), vmem_xalloc(), and vmem_xalloc_nextfit().

◆ poll_burst

uint32_t poll_burst = 5

static

Definition at line 100 of file kern_poll.c.

Referenced by netisr_poll(), netisr_pollmore(), and poll_burst_max_sysctl().

◆ poll_burst_max

uint32_t poll_burst_max = 150

static

Definition at line 101 of file kern_poll.c.

Referenced by netisr_pollmore(), poll_burst_max_sysctl(), and poll_each_burst_sysctl().

◆ poll_each_burst

uint32_t poll_each_burst = 5

static

Definition at line 102 of file kern_poll.c.

Referenced by ether_poll(), netisr_poll(), poll_burst_max_sysctl(), poll_each_burst_sysctl(), and poll_idle().

◆ poll_handlers

uint32_t poll_handlers

static

Definition at line 235 of file kern_poll.c.

Referenced by ether_poll(), ether_poll_deregister(), ether_poll_register(), hardclock_device_poll(), netisr_poll(), netisr_pollmore(), and poll_idle().

◆ poll_in_idle_loop

uint32_t poll_in_idle_loop =0

static

Definition at line 166 of file kern_poll.c.

Referenced by poll_idle().

◆ poll_mtx

struct mtx poll_mtx

static

Definition at line 55 of file kern_poll.c.

Referenced by ether_poll(), ether_poll_deregister(), ether_poll_register(), init_device_poll(), netisr_poll(), netisr_pollmore(), poll_burst_max_sysctl(), poll_each_burst_sysctl(), reg_frac_sysctl(), and user_frac_sysctl().

◆ poll_shutting_down

int poll_shutting_down

static

Definition at line 112 of file kern_poll.c.

Referenced by hardclock_device_poll(), and poll_shutdown().

◆ poll_start_t

struct timeval poll_start_t

static

Definition at line 371 of file kern_poll.c.

Referenced by netisr_poll(), and netisr_pollmore().

◆ pr

struct pollrec pr[POLL_LIST_LEN]

static

Definition at line 261 of file kern_poll.c.

Referenced by _cv_timedwait_sbt(), _cv_timedwait_sig_sbt(), _sleep(), cache_fplookup_dotdot(), cpuset_create_root(), cpuset_which(), do_jail_attach(), domain_init(), ether_poll(), ether_poll_deregister(), ether_poll_register(), getcredhostname(), kern_jail_get(), kern_jail_set(), kern_setitimer(), mb_reclaim(), mountcheckdirs(), mqfs_prison_remove(), msg_find_prison(), msg_prison_check(), msg_prison_cleanup(), msg_prison_get(), msg_prison_remove(), msg_prison_set(), msginit(), msleep_spin_sbt(), netisr_poll(), pause_sbt(), pf_proto_register(), pf_proto_unregister(), pfctlinput(), pffasttimo(), pffindproto(), pffindtype(), pfslowtimo(), pr_usrreqs_init(), prison_canseemount(), prison_check_af(), prison_complete(), prison_deref(), prison_deref_kill(), prison_enforce_statfs(), prison_find(), prison_find_child(), prison_find_name(), prison_free(), prison_free_locked(), prison_free_not_last(), prison_hold(), prison_hold_locked(), prison_isalive(), prison_isvalid(), prison_lock_xlock(), prison_priv_check(), prison_proc_free(), prison_proc_free_not_last(), prison_proc_hold(), prison_set_allow(), prison_set_allow_locked(), sem_find_prison(), sem_prison_check(), sem_prison_cleanup(), sem_prison_get(), sem_prison_remove(), sem_prison_set(), seminit(), shm_find_prison(), shm_find_segment_by_key(), shm_prison_check(), shm_prison_cleanup(), shm_prison_get(), shm_prison_remove(), shm_prison_set(), shminit(), sleepq_set_timeout_sbt(), sofree(), soreceive_dgram(), soreceive_generic(), soreceive_rcvoob(), sorflush(), soshutdown(), sys_jail_attach(), sys_jail_remove(), sysctl_hostid(), sysctl_hostname(), sysctl_jail_default_level(), sysctl_jail_list(), sysctl_kern_securelvl(), sysctl_msqids(), sysctl_osreldate(), sysctl_osrelease(), sysctl_sema(), sysctl_shmsegs(), taskqueue_enqueue_timeout_sbt(), and vfs_lookup().

◆ reg_frac

uint32_t reg_frac = 20

static

Definition at line 194 of file kern_poll.c.

Referenced by netisr_poll(), and reg_frac_sysctl().

◆ reg_frac_count

uint32_t reg_frac_count = 0

static

Definition at line 193 of file kern_poll.c.

Referenced by netisr_poll(), and reg_frac_sysctl().

◆ residual_burst

int residual_burst = 0

static

Definition at line 231 of file kern_poll.c.

Referenced by netisr_poll(), and netisr_pollmore().

◆ short_ticks

uint32_t short_ticks

static

Definition at line 219 of file kern_poll.c.

Referenced by hardclock_device_poll().

◆ stalled

uint32_t stalled

static

Definition at line 247 of file kern_poll.c.

Referenced by hardclock_device_poll().

◆ suspect

uint32_t suspect

static

Definition at line 243 of file kern_poll.c.

Referenced by hardclock_device_poll().

◆ user_frac

uint32_t user_frac = 50

static

Definition at line 170 of file kern_poll.c.

Referenced by netisr_pollmore(), and user_frac_sysctl().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ MAX_POLL_BURST_MAX

◆ MIN_POLL_BURST_MAX

◆ POLL_LIST_LEN

Function Documentation

◆ __FBSDID()

◆ ether_poll()

◆ ether_poll_deregister()

◆ ether_poll_register()

◆ hardclock_device_poll()

◆ init_device_poll()

◆ netisr_poll()

◆ netisr_pollmore()

◆ poll_burst_max_sysctl()

◆ poll_each_burst_sysctl()

◆ poll_idle()

◆ poll_shutdown()

◆ reg_frac_sysctl()

◆ SYSCTL_INT()

◆ SYSCTL_NODE()

◆ SYSCTL_PROC() [1/4]

◆ SYSCTL_PROC() [2/4]

◆ SYSCTL_PROC() [3/4]

◆ SYSCTL_PROC() [4/4]

◆ SYSCTL_UINT() [1/10]

◆ SYSCTL_UINT() [2/10]

◆ SYSCTL_UINT() [3/10]

◆ SYSCTL_UINT() [4/10]

◆ SYSCTL_UINT() [5/10]

◆ SYSCTL_UINT() [6/10]

◆ SYSCTL_UINT() [7/10]

◆ SYSCTL_UINT() [8/10]

◆ SYSCTL_UINT() [9/10]

◆ SYSCTL_UINT() [10/10]

◆ SYSINIT() [1/2]

◆ SYSINIT() [2/2]

◆ user_frac_sysctl()

Variable Documentation

◆ idlepoll

◆ idlepoll_kp

◆ idlepoll_sleeping

◆ lost_polls

◆ netisr_poll_scheduled

◆ netisr_pollmore_scheduled

◆ pending_polls

◆ phase

◆ poll_burst

◆ poll_burst_max

◆ poll_each_burst

◆ poll_handlers

◆ poll_in_idle_loop

◆ poll_mtx

◆ poll_shutting_down

◆ poll_start_t

◆ pr

◆ reg_frac

◆ reg_frac_count

◆ residual_burst

◆ short_ticks

◆ stalled

◆ suspect

◆ user_frac