#include <sys/cdefs.h>
#include "opt_vm.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/blockcount.h>
#include <sys/eventhandler.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/ktr.h>
#include <sys/mount.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/smp.h>
#include <sys/time.h>
#include <sys/vnode.h>
#include <sys/vmmeter.h>
#include <sys/rwlock.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_map.h>
#include <vm/vm_pageout.h>
#include <vm/vm_pager.h>
#include <vm/vm_phys.h>
#include <vm/vm_pagequeue.h>
#include <vm/swap_pager.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>

Include dependency graph for vm_pageout.c:

Data Structures
struct	scan_state

Macros
#define	VM_LAUNDER_RATE 10

#define	VM_INACT_SCAN_RATE 10

Functions
	__FBSDID ("$FreeBSD$")

static void	vm_pageout (void)

static void	vm_pageout_init (void)

static int	vm_pageout_clean (vm_page_t m, int *numpagedout)

static int	vm_pageout_cluster (vm_page_t m)

static void	vm_pageout_mightbe_oom (struct vm_domain *vmd, int page_shortage, int starting_page_shortage)

	SYSINIT (pagedaemon_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_FIRST, vm_pageout_init, NULL)

	SYSINIT (pagedaemon, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, &page_kp)

	SDT_PROVIDER_DEFINE (vm)

	SDT_PROBE_DEFINE (vm,,, vm__lowmem_scan)

	SYSCTL_INT (_vm, OID_AUTO, panic_on_oom, CTLFLAG_RWTUN, &vm_panic_on_oom, 0, "Panic on the given number of out-of-memory errors instead of " "killing the largest process")

	SYSCTL_INT (_vm, OID_AUTO, pageout_update_period, CTLFLAG_RWTUN, &vm_pageout_update_period, 0, "Maximum active LRU update period")

	SYSCTL_INT (_vm, OID_AUTO, pageout_cpus_per_thread, CTLFLAG_RDTUN, &pageout_cpus_per_thread, 0, "Number of CPUs per pagedaemon worker thread")

	SYSCTL_INT (_vm, OID_AUTO, lowmem_period, CTLFLAG_RWTUN, &lowmem_period, 0, "Low memory callback period")

	SYSCTL_INT (_vm, OID_AUTO, disable_swapspace_pageouts, CTLFLAG_RWTUN, &disable_swap_pageouts, 0, "Disallow swapout of dirty pages")

	SYSCTL_INT (_vm, OID_AUTO, pageout_lock_miss, CTLFLAG_RD, &pageout_lock_miss, 0, "vget() lock misses during pageout")

	SYSCTL_INT (_vm, OID_AUTO, pageout_oom_seq, CTLFLAG_RWTUN, &vm_pageout_oom_seq, 0, "back-to-back calls to oom detector to start OOM")

	SYSCTL_INT (_vm, OID_AUTO, act_scan_laundry_weight, CTLFLAG_RWTUN, &act_scan_laundry_weight, 0, "weight given to clean vs. dirty pages in active queue scans")

	SYSCTL_UINT (_vm, OID_AUTO, background_launder_rate, CTLFLAG_RWTUN, &vm_background_launder_rate, 0, "background laundering rate, in kilobytes per second")

	SYSCTL_UINT (_vm, OID_AUTO, background_launder_max, CTLFLAG_RWTUN, &vm_background_launder_max, 0, "background laundering cap, in kilobytes")

	SYSCTL_ULONG (_vm, OID_AUTO, max_user_wired, CTLFLAG_RW, &vm_page_max_user_wired, 0, "system-wide limit to user-wired page count")

static u_int	isqrt (u_int num)

static int	vm_pageout_launder (struct vm_domain *vmd, int launder, bool in_shortfall)

static void	vm_pageout_laundry_worker (void *arg)

static void	vm_pageout_init_scan (struct scan_state ss, struct vm_pagequeue pq, vm_page_t marker, vm_page_t after, int maxscan)

static void	vm_pageout_end_scan (struct scan_state *ss)

static __always_inline void	vm_pageout_collect_batch (struct scan_state *ss, const bool dequeue)

static __always_inline vm_page_t	vm_pageout_next (struct scan_state *ss, const bool dequeue)

static __always_inline bool	vm_pageout_defer (vm_page_t m, const uint8_t queue, const bool enqueued)

int	vm_pageout_flush (vm_page_t mc, int count, int flags, int mreq, int prunlen, boolean_t *eio)

static void	vm_pageout_swapon (void arg __unused, struct swdevt sp __unused)

static void	vm_pageout_swapoff (void arg __unused, struct swdevt sp __unused)

static int	vm_pageout_active_target (struct vm_domain *vmd)

static void	vm_pageout_scan_active (struct vm_domain *vmd, int page_shortage)

static int	vm_pageout_reinsert_inactive_page (struct vm_pagequeue *pq, vm_page_t marker, vm_page_t m)

static void	vm_pageout_reinsert_inactive (struct scan_state ss, struct vm_batchqueue bq, vm_page_t m)

static void	vm_pageout_scan_inactive (struct vm_domain *vmd, int page_shortage)

static int	vm_pageout_inactive_dispatch (struct vm_domain *vmd, int shortage)

static int	vm_pageout_inactive (struct vm_domain vmd, int shortage, int addl_shortage)

static long	vm_pageout_oom_pagecount (struct vmspace *vmspace)

	SYSCTL_INT (_vm, OID_AUTO, oom_pf_secs, CTLFLAG_RWTUN, &vm_oom_pf_secs, 0, "")

void	vm_pageout_oom (int shortage)

static bool	vm_pageout_lowmem (void)

static void	vm_pageout_worker (void *arg)

static void	vm_pageout_helper (void *arg)

static int	get_pageout_threads_per_domain (const struct vm_domain *vmd)

static void	vm_pageout_init_domain (int domain)

void	pagedaemon_wakeup (int domain)

Variables
struct proc *	pageproc

static struct kproc_desc	page_kp

static int	swapdev_enabled

int	vm_pageout_page_count = 32

static int	vm_panic_on_oom = 0

static int	vm_pageout_update_period

static int	pageout_cpus_per_thread = 16

static int	lowmem_period = 10

static int	disable_swap_pageouts

static int	pageout_lock_miss

static int	vm_pageout_oom_seq = 12

static int	act_scan_laundry_weight = 3

static u_int	vm_background_launder_rate = 4096

static u_int	vm_background_launder_max = 20 * 1024

u_long	vm_page_max_user_wired

static int	vm_pageout_oom_vote

static int	vm_oom_ratelim_last

static int	vm_oom_pf_secs = 10

static struct mtx	vm_oom_ratelim_mtx

Macro Definition Documentation

◆ VM_INACT_SCAN_RATE

#define VM_INACT_SCAN_RATE 10

Definition at line 149 of file vm_pageout.c.

◆ VM_LAUNDER_RATE

#define VM_LAUNDER_RATE 10

Definition at line 148 of file vm_pageout.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ get_pageout_threads_per_domain()

static int get_pageout_threads_per_domain ( const struct vm_domain * vmd )

static

Definition at line 2214 of file vm_pageout.c.

References pageout_cpus_per_thread, VM_DOMAIN_EMPTY, vm_ndomains, and vm_domain::vmd_domain.

Referenced by vm_pageout_init_domain().

Here is the caller graph for this function:

◆ isqrt()

static u_int isqrt ( u_int num )

static

Definition at line 965 of file vm_pageout.c.

Referenced by vm_pageout_laundry_worker().

Here is the caller graph for this function:

◆ pagedaemon_wakeup()

void pagedaemon_wakeup ( int domain )

Definition at line 2407 of file vm_pageout.c.

References pageproc, VM_DOMAIN, vm_domain_pageout_assert_unlocked, vm_domain_pageout_lock, vm_domain_pageout_unlock, and vm_domain::vmd_pageout_wanted.

Referenced by _vm_domain_allocate().

Here is the caller graph for this function:

◆ SDT_PROBE_DEFINE()

SDT_PROBE_DEFINE	(	vm	,
		vm__lowmem_scan
	)

◆ SDT_PROVIDER_DEFINE()

SDT_PROVIDER_DEFINE ( vm )

◆ SYSCTL_INT() [1/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		act_scan_laundry_weight	,
		CTLFLAG_RWTUN	,
		&	act_scan_laundry_weight,
		0	,
		"weight given to clean vs. dirty pages in active queue scans"
	)

◆ SYSCTL_INT() [2/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		disable_swapspace_pageouts	,
		CTLFLAG_RWTUN	,
		&	disable_swap_pageouts,
		0	,
		"Disallow swapout of dirty pages"
	)

◆ SYSCTL_INT() [3/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		lowmem_period	,
		CTLFLAG_RWTUN	,
		&	lowmem_period,
		0	,
		"Low memory callback period"
	)

◆ SYSCTL_INT() [4/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		oom_pf_secs	,
		CTLFLAG_RWTUN	,
		&	vm_oom_pf_secs,
		0	,
		""
	)

◆ SYSCTL_INT() [5/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		pageout_cpus_per_thread	,
		CTLFLAG_RDTUN	,
		&	pageout_cpus_per_thread,
		0	,
		"Number of CPUs per pagedaemon worker thread"
	)

◆ SYSCTL_INT() [6/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		pageout_lock_miss	,
		CTLFLAG_RD	,
		&	pageout_lock_miss,
		0	,
		"vget() lock misses during pageout"
	)

◆ SYSCTL_INT() [7/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		pageout_oom_seq	,
		CTLFLAG_RWTUN	,
		&	vm_pageout_oom_seq,
		0	,
		"back-to-back calls to oom detector to start OOM"
	)

◆ SYSCTL_INT() [8/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		pageout_update_period	,
		CTLFLAG_RWTUN	,
		&	vm_pageout_update_period,
		0	,
		"Maximum active LRU update period"
	)

◆ SYSCTL_INT() [9/9]

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		panic_on_oom	,
		CTLFLAG_RWTUN	,
		&	vm_panic_on_oom,
		0	,
		"Panic on the given number of out-of-memory errors instead of " "killing the largest process"
	)

◆ SYSCTL_UINT() [1/2]

SYSCTL_UINT	(	_vm	,
		OID_AUTO	,
		background_launder_max	,
		CTLFLAG_RWTUN	,
		&	vm_background_launder_max,
		0	,
		"background laundering	cap,
		in kilobytes"
	)

◆ SYSCTL_UINT() [2/2]

SYSCTL_UINT	(	_vm	,
		OID_AUTO	,
		background_launder_rate	,
		CTLFLAG_RWTUN	,
		&	vm_background_launder_rate,
		0	,
		"background laundering	rate,
		in kilobytes per second"
	)

◆ SYSCTL_ULONG()

SYSCTL_ULONG	(	_vm	,
		OID_AUTO	,
		max_user_wired	,
		CTLFLAG_RW	,
		&	vm_page_max_user_wired,
		0	,
		"system-wide limit to user-wired page count"
	)

◆ SYSINIT() [1/2]

SYSINIT	(	pagedaemon	,
		SI_SUB_KTHREAD_PAGE	,
		SI_ORDER_SECOND	,
		kproc_start	,
		&	page_kp
	)

◆ SYSINIT() [2/2]

SYSINIT	(	pagedaemon_init	,
		SI_SUB_KTHREAD_PAGE	,
		SI_ORDER_FIRST	,
		vm_pageout_init	,
		NULL
	)

◆ vm_pageout()

static void vm_pageout ( void )

static

Definition at line 2354 of file vm_pageout.c.

References swap_pager_swap_init(), uma_reclaim_worker(), VM_DOMAIN, VM_DOMAIN_EMPTY, vm_ndomains, vm_oom_ratelim_mtx, vm_pageout_helper(), vm_pageout_laundry_worker(), and vm_pageout_worker().

Here is the call graph for this function:

◆ vm_pageout_active_target()

static int vm_pageout_active_target ( struct vm_domain * vmd )

static

Definition at line 1155 of file vm_pageout.c.

References act_scan_laundry_weight, vm_pagequeue::pq_cnt, PQ_INACTIVE, PQ_LAUNDRY, vm_paging_target(), vm_domain::vmd_inactive_target, and vm_domain::vmd_pagequeues.

Referenced by vm_pageout_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_clean()

static int vm_pageout_clean	(	vm_page_t	m,
		int *	numpagedout
	)

static

Definition at line 604 of file vm_pageout.c.

References OBJT_VNODE, vm_object::type, VM_OBJECT_ASSERT_WLOCKED, vm_object_deallocate(), vm_object_reference_locked(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_in_laundry(), vm_page_try_remove_write(), vm_page_tryxbusy(), vm_page_xunbusy, and vm_pageout_cluster().

Referenced by vm_pageout_launder().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_cluster()

static int vm_pageout_cluster ( vm_page_t m )

static

Definition at line 352 of file vm_pageout.c.

References VM_OBJECT_ASSERT_WLOCKED, vm_page_assert_xbusied, vm_page_in_laundry(), vm_page_next(), vm_page_prev(), vm_page_test_dirty(), vm_page_try_remove_write(), vm_page_tryxbusy(), vm_page_wired(), vm_page_xunbusy, vm_pageout_flush(), vm_pageout_page_count, and VM_PAGER_PUT_NOREUSE.

Referenced by vm_pageout_clean().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_collect_batch()

static __always_inline void vm_pageout_collect_batch	(	struct scan_state *	ss,
		const bool	dequeue
	)

static

Definition at line 272 of file vm_pageout.c.

References scan_state::bq, vm_batchqueue::bq_cnt, scan_state::marker, scan_state::maxscan, PG_FICTITIOUS, PG_MARKER, PGA_ENQUEUED, scan_state::pq, vm_pagequeue::pq_pl, scan_state::scanned, vm_batchqueue_insert(), VM_BATCHQUEUE_SIZE, vm_page_aflag_clear(), vm_pagequeue_cnt_add(), vm_pagequeue_lock, vm_pagequeue_unlock, and VPO_UNMANAGED.

Referenced by vm_pageout_next().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_defer()

static __always_inline bool vm_pageout_defer	(	vm_page_t	m,
		const uint8_t	queue,
		const bool	enqueued
	)

static

Definition at line 331 of file vm_pageout.c.

References vm_page_astate::flags, PGA_ENQUEUED, PGA_QUEUE_OP_MASK, vm_page_astate::queue, vm_page_astate_load(), and vm_page_pqbatch_submit().

Referenced by vm_pageout_launder(), vm_pageout_scan_active(), and vm_pageout_scan_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_end_scan()

static void vm_pageout_end_scan ( struct scan_state * ss )

static

Definition at line 246 of file vm_pageout.c.

References scan_state::marker, PGA_ENQUEUED, scan_state::pq, vm_pagequeue::pq_pdpages, vm_pagequeue::pq_pl, scan_state::scanned, vm_page_aflag_clear(), and vm_pagequeue_assert_locked.

Referenced by vm_pageout_launder(), vm_pageout_scan_active(), and vm_pageout_scan_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_flush()

int vm_pageout_flush	(	vm_page_t *	mc,
		int	count,
		int	flags,
		int	mreq,
		int *	prunlen,
		boolean_t *	eio
	)

Definition at line 471 of file vm_pageout.c.

References vm_object::flags, OBJ_SWAP, PGA_WRITEABLE, VM_OBJECT_ASSERT_WLOCKED, vm_object_pip_add(), vm_object_pip_wakeup(), vm_page_activate(), vm_page_all_valid(), vm_page_busy_downgrade(), vm_page_deactivate_noreuse(), vm_page_in_laundry(), vm_page_sunbusy(), vm_page_undirty(), vm_page_unswappable(), VM_PAGER_AGAIN, VM_PAGER_BAD, VM_PAGER_ERROR, VM_PAGER_FAIL, VM_PAGER_OK, VM_PAGER_PEND, and vm_pager_put_pages().

Referenced by vm_object_page_collect_flush(), and vm_pageout_cluster().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_helper()

static void vm_pageout_helper ( void * arg )

static

Definition at line 2187 of file vm_pageout.c.

References VM_DOMAIN, vm_domain_pageout_lock, vm_domain_pageout_lockptr, vm_domain_pageout_unlock, vm_pageout_scan_inactive(), vm_domain::vmd_inactive_running, vm_domain::vmd_inactive_shortage, and vm_domain::vmd_inactive_starting.

Referenced by vm_pageout().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_inactive()

static int vm_pageout_inactive	(	struct vm_domain *	vmd,
		int	shortage,
		int *	addl_shortage
	)

static

Definition at line 1709 of file vm_pageout.c.

References vm_pagequeue::pq_cnt, PQ_LAUNDRY, swapdev_enabled, VM_LAUNDRY_BACKGROUND, VM_LAUNDRY_IDLE, VM_LAUNDRY_SHORTFALL, vm_pageout_inactive_dispatch(), vm_pageout_mightbe_oom(), vm_pagequeue_lock, vm_pagequeue_unlock, vm_swapout_run(), vm_swapout_run_idle(), vm_domain::vmd_addl_shortage, vm_domain::vmd_clean_pages_freed, vm_domain::vmd_laundry_request, vm_domain::vmd_pageout_deficit, and vm_domain::vmd_pagequeues.

Referenced by vm_pageout_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_inactive_dispatch()

static int vm_pageout_inactive_dispatch	(	struct vm_domain *	vmd,
		int	shortage
	)

static

Definition at line 1654 of file vm_pageout.c.

References vm_domain_pageout_lock, vm_domain_pageout_unlock, VM_INACT_SCAN_RATE, vm_pageout_scan_inactive(), vm_domain::vmd_inactive_freed, vm_domain::vmd_inactive_pps, vm_domain::vmd_inactive_running, vm_domain::vmd_inactive_shortage, vm_domain::vmd_inactive_starting, vm_domain::vmd_inactive_threads, and vm_domain::vmd_inactive_us.

Referenced by vm_pageout_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_init()

static void vm_pageout_init ( void )

static

Definition at line 2306 of file vm_pageout.c.

References vm_cnt, VM_DOMAIN, vm_ndomains, vm_page_max_user_wired, vm_pageout_init_domain(), vm_pageout_page_count, vm_pageout_update_period, vm_domain::vmd_free_min, vm_domain::vmd_free_reserved, vm_domain::vmd_free_severe, vm_domain::vmd_free_target, vm_domain::vmd_inactive_target, vm_domain::vmd_interrupt_free_min, and vm_domain::vmd_pageout_free_min.

Here is the call graph for this function:

◆ vm_pageout_init_domain()

static void vm_pageout_init_domain ( int domain )

static

Definition at line 2254 of file vm_pageout.c.

References get_pageout_threads_per_domain(), VM_DOMAIN, VM_INACT_SCAN_RATE, vm_pageout_page_count, vm_domain::vmd_background_launder_target, vm_domain::vmd_free_min, vm_domain::vmd_free_reserved, vm_domain::vmd_free_severe, vm_domain::vmd_free_target, vm_domain::vmd_inactive_target, vm_domain::vmd_inactive_threads, vm_domain::vmd_interrupt_free_min, vm_domain::vmd_oid, vm_domain::vmd_page_count, vm_domain::vmd_pageout_free_min, vm_domain::vmd_pageout_wakeup_thresh, and vm_domain::vmd_pid.

Referenced by vm_pageout_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_init_scan()

static void vm_pageout_init_scan	(	struct scan_state *	ss,
		struct vm_pagequeue *	pq,
		vm_page_t	marker,
		vm_page_t	after,
		int	maxscan
	)

static

Definition at line 223 of file vm_pageout.c.

References scan_state::bq, scan_state::marker, scan_state::maxscan, PGA_ENQUEUED, scan_state::pq, vm_pagequeue::pq_pl, scan_state::scanned, vm_batchqueue_init(), vm_page_aflag_set(), vm_pagequeue_assert_locked, and vm_pagequeue_unlock.

Referenced by vm_pageout_launder(), vm_pageout_scan_active(), and vm_pageout_scan_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_launder()

static int vm_pageout_launder	(	struct vm_domain *	vmd,
		int	launder,
		bool	in_shortfall
	)

static

Definition at line 721 of file vm_pageout.c.

References _vm_page_queue(), ACT_ADVANCE, ACT_MAX, disable_swap_pageouts, vm_object::flags, vm_page_astate::flags, OBJ_DEAD, OBJ_SWAP, OBJT_DEFAULT, pageout_lock_miss, PG_MARKER, PGA_REFERENCED, PGA_REQUEUE, pmap_ts_referenced(), PQ_ACTIVE, vm_pagequeue::pq_cnt, PQ_LAUNDRY, PQ_NONE, PQ_UNSWAPPABLE, vm_object::ref_count, scan_state::scanned, swapdev_enabled, vm_object::type, VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_astate_load(), vm_page_free(), vm_page_launder(), vm_page_none_valid(), vm_page_pqstate_commit(), vm_page_test_dirty(), vm_page_try_remove_all(), vm_page_tryxbusy(), vm_page_wired(), vm_page_xunbusy, vm_pageout_clean(), vm_pageout_defer(), vm_pageout_end_scan(), vm_pageout_init_scan(), vm_pageout_next(), vm_pagequeue_lock, vm_pagequeue_unlock, vm_domain::vmd_markers, and vm_domain::vmd_pagequeues.

Referenced by vm_pageout_laundry_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_laundry_worker()

static void vm_pageout_laundry_worker ( void * arg )

static

Definition at line 989 of file vm_pageout.c.

References isqrt(), vm_pagequeue::pq_cnt, PQ_INACTIVE, PQ_LAUNDRY, vm_background_launder_max, vm_background_launder_rate, VM_DOMAIN, VM_INACT_SCAN_RATE, VM_LAUNDER_RATE, VM_LAUNDRY_IDLE, VM_LAUNDRY_SHORTFALL, vm_laundry_target(), vm_pageout_launder(), vm_pageout_swapoff(), vm_pageout_swapon(), vm_pagequeue_lock, vm_pagequeue_lockptr, vm_pagequeue_unlock, vm_domain::vmd_background_launder_target, vm_domain::vmd_clean_pages_freed, vm_domain::vmd_free_min, vm_domain::vmd_free_target, vm_domain::vmd_laundry_request, vm_domain::vmd_pageout_deficit, vm_domain::vmd_pagequeues, and vm_domain::vmd_segs.

Referenced by vm_pageout().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_lowmem()

static bool vm_pageout_lowmem ( void )

static

Definition at line 2049 of file vm_pageout.c.

References lowmem_period, uma_reclaim(), UMA_RECLAIM_TRIM, uma_reclaim_wakeup(), and VM_LOW_PAGES.

Referenced by vm_pageout_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_mightbe_oom()

static void vm_pageout_mightbe_oom	(	struct vm_domain *	vmd,
		int	page_shortage,
		int	starting_page_shortage
	)

static

Definition at line 1794 of file vm_pageout.c.

References vm_ndomains, VM_OOM_MEM, vm_pageout_oom(), vm_pageout_oom_seq, vm_pageout_oom_vote, vm_domain::vmd_oom, and vm_domain::vmd_oom_seq.

Referenced by vm_pageout_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_next()

static __always_inline vm_page_t vm_pageout_next	(	struct scan_state *	ss,
		const bool	dequeue
	)

static

Definition at line 318 of file vm_pageout.c.

References scan_state::bq, vm_batchqueue::bq_cnt, vm_batchqueue_pop(), and vm_pageout_collect_batch().

Referenced by vm_pageout_launder(), vm_pageout_scan_active(), and vm_pageout_scan_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_oom()

void vm_pageout_oom ( int shortage )

Definition at line 1903 of file vm_pageout.c.

References swap_pager_avail, vmspace::vm_map, vm_map_trylock_read, vm_map_unlock_read, VM_OOM_MEM, VM_OOM_MEM_PF, vm_oom_pf_secs, vm_oom_ratelim_last, vm_oom_ratelim_mtx, VM_OOM_SWAPZ, vm_pageout_oom_pagecount(), vm_panic_on_oom, vmspace_acquire_ref(), vmspace_free(), and vmspace_swap_count().

Referenced by swp_pager_meta_build(), vm_fault_allocate_oom(), and vm_pageout_mightbe_oom().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_oom_pagecount()

static long vm_pageout_oom_pagecount ( struct vmspace * vmspace )

static

Definition at line 1869 of file vm_pageout.c.

References vm_map_entry::eflags, vm_object::flags, vm_map::lock, MAP_ENTRY_IS_SUB_MAP, MAP_ENTRY_NEEDS_COPY, OBJ_SWAP, vm_map_entry::object, OBJT_DEFAULT, OBJT_PHYS, OBJT_VNODE, vm_object::ref_count, vm_object::resident_page_count, vm_map::system_map, vm_object::type, vmspace::vm_map, VM_MAP_ENTRY_FOREACH, and vm_map_object::vm_object.

Referenced by vm_pageout_oom().

Here is the caller graph for this function:

◆ vm_pageout_reinsert_inactive()

static void vm_pageout_reinsert_inactive	(	struct scan_state *	ss,
		struct vm_batchqueue *	bq,
		vm_page_t	m
	)

static

Definition at line 1399 of file vm_pageout.c.

References scan_state::marker, scan_state::pq, vm_batchqueue_init(), vm_batchqueue_insert(), vm_batchqueue_pop(), vm_pageout_reinsert_inactive_page(), vm_pagequeue_cnt_add(), vm_pagequeue_lock, and vm_pagequeue_unlock.

Referenced by vm_pageout_scan_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_reinsert_inactive_page()

static int vm_pageout_reinsert_inactive_page	(	struct vm_pagequeue *	pq,
		vm_page_t	marker,
		vm_page_t	m
	)

static

Definition at line 1377 of file vm_pageout.c.

References vm_page_astate::flags, PGA_ENQUEUED, PQ_INACTIVE, vm_page_astate::queue, vm_page_aflag_set(), vm_page_astate_load(), and vm_pagequeue_assert_locked.

Referenced by vm_pageout_reinsert_inactive().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_scan_active()

static void vm_pageout_scan_active	(	struct vm_domain *	vmd,
		int	page_shortage
	)

static

Definition at line 1171 of file vm_pageout.c.

References _vm_page_queue(), ACT_ADVANCE, ACT_DECLINE, ACT_MAX, act_scan_laundry_weight, vm_page_astate::flags, scan_state::marker, PG_MARKER, PGA_REFERENCED, PGA_REQUEUE, PGA_SWAP_FREE, pmap_ts_referenced(), PQ_ACTIVE, vm_pagequeue::pq_cnt, PQ_INACTIVE, PQ_LAUNDRY, PQ_NONE, vm_pagequeue::pq_pl, vm_page_astate::queue, scan_state::scanned, VM_OBJECT_TRYWLOCK, VM_OBJECT_WUNLOCK, vm_page_astate_load(), vm_page_pqstate_commit(), vm_pageout_defer(), vm_pageout_end_scan(), vm_pageout_init_scan(), vm_pageout_next(), vm_pageout_update_period, vm_pagequeue_lock, vm_pagequeue_unlock, vm_pager_page_unswapped(), vm_domain::vmd_clock, vm_domain::vmd_last_active_scan, vm_domain::vmd_markers, and vm_domain::vmd_pagequeues.

Referenced by vm_pageout_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_scan_inactive()

static void vm_pageout_scan_inactive	(	struct vm_domain *	vmd,
		int	page_shortage
	)

static

Definition at line 1425 of file vm_pageout.c.

References _vm_page_queue(), ACT_ADVANCE, ACT_MAX, scan_state::bq, vm_object::flags, vm_page_astate::flags, OBJ_DEAD, PG_MARKER, PGA_REFERENCED, PGA_REQUEUE, PGA_SWAP_FREE, pmap_ts_referenced(), PQ_ACTIVE, vm_pagequeue::pq_cnt, PQ_INACTIVE, PQ_NONE, vm_object::ref_count, vm_batchqueue_init(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_astate_load(), vm_page_free(), vm_page_init_marker(), vm_page_launder(), vm_page_none_valid(), vm_page_pqstate_commit(), vm_page_test_dirty(), vm_page_try_remove_all(), vm_page_tryxbusy(), vm_page_wired(), vm_page_xunbusy, vm_pageout_defer(), vm_pageout_end_scan(), vm_pageout_init_scan(), vm_pageout_next(), vm_pageout_reinsert_inactive(), vm_pagequeue_lock, vm_pagequeue_unlock, vm_pager_page_unswapped(), vm_domain::vmd_addl_shortage, vm_domain::vmd_inactive_freed, vm_domain::vmd_inactive_us, and vm_domain::vmd_pagequeues.

Referenced by vm_pageout_helper(), and vm_pageout_inactive_dispatch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_swapoff()

static void vm_pageout_swapoff	(	void *arg	__unused,
		struct swdevt *sp	__unused
	)

static

Definition at line 588 of file vm_pageout.c.

References swap_pager_nswapdev(), and swapdev_enabled.

Referenced by vm_pageout_laundry_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vm_pageout_swapon()

static void vm_pageout_swapon	(	void *arg	__unused,
		struct swdevt *sp	__unused
	)

static

Definition at line 581 of file vm_pageout.c.

References swapdev_enabled.

Referenced by vm_pageout_laundry_worker().

Here is the caller graph for this function:

◆ vm_pageout_worker()

static void vm_pageout_worker ( void * arg )

static

Definition at line 2095 of file vm_pageout.c.

References VM_DOMAIN, vm_domain_pageout_lock, vm_domain_pageout_lockptr, vm_domain_pageout_unlock, VM_INACT_SCAN_RATE, vm_pageout_active_target(), vm_pageout_inactive(), vm_pageout_lowmem(), vm_pageout_scan_active(), vm_paging_needed(), vm_domain::vmd_last_active_scan, vm_domain::vmd_pageout_wanted, vm_domain::vmd_pid, and vm_domain::vmd_segs.

Referenced by vm_pageout().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ act_scan_laundry_weight

int act_scan_laundry_weight = 3

static

Definition at line 189 of file vm_pageout.c.

Referenced by vm_pageout_active_target(), and vm_pageout_scan_active().

◆ disable_swap_pageouts

int disable_swap_pageouts

static

Definition at line 174 of file vm_pageout.c.

Referenced by vm_pageout_launder().

◆ lowmem_period

int lowmem_period = 10

static

Definition at line 170 of file vm_pageout.c.

Referenced by vm_pageout_lowmem().

◆ page_kp

struct kproc_desc page_kp

static

Initial value:

= {
        "pagedaemon",
        vm_pageout,
        &pageproc
}

Definition at line 136 of file vm_pageout.c.

◆ pageout_cpus_per_thread

int pageout_cpus_per_thread = 16

static

Definition at line 165 of file vm_pageout.c.

Referenced by get_pageout_threads_per_domain().

◆ pageout_lock_miss

int pageout_lock_miss

static

Definition at line 179 of file vm_pageout.c.

Referenced by vm_pageout_launder().

◆ pageproc

struct proc* pageproc

Definition at line 134 of file vm_pageout.c.

Referenced by pagedaemon_wakeup(), swap_pager_putpages(), swblk_trie_alloc(), swp_pager_meta_build(), vm_domain_allocate(), vm_page_reclaim_contig_domain(), vm_page_zone_import(), vm_wait_domain(), and vm_wait_doms().

◆ swapdev_enabled

int swapdev_enabled

static

Definition at line 151 of file vm_pageout.c.

Referenced by vm_pageout_inactive(), vm_pageout_launder(), vm_pageout_swapoff(), and vm_pageout_swapon().

◆ vm_background_launder_max

u_int vm_background_launder_max = 20 * 1024

static

Definition at line 199 of file vm_pageout.c.

Referenced by vm_pageout_laundry_worker().

◆ vm_background_launder_rate

u_int vm_background_launder_rate = 4096

static

Definition at line 194 of file vm_pageout.c.

Referenced by vm_pageout_laundry_worker().

◆ vm_oom_pf_secs

int vm_oom_pf_secs = 10

static

Definition at line 1897 of file vm_pageout.c.

Referenced by vm_pageout_oom().

◆ vm_oom_ratelim_last

int vm_oom_ratelim_last

static

Definition at line 1896 of file vm_pageout.c.

Referenced by vm_pageout_oom().

◆ vm_oom_ratelim_mtx

struct mtx vm_oom_ratelim_mtx

static

Definition at line 1900 of file vm_pageout.c.

Referenced by vm_pageout(), and vm_pageout_oom().

◆ vm_page_max_user_wired

u_long vm_page_max_user_wired

Definition at line 204 of file vm_pageout.c.

Referenced by kern_mlock(), vm_pageout_init(), and vslock().

◆ vm_pageout_oom_seq

int vm_pageout_oom_seq = 12

static

Definition at line 184 of file vm_pageout.c.

Referenced by vm_pageout_mightbe_oom().

◆ vm_pageout_oom_vote

int vm_pageout_oom_vote

static

Definition at line 1786 of file vm_pageout.c.

Referenced by vm_pageout_mightbe_oom().

◆ vm_pageout_page_count

int vm_pageout_page_count = 32

Definition at line 152 of file vm_pageout.c.

Referenced by vm_object_page_collect_flush(), vm_pageout_cluster(), vm_pageout_init(), and vm_pageout_init_domain().

◆ vm_pageout_update_period

int vm_pageout_update_period

static

Definition at line 160 of file vm_pageout.c.

Referenced by vm_pageout_init(), and vm_pageout_scan_active().

◆ vm_panic_on_oom

int vm_panic_on_oom = 0

static

Definition at line 154 of file vm_pageout.c.

Referenced by vm_pageout_oom().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ VM_INACT_SCAN_RATE

◆ VM_LAUNDER_RATE

Function Documentation

◆ __FBSDID()

◆ get_pageout_threads_per_domain()

◆ isqrt()

◆ pagedaemon_wakeup()

◆ SDT_PROBE_DEFINE()

◆ SDT_PROVIDER_DEFINE()

◆ SYSCTL_INT() [1/9]

◆ SYSCTL_INT() [2/9]

◆ SYSCTL_INT() [3/9]

◆ SYSCTL_INT() [4/9]

◆ SYSCTL_INT() [5/9]

◆ SYSCTL_INT() [6/9]

◆ SYSCTL_INT() [7/9]

◆ SYSCTL_INT() [8/9]

◆ SYSCTL_INT() [9/9]

◆ SYSCTL_UINT() [1/2]

◆ SYSCTL_UINT() [2/2]

◆ SYSCTL_ULONG()

◆ SYSINIT() [1/2]

◆ SYSINIT() [2/2]

◆ vm_pageout()

◆ vm_pageout_active_target()

◆ vm_pageout_clean()

◆ vm_pageout_cluster()

◆ vm_pageout_collect_batch()

◆ vm_pageout_defer()

◆ vm_pageout_end_scan()

◆ vm_pageout_flush()

◆ vm_pageout_helper()

◆ vm_pageout_inactive()

◆ vm_pageout_inactive_dispatch()

◆ vm_pageout_init()

◆ vm_pageout_init_domain()

◆ vm_pageout_init_scan()

◆ vm_pageout_launder()

◆ vm_pageout_laundry_worker()

◆ vm_pageout_lowmem()

◆ vm_pageout_mightbe_oom()

◆ vm_pageout_next()

◆ vm_pageout_oom()

◆ vm_pageout_oom_pagecount()

◆ vm_pageout_reinsert_inactive()

◆ vm_pageout_reinsert_inactive_page()

◆ vm_pageout_scan_active()

◆ vm_pageout_scan_inactive()

◆ vm_pageout_swapoff()

◆ vm_pageout_swapon()

◆ vm_pageout_worker()

Variable Documentation

◆ act_scan_laundry_weight

◆ disable_swap_pageouts

◆ lowmem_period

◆ page_kp

◆ pageout_cpus_per_thread

◆ pageout_lock_miss

◆ pageproc

◆ swapdev_enabled

◆ vm_background_launder_max

◆ vm_background_launder_rate

◆ vm_oom_pf_secs

◆ vm_oom_ratelim_last

◆ vm_oom_ratelim_mtx

◆ vm_page_max_user_wired

◆ vm_pageout_oom_seq

◆ vm_pageout_oom_vote

◆ vm_pageout_page_count

◆ vm_pageout_update_period

◆ vm_panic_on_oom