vm_pagequeue.h File Reference

#include <vm/uma.h>
#include <sys/_blockcount.h>
#include <sys/pidctrl.h>

Include dependency graph for vm_pagequeue.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	vm_pagequeue
struct	vm_batchqueue
struct	vm_domain
struct	vm_domain::vm_pgcache
struct	vm_pgcache

Macros
#define	VM_BATCHQUEUE_SIZE   7
#define	VM_DOMAIN(n)   (&vm_dom[(n)])
#define	VM_DOMAIN_EMPTY(n)   (vm_dom[(n)].vmd_page_count == 0)
#define	vm_pagequeue_assert_locked(pq)   mtx_assert(&(pq)->pq_mutex, MA_OWNED)
#define	vm_pagequeue_lock(pq)   mtx_lock(&(pq)->pq_mutex)
#define	vm_pagequeue_lockptr(pq)   (&(pq)->pq_mutex)
#define	vm_pagequeue_trylock(pq)   mtx_trylock(&(pq)->pq_mutex)
#define	vm_pagequeue_unlock(pq)   mtx_unlock(&(pq)->pq_mutex)
#define	vm_domain_free_assert_locked(n)    mtx_assert(vm_domain_free_lockptr((n)), MA_OWNED)
#define	vm_domain_free_assert_unlocked(n)    mtx_assert(vm_domain_free_lockptr((n)), MA_NOTOWNED)
#define	vm_domain_free_lock(d)    mtx_lock(vm_domain_free_lockptr((d)))
#define	vm_domain_free_lockptr(d)    (&(d)->vmd_free_mtx)
#define	vm_domain_free_trylock(d)    mtx_trylock(vm_domain_free_lockptr((d)))
#define	vm_domain_free_unlock(d)    mtx_unlock(vm_domain_free_lockptr((d)))
#define	vm_domain_pageout_lockptr(d)    (&(d)->vmd_pageout_mtx)
#define	vm_domain_pageout_assert_locked(n)    mtx_assert(vm_domain_pageout_lockptr((n)), MA_OWNED)
#define	vm_domain_pageout_assert_unlocked(n)    mtx_assert(vm_domain_pageout_lockptr((n)), MA_NOTOWNED)
#define	vm_domain_pageout_lock(d)    mtx_lock(vm_domain_pageout_lockptr((d)))
#define	vm_domain_pageout_unlock(d)    mtx_unlock(vm_domain_pageout_lockptr((d)))
#define	vm_pagequeue_cnt_inc(pq)   vm_pagequeue_cnt_add((pq), 1)
#define	vm_pagequeue_cnt_dec(pq)   vm_pagequeue_cnt_add((pq), -1)

Enumerations
enum	{ VM_LAUNDRY_IDLE = 0 , VM_LAUNDRY_BACKGROUND , VM_LAUNDRY_SHORTFALL }

Functions
struct vm_pagequeue	__aligned (CACHE_LINE_SIZE)
static __inline void	vm_pagequeue_cnt_add (struct vm_pagequeue *pq, int addend)
static void	vm_pagequeue_remove (struct vm_pagequeue *pq, vm_page_t m)
static void	vm_batchqueue_init (struct vm_batchqueue *bq)
static bool	vm_batchqueue_insert (struct vm_batchqueue *bq, vm_page_t m)
static vm_page_t	vm_batchqueue_pop (struct vm_batchqueue *bq)
void	vm_domain_set (struct vm_domain *vmd)
void	vm_domain_clear (struct vm_domain *vmd)
int	vm_domain_allocate (struct vm_domain *vmd, int req, int npages)
static struct vm_domain *	vm_pagequeue_domain (vm_page_t m)
static int	vm_paging_target (struct vm_domain *vmd)
static int	vm_paging_needed (struct vm_domain *vmd, u_int free_count)
static int	vm_paging_min (struct vm_domain *vmd)
static int	vm_paging_severe (struct vm_domain *vmd)
static int	vm_laundry_target (struct vm_domain *vmd)
void	pagedaemon_wakeup (int domain)
static void	vm_domain_freecnt_inc (struct vm_domain *vmd, int adj)

Variables
struct mtx	pq_mutex
struct pglist	pq_pl
int	pq_cnt
const char *const	pq_name
uint64_t	pq_pdpages
vm_page_t	bq_pa [VM_BATCHQUEUE_SIZE]
int	bq_cnt
struct vm_pagequeue	vmd_pagequeues [PQ_COUNT]
struct mtx_padalign	vmd_free_mtx
struct mtx_padalign	vmd_pageout_mtx
struct vm_pgcache	vmd_pgcache [VM_NFREEPOOL]
struct vmem *	vmd_kernel_arena
struct vmem *	vmd_kernel_rwx_arena
u_int	vmd_domain
u_int	vmd_page_count
long	vmd_segs
u_int	vmd_pageout_deficit
uint8_t	vmd_pad [CACHE_LINE_SIZE -(sizeof(u_int) *2)]
struct pidctrl	vmd_pid
boolean_t	vmd_oom
u_int	vmd_inactive_threads
u_int	vmd_inactive_shortage
blockcount_t	vmd_inactive_running
blockcount_t	vmd_inactive_starting
volatile u_int	vmd_addl_shortage
volatile u_int	vmd_inactive_freed
volatile u_int	vmd_inactive_us
u_int	vmd_inactive_pps
int	vmd_oom_seq
int	vmd_last_active_scan
struct vm_page	vmd_markers [PQ_COUNT]
struct vm_page	vmd_inacthead
struct vm_page	vmd_clock [2]
int	vmd_pageout_wanted
int	vmd_pageout_pages_needed
bool	vmd_minset
bool	vmd_severeset
enum { ... }	vmd_laundry_request
u_int	vmd_clean_pages_freed
u_int	vmd_background_launder_target
u_int	vmd_free_reserved
u_int	vmd_free_target
u_int	vmd_free_min
u_int	vmd_inactive_target
u_int	vmd_pageout_free_min
u_int	vmd_pageout_wakeup_thresh
u_int	vmd_interrupt_free_min
u_int	vmd_free_severe
struct sysctl_oid *	vmd_oid
char	vmd_name [sizeof(__XSTRING(MAXMEMDOM))]
struct vm_domain	vm_dom [MAXMEMDOM]

Macro Definition Documentation

VM_BATCHQUEUE_SIZE

#define VM_BATCHQUEUE_SIZE   7

Definition at line 78 of file vm_pagequeue.h.

VM_DOMAIN

#define VM_DOMAIN

(

n

)

(&vm_dom[(n)])

Definition at line 301 of file vm_pagequeue.h.

VM_DOMAIN_EMPTY

#define VM_DOMAIN_EMPTY

(

n

)

(vm_dom[(n)].vmd_page_count == 0)

Definition at line 302 of file vm_pagequeue.h.

vm_domain_free_assert_locked

#define vm_domain_free_assert_locked

(

n

)

mtx_assert(vm_domain_free_lockptr((n)), MA_OWNED)

Definition at line 310 of file vm_pagequeue.h.

vm_domain_free_assert_unlocked

#define vm_domain_free_assert_unlocked

(

n

)

mtx_assert(vm_domain_free_lockptr((n)), MA_NOTOWNED)

Definition at line 312 of file vm_pagequeue.h.

vm_domain_free_lock

#define vm_domain_free_lock

(

d

)

mtx_lock(vm_domain_free_lockptr((d)))

Definition at line 314 of file vm_pagequeue.h.

vm_domain_free_lockptr

#define vm_domain_free_lockptr

(

d

)

(&(d)->vmd_free_mtx)

Definition at line 316 of file vm_pagequeue.h.

vm_domain_free_trylock

#define vm_domain_free_trylock

(

d

)

mtx_trylock(vm_domain_free_lockptr((d)))

Definition at line 318 of file vm_pagequeue.h.

vm_domain_free_unlock

#define vm_domain_free_unlock

(

d

)

mtx_unlock(vm_domain_free_lockptr((d)))

Definition at line 320 of file vm_pagequeue.h.

vm_domain_pageout_assert_locked

#define vm_domain_pageout_assert_locked

(

n

)

mtx_assert(vm_domain_pageout_lockptr((n)), MA_OWNED)

Definition at line 325 of file vm_pagequeue.h.

vm_domain_pageout_assert_unlocked

#define vm_domain_pageout_assert_unlocked

(

n

)

mtx_assert(vm_domain_pageout_lockptr((n)), MA_NOTOWNED)

Definition at line 327 of file vm_pagequeue.h.

vm_domain_pageout_lock

#define vm_domain_pageout_lock

(

d

)

mtx_lock(vm_domain_pageout_lockptr((d)))

Definition at line 329 of file vm_pagequeue.h.

vm_domain_pageout_lockptr

#define vm_domain_pageout_lockptr

(

d

)

(&(d)->vmd_pageout_mtx)

Definition at line 323 of file vm_pagequeue.h.

vm_domain_pageout_unlock

#define vm_domain_pageout_unlock

(

d

)

mtx_unlock(vm_domain_pageout_lockptr((d)))

Definition at line 331 of file vm_pagequeue.h.

vm_pagequeue_assert_locked

#define vm_pagequeue_assert_locked

(

pq

)

mtx_assert(&(pq)->pq_mutex, MA_OWNED)

Definition at line 304 of file vm_pagequeue.h.

vm_pagequeue_cnt_dec

#define vm_pagequeue_cnt_dec

(

pq

)

vm_pagequeue_cnt_add((pq), -1)

Definition at line 342 of file vm_pagequeue.h.

vm_pagequeue_cnt_inc

#define vm_pagequeue_cnt_inc

(

pq

)

vm_pagequeue_cnt_add((pq), 1)

Definition at line 341 of file vm_pagequeue.h.

vm_pagequeue_lock

#define vm_pagequeue_lock

(

pq

)

mtx_lock(&(pq)->pq_mutex)

Definition at line 305 of file vm_pagequeue.h.

vm_pagequeue_lockptr

#define vm_pagequeue_lockptr

(

pq

)

(&(pq)->pq_mutex)

Definition at line 306 of file vm_pagequeue.h.

vm_pagequeue_trylock

#define vm_pagequeue_trylock

(

pq

)

mtx_trylock(&(pq)->pq_mutex)

Definition at line 307 of file vm_pagequeue.h.

vm_pagequeue_unlock

#define vm_pagequeue_unlock

(

pq

)

mtx_unlock(&(pq)->pq_mutex)

Definition at line 308 of file vm_pagequeue.h.

Enumeration Type Documentation

anonymous enum

anonymous enum

Enumerator
VM_LAUNDRY_IDLE
VM_LAUNDRY_BACKGROUND
VM_LAUNDRY_SHORTFALL

Definition at line 38 of file vm_pagequeue.h.

Function Documentation

__aligned()

struct vm_pagequeue __aligned

(

CACHE_LINE_SIZE

)

pagedaemon_wakeup()

void pagedaemon_wakeup

(

int

domain

)

Definition at line 2407 of file vm_pageout.c.

References pageproc, VM_DOMAIN, vm_domain_pageout_assert_unlocked, vm_domain_pageout_lock, vm_domain_pageout_unlock, and vm_domain::vmd_pageout_wanted.

Referenced by _vm_domain_allocate().

Here is the caller graph for this function:

vm_batchqueue_init()

static void vm_batchqueue_init ( struct vm_batchqueue *  bq )

inlinestatic

Definition at line 353 of file vm_pagequeue.h.

References vm_batchqueue::bq_cnt.

Referenced by vm_pageout_init_scan(), vm_pageout_reinsert_inactive(), vm_pageout_scan_inactive(), and vm_pqbatch_process().

Here is the caller graph for this function:

vm_batchqueue_insert()

static bool vm_batchqueue_insert ( struct vm_batchqueue *  bq, vm_page_t  m  )

inlinestatic

Definition at line 360 of file vm_pagequeue.h.

References vm_batchqueue::bq_cnt, and vm_batchqueue::bq_pa.

Referenced by vm_page_pqbatch_submit(), vm_pageout_collect_batch(), and vm_pageout_reinsert_inactive().

Here is the caller graph for this function:

vm_batchqueue_pop()

static vm_page_t vm_batchqueue_pop ( struct vm_batchqueue *  bq )

inlinestatic

Definition at line 371 of file vm_pagequeue.h.

References vm_batchqueue::bq_cnt, and vm_batchqueue::bq_pa.

Referenced by vm_pageout_next(), and vm_pageout_reinsert_inactive().

Here is the caller graph for this function:

vm_domain_allocate()

int vm_domain_allocate	(	struct vm_domain *	vmd,
		int	req,
		int	npages
	)

Definition at line 1989 of file vm_page.c.

References _vm_domain_allocate(), pageproc, VM_ALLOC_CLASS_MASK, VM_ALLOC_INTERRUPT, and VM_ALLOC_SYSTEM.

Referenced by _vm_page_alloc_noobj_domain(), vm_page_alloc_domain_after(), and vm_page_find_contig_domain().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_domain_clear()

void vm_domain_clear

(

struct vm_domain *

vmd

)

Definition at line 3163 of file vm_page.c.

References vm_domainset_lock, vm_min_domains, vm_min_waiters, vm_pageproc_waiters, vm_paging_min(), vm_paging_severe(), vm_severe_domains, vm_severe_waiters, vm_domain::vmd_domain, vm_domain::vmd_minset, vm_domain::vmd_pageout_free_min, vm_domain::vmd_pageout_pages_needed, and vm_domain::vmd_severeset.

Referenced by vm_domain_freecnt_inc().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_domain_freecnt_inc()

static void vm_domain_freecnt_inc ( struct vm_domain *  vmd, int  adj  )

inlinestatic

Definition at line 450 of file vm_pagequeue.h.

References vm_domain_clear(), vmd_free_min, vm_domain::vmd_free_min, vmd_free_severe, vm_domain::vmd_free_severe, vmd_pageout_free_min, and vm_domain::vmd_pageout_free_min.

Referenced by _vm_page_alloc_noobj_domain(), kmem_bootstrap_free(), vm_page_alloc_domain_after(), vm_page_blacklist_add(), vm_page_find_contig_domain(), vm_page_free_toq(), vm_page_reclaim_run(), vm_page_startup(), vm_page_zone_import(), and vm_page_zone_release().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_domain_set()

void vm_domain_set

(

struct vm_domain *

vmd

)

Definition at line 3144 of file vm_page.c.

References vm_domainset_lock, vm_min_domains, vm_paging_min(), vm_paging_severe(), vm_severe_domains, vm_domain::vmd_domain, vm_domain::vmd_minset, and vm_domain::vmd_severeset.

Referenced by _vm_domain_allocate().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_laundry_target()

static int vm_laundry_target ( struct vm_domain *  vmd )

inlinestatic

Definition at line 441 of file vm_pagequeue.h.

References vm_paging_target().

Referenced by vm_pageout_laundry_worker().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_pagequeue_cnt_add()

static __inline void vm_pagequeue_cnt_add ( struct vm_pagequeue *  pq, int  addend  )

static

Definition at line 335 of file vm_pagequeue.h.

References vm_pagequeue::pq_cnt, and vm_pagequeue_assert_locked.

Referenced by vm_pageout_collect_batch(), and vm_pageout_reinsert_inactive().

Here is the caller graph for this function:

vm_pagequeue_domain()

static struct vm_domain * vm_pagequeue_domain ( vm_page_t  m )

inlinestatic

Definition at line 389 of file vm_pagequeue.h.

References VM_DOMAIN, and vm_page_domain().

Referenced by _vm_page_pagequeue(), _vm_page_pqstate_commit_requeue(), kmem_bootstrap_free(), vm_page_blacklist_add(), vm_page_free_toq(), vm_phys_enqueue_contig(), and vm_phys_free_contig().

Here is the call graph for this function:

Here is the caller graph for this function:

vm_pagequeue_remove()

static void vm_pagequeue_remove ( struct vm_pagequeue *  pq, vm_page_t  m  )

inlinestatic

Definition at line 345 of file vm_pagequeue.h.

References vm_pagequeue::pq_pl, and vm_pagequeue_cnt_dec.

vm_paging_min()

static int vm_paging_min ( struct vm_domain *  vmd )

inlinestatic

Definition at line 420 of file vm_pagequeue.h.

References vm_domain::vmd_free_min.

Referenced by vm_domain_clear(), and vm_domain_set().

Here is the caller graph for this function:

vm_paging_needed()

static int vm_paging_needed ( struct vm_domain *  vmd, u_int  free_count  )

inlinestatic

Definition at line 410 of file vm_pagequeue.h.

References vmd_pageout_wakeup_thresh.

Referenced by _vm_domain_allocate(), and vm_pageout_worker().

Here is the caller graph for this function:

vm_paging_severe()

static int vm_paging_severe ( struct vm_domain *  vmd )

inlinestatic

Definition at line 430 of file vm_pagequeue.h.

References vm_domain::vmd_free_severe.

Referenced by vm_domain_clear(), and vm_domain_set().

Here is the caller graph for this function:

vm_paging_target()

static int vm_paging_target ( struct vm_domain *  vmd )

inlinestatic

Definition at line 400 of file vm_pagequeue.h.

References vm_domain::vmd_free_target.

Referenced by vm_laundry_target(), and vm_pageout_active_target().

Here is the caller graph for this function:

Variable Documentation

bq_cnt

int bq_cnt

Definition at line 1 of file vm_pagequeue.h.

bq_pa

vm_page_t bq_pa[VM_BATCHQUEUE_SIZE]

Definition at line 0 of file vm_pagequeue.h.

pq_cnt

int pq_cnt

Definition at line 2 of file vm_pagequeue.h.

Referenced by vm_pagequeue_count().

pq_mutex

struct mtx pq_mutex

Definition at line 0 of file vm_pagequeue.h.

pq_name

const char* const pq_name

Definition at line 3 of file vm_pagequeue.h.

pq_pdpages

uint64_t pq_pdpages

Definition at line 4 of file vm_pagequeue.h.

pq_pl

struct pglist pq_pl

Definition at line 1 of file vm_pagequeue.h.

vm_dom

struct vm_domain vm_dom[MAXMEMDOM]

extern

Definition at line 117 of file vm_page.c.

Referenced by _kmem_unback(), kmem_alloc_attr_domain(), kmem_alloc_contig_domain(), kmem_init(), kmem_malloc_domain(), vm_free_count(), and vm_pagequeue_count().

vmd_addl_shortage

volatile u_int vmd_addl_shortage

Definition at line 24 of file vm_pagequeue.h.

vmd_background_launder_target

u_int vmd_background_launder_target

Definition at line 46 of file vm_pagequeue.h.

vmd_clean_pages_freed

u_int vmd_clean_pages_freed

Definition at line 45 of file vm_pagequeue.h.

vmd_clock

struct vm_page vmd_clock[2]

Definition at line 32 of file vm_pagequeue.h.

vmd_domain

u_int vmd_domain

Definition at line 10 of file vm_pagequeue.h.

vmd_free_min

u_int vmd_free_min

Definition at line 49 of file vm_pagequeue.h.

Referenced by _vm_domain_allocate(), and vm_domain_freecnt_inc().

vmd_free_mtx

struct mtx_padalign vmd_free_mtx

Definition at line 1 of file vm_pagequeue.h.

vmd_free_reserved

u_int vmd_free_reserved

Definition at line 47 of file vm_pagequeue.h.

Referenced by vm_page_reclaim_contig_domain().

vmd_free_severe

u_int vmd_free_severe

Definition at line 54 of file vm_pagequeue.h.

Referenced by _vm_domain_allocate(), and vm_domain_freecnt_inc().

vmd_free_target

u_int vmd_free_target

Definition at line 48 of file vm_pagequeue.h.

vmd_inacthead

struct vm_page vmd_inacthead

Definition at line 31 of file vm_pagequeue.h.

vmd_inactive_freed

volatile u_int vmd_inactive_freed

Definition at line 25 of file vm_pagequeue.h.

vmd_inactive_pps

u_int vmd_inactive_pps

Definition at line 27 of file vm_pagequeue.h.

vmd_inactive_running

blockcount_t vmd_inactive_running

Definition at line 22 of file vm_pagequeue.h.

vmd_inactive_shortage

u_int vmd_inactive_shortage

Definition at line 21 of file vm_pagequeue.h.

vmd_inactive_starting

blockcount_t vmd_inactive_starting

Definition at line 23 of file vm_pagequeue.h.

vmd_inactive_target

u_int vmd_inactive_target

Definition at line 50 of file vm_pagequeue.h.

vmd_inactive_threads

u_int vmd_inactive_threads

Definition at line 20 of file vm_pagequeue.h.

vmd_inactive_us

volatile u_int vmd_inactive_us

Definition at line 26 of file vm_pagequeue.h.

vmd_interrupt_free_min

u_int vmd_interrupt_free_min

Definition at line 53 of file vm_pagequeue.h.

vmd_kernel_arena

struct vmem* vmd_kernel_arena

Definition at line 8 of file vm_pagequeue.h.

Referenced by kmem_init().

vmd_kernel_rwx_arena

struct vmem* vmd_kernel_rwx_arena

Definition at line 9 of file vm_pagequeue.h.

Referenced by kmem_init().

vmd_last_active_scan

int vmd_last_active_scan

Definition at line 29 of file vm_pagequeue.h.

enum { ... } vmd_laundry_request

vmd_markers

struct vm_page vmd_markers[PQ_COUNT]

Definition at line 30 of file vm_pagequeue.h.

vmd_minset

bool vmd_minset

Definition at line 36 of file vm_pagequeue.h.

vmd_name

char vmd_name[sizeof(__XSTRING(MAXMEMDOM))]

Definition at line 58 of file vm_pagequeue.h.

Referenced by zone_alloc_sysctl().

vmd_oid

struct sysctl_oid* vmd_oid

Definition at line 57 of file vm_pagequeue.h.

vmd_oom

boolean_t vmd_oom

Definition at line 19 of file vm_pagequeue.h.

vmd_oom_seq

int vmd_oom_seq

Definition at line 28 of file vm_pagequeue.h.

vmd_pad

uint8_t vmd_pad[CACHE_LINE_SIZE -(sizeof(u_int) *2)]

Definition at line 15 of file vm_pagequeue.h.

vmd_page_count

u_int vmd_page_count

Definition at line 11 of file vm_pagequeue.h.

vmd_pageout_deficit

u_int vmd_pageout_deficit

Definition at line 14 of file vm_pagequeue.h.

vmd_pageout_free_min

u_int vmd_pageout_free_min

Definition at line 51 of file vm_pagequeue.h.

Referenced by vm_domain_freecnt_inc().

vmd_pageout_mtx

struct mtx_padalign vmd_pageout_mtx

Definition at line 2 of file vm_pagequeue.h.

vmd_pageout_pages_needed

int vmd_pageout_pages_needed

Definition at line 35 of file vm_pagequeue.h.

vmd_pageout_wakeup_thresh

u_int vmd_pageout_wakeup_thresh

Definition at line 52 of file vm_pagequeue.h.

Referenced by vm_paging_needed().

vmd_pageout_wanted

int vmd_pageout_wanted

Definition at line 34 of file vm_pagequeue.h.

vmd_pagequeues

struct vm_pagequeue vmd_pagequeues[PQ_COUNT]

Definition at line 0 of file vm_pagequeue.h.

Referenced by _vm_page_pagequeue(), and vm_pagequeue_count().

vmd_pgcache

struct vm_pgcache vmd_pgcache[VM_NFREEPOOL]

vmd_pid

struct pidctrl vmd_pid

Definition at line 18 of file vm_pagequeue.h.

vmd_segs

long vmd_segs

Definition at line 12 of file vm_pagequeue.h.

vmd_severeset

bool vmd_severeset

Definition at line 37 of file vm_pagequeue.h.