#include <sys/cdefs.h>
#include "opt_vm.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/sysctl.h>
#include <sys/proc.h>
#include <sys/vnode.h>
#include <sys/mount.h>
#include <sys/bio.h>
#include <sys/buf.h>
#include <sys/vmmeter.h>
#include <sys/ktr.h>
#include <sys/limits.h>
#include <sys/conf.h>
#include <sys/refcount.h>
#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/domainset.h>
#include <sys/user.h>
#include <machine/atomic.h>
#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pager.h>
#include <vm/vm_map.h>
#include <vm/vnode_pager.h>
#include <vm/vm_extern.h>
#include <vm/uma.h>

Include dependency graph for vnode_pager.c:

Functions
	__FBSDID ("$FreeBSD$")

static int	vnode_pager_addr (struct vnode vp, vm_ooffset_t address, daddr_t rtaddress, int *run)

static int	vnode_pager_input_smlfs (vm_object_t object, vm_page_t m)

static int	vnode_pager_input_old (vm_object_t object, vm_page_t m)

static void	vnode_pager_dealloc (vm_object_t)

static int	vnode_pager_getpages (vm_object_t, vm_page_t , int, int , int *)

static int	vnode_pager_getpages_async (vm_object_t, vm_page_t , int, int , int , vop_getpages_iodone_t, void )

static void	vnode_pager_putpages (vm_object_t, vm_page_t , int, int, int )

static boolean_t	vnode_pager_haspage (vm_object_t, vm_pindex_t, int , int )

static vm_object_t	vnode_pager_alloc (void , vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred cred)

static int	vnode_pager_generic_getpages_done (struct buf *)

static void	vnode_pager_generic_getpages_done_async (struct buf *)

static void	vnode_pager_update_writecount (vm_object_t, vm_offset_t, vm_offset_t)

static void	vnode_pager_release_writecount (vm_object_t, vm_offset_t, vm_offset_t)

static void	vnode_pager_getvp (vm_object_t, struct vnode *, bool )

	SYSCTL_PROC (_debug, OID_AUTO, vnode_domainset, CTLTYPE_STRING\|CTLFLAG_MPSAFE\|CTLFLAG_RW, &vnode_domainset, 0, sysctl_handle_domainset, "A", "Default vnode NUMA policy")

	SYSCTL_INT (_vm, OID_AUTO, vnode_pbufs, CTLFLAG_RDTUN\|CTLFLAG_NOFETCH, &nvnpbufs, 0, "number of physical buffers allocated for vnode pager")

static void	vnode_pager_init (void *dummy)

	SYSINIT (vnode_pager, SI_SUB_CPU, SI_ORDER_ANY, vnode_pager_init, NULL)

int	vnode_create_vobject (struct vnode vp, off_t isize, struct thread td)

void	vnode_destroy_vobject (struct vnode *vp)

static void	vnode_pager_subpage_purge (struct vm_page *m, int base, int end)

void	vnode_pager_setsize (struct vnode *vp, vm_ooffset_t nsize)

void	vnode_pager_purge_range (struct vnode *vp, vm_ooffset_t start, vm_ooffset_t end)

int	vnode_pager_local_getpages (struct vop_getpages_args *ap)

int	vnode_pager_local_getpages_async (struct vop_getpages_async_args *ap)

int	vnode_pager_generic_getpages (struct vnode vp, vm_page_t m, int count, int a_rbehind, int a_rahead, vop_getpages_iodone_t iodone, void *arg)

static int	vn_off2bidx (vm_ooffset_t offset)

static bool	vn_dirty_blk (vm_page_t m, vm_ooffset_t offset)

int	vnode_pager_generic_putpages (struct vnode vp, vm_page_t ma, int bytecount, int flags, int *rtvals)

int	vnode_pager_putpages_ioflags (int pager_flags)

void	vnode_pager_undirty_pages (vm_page_t ma, int rtvals, int written, off_t eof, int lpos)

Variables
const struct pagerops	vnodepagerops

static struct domainset *	vnode_domainset = NULL

static int	nvnpbufs

static uma_zone_t	vnode_pbuf_zone

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ SYSCTL_INT()

SYSCTL_INT	(	_vm	,
		OID_AUTO	,
		vnode_pbufs	,
		CTLFLAG_RDTUN\|	CTLFLAG_NOFETCH,
		&	nvnpbufs,
		0	,
		"number of physical buffers allocated for vnode pager"
	)

◆ SYSCTL_PROC()

SYSCTL_PROC	(	_debug	,
		OID_AUTO	,
		vnode_domainset	,
		CTLTYPE_STRING\|CTLFLAG_MPSAFE\|	CTLFLAG_RW,
		&	vnode_domainset,
		0	,
		sysctl_handle_domainset	,
		"A"	,
		"Default vnode NUMA policy"
	)

◆ SYSINIT()

SYSINIT	(	vnode_pager	,
		SI_SUB_CPU	,
		SI_ORDER_ANY	,
		vnode_pager_init	,
		NULL
	)

◆ vn_dirty_blk()

static bool vn_dirty_blk	(	vm_page_t	m,
		vm_ooffset_t	offset
	)

static

Definition at line 1331 of file vnode_pager.c.

References IDX_TO_OFF, and vn_off2bidx().

Referenced by vnode_pager_generic_putpages().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vn_off2bidx()

static int vn_off2bidx ( vm_ooffset_t offset )

static

Definition at line 1324 of file vnode_pager.c.

Referenced by vn_dirty_blk(), and vnode_pager_generic_putpages().

Here is the caller graph for this function:

◆ vnode_create_vobject()

int vnode_create_vobject	(	struct vnode *	vp,
		off_t	isize,
		struct thread *	td
	)

Definition at line 154 of file vnode_pager.c.

References IDX_TO_OFF, vm_object::ref_count, VM_OBJECT_RLOCK, VM_OBJECT_RUNLOCK, and vnode_pager_alloc().

Here is the call graph for this function:

◆ vnode_destroy_vobject()

void vnode_destroy_vobject ( struct vnode * vp )

Definition at line 197 of file vnode_pager.c.

References vm_object::flags, vm_object::handle, OBJ_DEAD, OBJPC_SYNC, OBJT_VNODE, vm_object::ref_count, vm_object::type, umtx_shm_object_terminated(), vm_object_page_clean(), vm_object_set_flag(), vm_object_terminate(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, and vm_pager_deallocate().

Here is the call graph for this function:

◆ vnode_pager_addr()

static int vnode_pager_addr	(	struct vnode *	vp,
		vm_ooffset_t	address,
		daddr_t *	rtaddress,
		int *	run
	)

static

Definition at line 616 of file vnode_pager.c.

Referenced by vnode_pager_input_smlfs().

Here is the caller graph for this function:

◆ vnode_pager_alloc()

vm_object_t vnode_pager_alloc	(	void *	handle,
		vm_ooffset_t	size,
		vm_prot_t	prot,
		vm_ooffset_t	offset,
		struct ucred *	cred
	)

static

Definition at line 242 of file vnode_pager.c.

References vm_object::flags, OBJ_COLORED, OBJ_SIZEVNLOCK, OBJT_DEAD, OBJT_VNODE, OFF_TO_IDX, vm_object::ref_count, vm_object_allocate(), vm_object_color(), vm_object_destroy(), vm_object_reference(), vm_object_set_flag(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, and vnode_domainset.

Referenced by vnode_create_vobject().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_dealloc()

static void vnode_pager_dealloc ( vm_object_t object )

static

Definition at line 311 of file vnode_pager.c.

References OBJT_DEAD, vm_object::un_pager, VM_OBJECT_ASSERT_WLOCKED, vm_object_pip_wait(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_object::vnp, and vm_object::writemappings.

Here is the call graph for this function:

◆ vnode_pager_generic_getpages()

int vnode_pager_generic_getpages	(	struct vnode *	vp,
		vm_page_t *	m,
		int	count,
		int *	a_rbehind,
		int *	a_rahead,
		vop_getpages_iodone_t	iodone,
		void *	arg
	)

Definition at line 873 of file vnode_pager.c.

References bogus_page, IDX_TO_OFF, pbgetbo(), pbrelbo(), pmap_qenter(), pmap_zero_page(), vm_object::size, uma_zalloc(), uma_zfree(), vm_object::un_pager, VM_ALLOC_NORMAL, VM_OBJECT_WLOCK, VM_OBJECT_WOWNED, VM_OBJECT_WUNLOCK, vm_page_alloc(), vm_page_none_valid(), vm_page_valid(), VM_PAGER_BAD, VM_PAGER_ERROR, VM_PAGER_OK, vnode_pager_generic_getpages_done(), vnode_pager_generic_getpages_done_async(), vnode_pager_input_old(), vnode_pager_input_smlfs(), vnode_pbuf_zone, vm_object::vnp, and vm_object::vnp_size.

Referenced by vnode_pager_local_getpages(), and vnode_pager_local_getpages_async().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_generic_getpages_done()

static int vnode_pager_generic_getpages_done ( struct buf * bp )

static

Definition at line 1198 of file vnode_pager.c.

References bogus_page, IDX_TO_OFF, pmap_qenter(), pmap_qremove(), vm_object::un_pager, VM_OBJECT_RLOCK, VM_OBJECT_RUNLOCK, VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_bits(), vm_page_free_invalid(), vm_page_readahead_finish(), vm_page_set_valid_range(), vm_page_valid(), vm_object::vnp, and vm_object::vnp_size.

Referenced by vnode_pager_generic_getpages(), and vnode_pager_generic_getpages_done_async().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_generic_getpages_done_async()

static void vnode_pager_generic_getpages_done_async ( struct buf * bp )

static

Definition at line 1182 of file vnode_pager.c.

References pbrelbo(), uma_zfree(), vnode_pager_generic_getpages_done(), and vnode_pbuf_zone.

Referenced by vnode_pager_generic_getpages().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_generic_putpages()

int vnode_pager_generic_putpages	(	struct vnode *	vp,
		vm_page_t *	ma,
		int	bytecount,
		int	flags,
		int *	rtvals
	)

Definition at line 1351 of file vnode_pager.c.

References IDX_TO_OFF, OFF_TO_IDX, vm_object::un_pager, VM_OBJECT_RLOCK, VM_OBJECT_RUNLOCK, vm_page_assert_sbusied, vm_page_clear_dirty(), VM_PAGER_BAD, VM_PAGER_ERROR, VM_PAGER_OK, vn_dirty_blk(), vn_off2bidx(), vnode_pager_putpages_ioflags(), vm_object::vnp, and vm_object::vnp_size.

Here is the call graph for this function:

◆ vnode_pager_getpages()

static int vnode_pager_getpages	(	vm_object_t	object,
		vm_page_t *	m,
		int	count,
		int *	rbehind,
		int *	rahead
	)

static

Definition at line 815 of file vnode_pager.c.

◆ vnode_pager_getpages_async()

static int vnode_pager_getpages_async	(	vm_object_t	object,
		vm_page_t *	m,
		int	count,
		int *	rbehind,
		int *	rahead,
		vop_getpages_iodone_t	iodone,
		void *	arg
	)

static

Definition at line 830 of file vnode_pager.c.

◆ vnode_pager_getvp()

static void vnode_pager_getvp	(	vm_object_t	object,
		struct vnode **	vpp,
		bool *	vp_heldp
	)

static

Definition at line 1685 of file vnode_pager.c.

◆ vnode_pager_haspage()

static boolean_t vnode_pager_haspage	(	vm_object_t	object,
		vm_pindex_t	pindex,
		int *	before,
		int *	after
	)

static

Definition at line 353 of file vnode_pager.c.

References IDX_TO_OFF, vm_object::size, vm_object::un_pager, VM_OBJECT_ASSERT_LOCKED, VM_OBJECT_DROP, VM_OBJECT_PICKUP, vm_object::vnp, and vm_object::vnp_size.

◆ vnode_pager_init()

static void vnode_pager_init ( void * dummy )

static

Definition at line 139 of file vnode_pager.c.

References nvnpbufs, pbuf_zsecond_create(), and vnode_pbuf_zone.

Here is the call graph for this function:

◆ vnode_pager_input_old()

static int vnode_pager_input_old	(	vm_object_t	object,
		vm_page_t	m
	)

static

Definition at line 742 of file vnode_pager.c.

References IDX_TO_OFF, vm_object::un_pager, VM_OBJECT_ASSERT_WLOCKED, VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_valid(), VM_PAGER_BAD, VM_PAGER_ERROR, VM_PAGER_OK, vm_object::vnp, and vm_object::vnp_size.

Referenced by vnode_pager_generic_getpages().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_input_smlfs()

static int vnode_pager_input_smlfs	(	vm_object_t	object,
		vm_page_t	m
	)

static

Definition at line 649 of file vnode_pager.c.

References IDX_TO_OFF, pbgetbo(), pbrelbo(), uma_zalloc(), uma_zfree(), vm_object::un_pager, vm_page_bits(), vm_page_bits_set(), VM_PAGER_BAD, VM_PAGER_ERROR, VM_PAGER_OK, vnode_pager_addr(), vnode_pbuf_zone, vm_object::vnp, and vm_object::vnp_size.

Referenced by vnode_pager_generic_getpages().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_local_getpages()

int vnode_pager_local_getpages ( struct vop_getpages_args * ap )

Definition at line 849 of file vnode_pager.c.

References vnode_pager_generic_getpages().

Here is the call graph for this function:

◆ vnode_pager_local_getpages_async()

int vnode_pager_local_getpages_async ( struct vop_getpages_async_args * ap )

Definition at line 857 of file vnode_pager.c.

References vnode_pager_generic_getpages().

Here is the call graph for this function:

◆ vnode_pager_purge_range()

void vnode_pager_purge_range	(	struct vnode *	vp,
		vm_ooffset_t	start,
		vm_ooffset_t	end
	)

Definition at line 561 of file vnode_pager.c.

References OBJ_MAX_SIZE, OFF_TO_IDX, VM_ALLOC_NOCREAT, vm_object_page_remove(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_grab(), vm_page_none_valid(), vm_page_xunbusy, and vnode_pager_subpage_purge().

Here is the call graph for this function:

◆ vnode_pager_putpages()

static void vnode_pager_putpages	(	vm_object_t	object,
		vm_page_t *	m,
		int	count,
		int	flags,
		int *	rtvals
	)

static

Definition at line 1290 of file vnode_pager.c.

References VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, and VM_PAGER_PUT_SYNC.

◆ vnode_pager_putpages_ioflags()

int vnode_pager_putpages_ioflags ( int pager_flags )

Definition at line 1522 of file vnode_pager.c.

References VM_PAGER_CLUSTER_OK, VM_PAGER_PUT_INVAL, VM_PAGER_PUT_NOREUSE, and VM_PAGER_PUT_SYNC.

Referenced by vnode_pager_generic_putpages().

Here is the caller graph for this function:

◆ vnode_pager_release_writecount()

static void vnode_pager_release_writecount	(	vm_object_t	object,
		vm_offset_t	start,
		vm_offset_t	end
	)

static

Definition at line 1635 of file vnode_pager.c.

References OBJT_VNODE, vm_object::type, vm_object::un_pager, VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vnode_pager_update_writecount(), vm_object::vnp, and vm_object::writemappings.

Here is the call graph for this function:

◆ vnode_pager_setsize()

void vnode_pager_setsize	(	struct vnode *	vp,
		vm_ooffset_t	nsize
	)

Definition at line 486 of file vnode_pager.c.

References OBJT_DEAD, OBJT_VNODE, OFF_TO_IDX, vm_object::size, vm_object::type, vm_object::un_pager, VM_ALLOC_NOCREAT, vm_object_page_remove(), VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_page_grab(), vm_page_none_valid(), vm_page_xunbusy, vnode_pager_subpage_purge(), vm_object::vnp, and vm_object::vnp_size.

Here is the call graph for this function:

◆ vnode_pager_subpage_purge()

static void vnode_pager_subpage_purge	(	struct vm_page *	m,
		int	base,
		int	end
	)

static

Definition at line 434 of file vnode_pager.c.

References pmap_zero_page_area(), vm_page_clear_dirty(), and vm_page_set_valid_range().

Referenced by vnode_pager_purge_range(), and vnode_pager_setsize().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vnode_pager_undirty_pages()

void vnode_pager_undirty_pages	(	vm_page_t *	ma,
		int *	rtvals,
		int	written,
		off_t	eof,
		int	lpos
	)

Definition at line 1556 of file vnode_pager.c.

References OFF_TO_IDX, vm_page_clear_dirty(), vm_page_undirty(), VM_PAGER_AGAIN, VM_PAGER_BAD, and VM_PAGER_OK.

Here is the call graph for this function:

◆ vnode_pager_update_writecount()

static void vnode_pager_update_writecount	(	vm_object_t	object,
		vm_offset_t	start,
		vm_offset_t	end
	)

static

Definition at line 1606 of file vnode_pager.c.

References OBJT_VNODE, vm_object::type, vm_object::un_pager, VM_OBJECT_WLOCK, VM_OBJECT_WUNLOCK, vm_object::vnp, and vm_object::writemappings.

Referenced by vnode_pager_release_writecount().

Here is the caller graph for this function:

Variable Documentation

◆ nvnpbufs

int nvnpbufs

static

Definition at line 132 of file vnode_pager.c.

Referenced by vnode_pager_init().

◆ vnode_domainset

struct domainset* vnode_domainset = NULL

static

Definition at line 126 of file vnode_pager.c.

Referenced by vnode_pager_alloc().

◆ vnode_pbuf_zone

uma_zone_t vnode_pbuf_zone

static

Definition at line 136 of file vnode_pager.c.

Referenced by vnode_pager_generic_getpages(), vnode_pager_generic_getpages_done_async(), vnode_pager_init(), and vnode_pager_input_smlfs().

◆ vnodepagerops

const struct pagerops vnodepagerops

Initial value:

= {
        .pgo_kvme_type = KVME_TYPE_VNODE,
        .pgo_alloc =    vnode_pager_alloc,
        .pgo_dealloc =  vnode_pager_dealloc,
        .pgo_getpages = vnode_pager_getpages,
        .pgo_getpages_async = vnode_pager_getpages_async,
        .pgo_putpages = vnode_pager_putpages,
        .pgo_haspage =  vnode_pager_haspage,
        .pgo_update_writecount = vnode_pager_update_writecount,
        .pgo_release_writecount = vnode_pager_release_writecount,
        .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_,
        .pgo_mightbedirty = vm_object_mightbedirty_,
        .pgo_getvp = vnode_pager_getvp,
}

Definition at line 111 of file vnode_pager.c.

Functions

Variables

Function Documentation

◆ __FBSDID()

◆ SYSCTL_INT()

◆ SYSCTL_PROC()

◆ SYSINIT()

◆ vn_dirty_blk()

◆ vn_off2bidx()

◆ vnode_create_vobject()

◆ vnode_destroy_vobject()

◆ vnode_pager_addr()

◆ vnode_pager_alloc()

◆ vnode_pager_dealloc()

◆ vnode_pager_generic_getpages()

◆ vnode_pager_generic_getpages_done()

◆ vnode_pager_generic_getpages_done_async()

◆ vnode_pager_generic_putpages()

◆ vnode_pager_getpages()

◆ vnode_pager_getpages_async()

◆ vnode_pager_getvp()

◆ vnode_pager_haspage()

◆ vnode_pager_init()

◆ vnode_pager_input_old()

◆ vnode_pager_input_smlfs()

◆ vnode_pager_local_getpages()

◆ vnode_pager_local_getpages_async()

◆ vnode_pager_purge_range()

◆ vnode_pager_putpages()

◆ vnode_pager_putpages_ioflags()

◆ vnode_pager_release_writecount()

◆ vnode_pager_setsize()

◆ vnode_pager_subpage_purge()

◆ vnode_pager_undirty_pages()

◆ vnode_pager_update_writecount()

Variable Documentation

◆ nvnpbufs

◆ vnode_domainset

◆ vnode_pbuf_zone

◆ vnodepagerops