#include <sys/cdefs.h>
#include "opt_ddb.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/queue.h>
#include <sys/callout.h>
#include <sys/hash.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/condvar.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/vmem.h>
#include <sys/vmmeter.h>
#include "opt_vm.h"
#include <vm/uma.h>
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_kern.h>
#include <vm/vm_extern.h>
#include <vm/vm_param.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_phys.h>
#include <vm/vm_pagequeue.h>
#include <vm/uma_int.h>

Include dependency graph for subr_vmem.c:

Data Structures
struct	qcache

struct	vmem_btag

struct	vmem

Macros
#define	VMEM_OPTORDER 5

#define	VMEM_OPTVALUE (1 << VMEM_OPTORDER)

#define	VMEM_MAXORDER (VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)

#define	VMEM_HASHSIZE_MIN 16

#define	VMEM_HASHSIZE_MAX 131072

#define	VMEM_QCACHE_IDX_MAX 16

#define	VMEM_FITMASK (M_BESTFIT \| M_FIRSTFIT \| M_NEXTFIT)

#define	VMEM_FLAGS

#define	BT_FLAGS (M_NOWAIT \| M_WAITOK \| M_USE_RESERVE \| M_NOVM)

#define	QC_NAME_MAX 16

#define	QC_POOL_TO_QCACHE(pool) ((qcache_t *)(pool->pr_qcache))

#define	VMEM_NAME_MAX 16

#define	bt_hashlist bt_u.u_hashlist

#define	bt_freelist bt_u.u_freelist

#define	BT_TYPE_SPAN 1 /* Allocated from importfn */

#define	BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */

#define	BT_TYPE_FREE 3 /* Available space. */

#define	BT_TYPE_BUSY 4 /* Used space. */

#define	BT_TYPE_CURSOR 5 /* Cursor for nextfit allocations. */

#define	BT_ISSPAN_P(bt) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC)

#define	BT_END(bt) ((bt)->bt_start + (bt)->bt_size - 1)

#define	VMEM_CONDVAR_INIT(vm, wchan) cv_init(&vm->vm_cv, wchan)

#define	VMEM_CONDVAR_DESTROY(vm) cv_destroy(&vm->vm_cv)

#define	VMEM_CONDVAR_WAIT(vm) cv_wait(&vm->vm_cv, &vm->vm_lock)

#define	VMEM_CONDVAR_BROADCAST(vm) cv_broadcast(&vm->vm_cv)

#define	VMEM_LOCK(vm) mtx_lock(&vm->vm_lock)

#define	VMEM_TRYLOCK(vm) mtx_trylock(&vm->vm_lock)

#define	VMEM_UNLOCK(vm) mtx_unlock(&vm->vm_lock)

#define	VMEM_LOCK_INIT(vm, name) mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF)

#define	VMEM_LOCK_DESTROY(vm) mtx_destroy(&vm->vm_lock)

#define	VMEM_ASSERT_LOCKED(vm) mtx_assert(&vm->vm_lock, MA_OWNED);

#define	VMEM_ALIGNUP(addr, align) (-(-(addr) & -(align)))

#define	VMEM_CROSS_P(addr1, addr2, boundary) ((((addr1) ^ (addr2)) & -(boundary)) != 0)

#define	ORDER2SIZE(order)

#define	SIZE2ORDER(size)

#define	BT_MAXALLOC 4

#define	BT_MAXFREE (BT_MAXALLOC * 8)

#define	vmem_printf printf

Typedefs
typedef struct vmem_btag	bt_t

typedef struct qcache	qcache_t

Functions
	__FBSDID ("$FreeBSD$")

	MALLOC_DEFINE (M_VMEM, "vmem", "vmem internal structures")

	TAILQ_HEAD (vmem_seglist, vmem_btag)

	LIST_HEAD (vmem_freelist, vmem_btag)

	LIST_HEAD (vmem_hashlist, vmem_btag)

static	LIST_HEAD (vmem)

static bool	bt_isfree (bt_t *bt)

static __noinline int	_bt_fill (vmem_t *vm, int flags)

static int	bt_fill (vmem_t *vm, int flags)

static bt_t *	bt_alloc (vmem_t *vm)

static void	bt_freetrim (vmem_t *vm, int freelimit)

static void	bt_free (vmem_t vm, bt_t bt)

static void	bt_save (vmem_t *vm)

static void	bt_restore (vmem_t *vm)

static struct vmem_freelist *	bt_freehead_tofree (vmem_t *vm, vmem_size_t size)

static struct vmem_freelist *	bt_freehead_toalloc (vmem_t *vm, vmem_size_t size, int strat)

static struct vmem_hashlist *	bt_hashhead (vmem_t *vm, vmem_addr_t addr)

static bt_t *	bt_lookupbusy (vmem_t *vm, vmem_addr_t addr)

static void	bt_rembusy (vmem_t vm, bt_t bt)

static void	bt_insbusy (vmem_t vm, bt_t bt)

static void	bt_remseg (vmem_t vm, bt_t bt)

static void	bt_insseg (vmem_t vm, bt_t bt, bt_t *prev)

static void	bt_insseg_tail (vmem_t vm, bt_t bt)

static void	bt_remfree (vmem_t vm __unused, bt_t bt)

static void	bt_insfree (vmem_t vm, bt_t bt)

static int	qc_import (void arg, void *store, int cnt, int domain, int flags)

static void	qc_release (void arg, void *store, int cnt)

static void	qc_init (vmem_t *vm, vmem_size_t qcache_max)

static void	qc_destroy (vmem_t *vm)

static void	qc_drain (vmem_t *vm)

static void *	vmem_bt_alloc (uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, int wait)

void	vmem_startup (void)

static int	vmem_rehash (vmem_t *vm, vmem_size_t newhashsize)

static void	vmem_periodic_kick (void *dummy)

static void	vmem_periodic (void *unused, int pending)

static void	vmem_start_callout (void *unused)

	SYSINIT (vfs, SI_SUB_CONFIGURE, SI_ORDER_ANY, vmem_start_callout, NULL)

static void	vmem_add1 (vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int type)

static void	vmem_destroy1 (vmem_t *vm)

static int	vmem_import (vmem_t *vm, vmem_size_t size, vmem_size_t align, int flags)

static int	vmem_fit (const bt_t bt, vmem_size_t size, vmem_size_t align, vmem_size_t phase, vmem_size_t nocross, vmem_addr_t minaddr, vmem_addr_t maxaddr, vmem_addr_t addrp)

static void	vmem_clip (vmem_t vm, bt_t bt, vmem_addr_t start, vmem_size_t size)

static int	vmem_try_fetch (vmem_t *vm, const vmem_size_t size, vmem_size_t align, int flags)

static int	vmem_try_release (vmem_t vm, struct vmem_btag bt, const bool remfree)

static int	vmem_xalloc_nextfit (vmem_t vm, const vmem_size_t size, vmem_size_t align, const vmem_size_t phase, const vmem_size_t nocross, int flags, vmem_addr_t addrp)

void	vmem_set_import (vmem_t vm, vmem_import_t importfn, vmem_release_t releasefn, void arg, vmem_size_t import_quantum)

void	vmem_set_limit (vmem_t *vm, vmem_size_t limit)

void	vmem_set_reclaim (vmem_t vm, vmem_reclaim_t reclaimfn)

vmem_t *	vmem_init (vmem_t vm, const char name, vmem_addr_t base, vmem_size_t size, vmem_size_t quantum, vmem_size_t qcache_max, int flags)

vmem_t *	vmem_create (const char *name, vmem_addr_t base, vmem_size_t size, vmem_size_t quantum, vmem_size_t qcache_max, int flags)

void	vmem_destroy (vmem_t *vm)

vmem_size_t	vmem_roundup_size (vmem_t *vm, vmem_size_t size)

int	vmem_alloc (vmem_t vm, vmem_size_t size, int flags, vmem_addr_t addrp)

int	vmem_xalloc (vmem_t vm, const vmem_size_t size0, vmem_size_t align, const vmem_size_t phase, const vmem_size_t nocross, const vmem_addr_t minaddr, const vmem_addr_t maxaddr, int flags, vmem_addr_t addrp)

void	vmem_free (vmem_t *vm, vmem_addr_t addr, vmem_size_t size)

void	vmem_xfree (vmem_t *vm, vmem_addr_t addr, vmem_size_t size __unused)

int	vmem_add (vmem_t *vm, vmem_addr_t addr, vmem_size_t size, int flags)

vmem_size_t	vmem_size (vmem_t *vm, int typemask)

Variables
static struct callout	vmem_periodic_ch

static int	vmem_periodic_interval

static struct task	vmem_periodic_wk

static struct mtx_padalign __exclusive_cache_line	vmem_list_lock

static struct mtx_padalign __exclusive_cache_line	vmem_bt_lock

Macro Definition Documentation

◆ BT_END

#define BT_END ( bt ) ((bt)->bt_start + (bt)->bt_size - 1)

Definition at line 181 of file subr_vmem.c.

◆ BT_FLAGS

#define BT_FLAGS (M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM)

Definition at line 95 of file subr_vmem.c.

◆ bt_freelist

#define bt_freelist bt_u.u_freelist

Definition at line 129 of file subr_vmem.c.

◆ bt_hashlist

#define bt_hashlist bt_u.u_hashlist

Definition at line 128 of file subr_vmem.c.

◆ BT_ISSPAN_P

#define BT_ISSPAN_P ( bt ) ((bt)->bt_type <= BT_TYPE_SPAN_STATIC)

Definition at line 179 of file subr_vmem.c.

◆ BT_MAXALLOC

#define BT_MAXALLOC 4

◆ BT_MAXFREE

#define BT_MAXFREE (BT_MAXALLOC * 8)

◆ BT_TYPE_BUSY

#define BT_TYPE_BUSY 4 /* Used space. */

Definition at line 177 of file subr_vmem.c.

◆ BT_TYPE_CURSOR

#define BT_TYPE_CURSOR 5 /* Cursor for nextfit allocations. */

Definition at line 178 of file subr_vmem.c.

◆ BT_TYPE_FREE

#define BT_TYPE_FREE 3 /* Available space. */

Definition at line 176 of file subr_vmem.c.

◆ BT_TYPE_SPAN

#define BT_TYPE_SPAN 1 /* Allocated from importfn */

Definition at line 174 of file subr_vmem.c.

◆ BT_TYPE_SPAN_STATIC

#define BT_TYPE_SPAN_STATIC 2 /* vmem_add() or create. */

Definition at line 175 of file subr_vmem.c.

◆ ORDER2SIZE

#define ORDER2SIZE ( order )

Value:

((order) < VMEM_OPTVALUE ? ((order) + 1) : \

(vmem_size_t)1 << ((order) - (VMEM_OPTVALUE - VMEM_OPTORDER - 1)))

VMEM_OPTVALUE

#define VMEM_OPTVALUE

Definition: subr_vmem.c:81

VMEM_OPTORDER

#define VMEM_OPTORDER

Definition: subr_vmem.c:80

◆ QC_NAME_MAX

#define QC_NAME_MAX 16

Definition at line 97 of file subr_vmem.c.

◆ QC_POOL_TO_QCACHE

#define QC_POOL_TO_QCACHE ( pool ) ((qcache_t *)(pool->pr_qcache))

Definition at line 117 of file subr_vmem.c.

◆ SIZE2ORDER

#define SIZE2ORDER ( size )

Value:

((size) <= VMEM_OPTVALUE ? ((size) - 1) : \

(flsl(size) + (VMEM_OPTVALUE - VMEM_OPTORDER - 2)))

◆ VMEM_ALIGNUP

#define VMEM_ALIGNUP	(	addr,
		align
	)	(-(-(addr) & -(align)))

◆ VMEM_ASSERT_LOCKED

#define VMEM_ASSERT_LOCKED ( vm ) mtx_assert(&vm->vm_lock, MA_OWNED);

◆ VMEM_CONDVAR_BROADCAST

#define VMEM_CONDVAR_BROADCAST ( vm ) cv_broadcast(&vm->vm_cv)

◆ VMEM_CONDVAR_DESTROY

#define VMEM_CONDVAR_DESTROY ( vm ) cv_destroy(&vm->vm_cv)

◆ VMEM_CONDVAR_INIT

#define VMEM_CONDVAR_INIT	(	vm,
		wchan
	)	cv_init(&vm->vm_cv, wchan)

◆ VMEM_CONDVAR_WAIT

#define VMEM_CONDVAR_WAIT ( vm ) cv_wait(&vm->vm_cv, &vm->vm_lock)

◆ VMEM_CROSS_P

#define VMEM_CROSS_P	(	addr1,
		addr2,
		boundary
	)	((((addr1) ^ (addr2)) & -(boundary)) != 0)

◆ VMEM_FITMASK

#define VMEM_FITMASK (M_BESTFIT | M_FIRSTFIT | M_NEXTFIT)

Definition at line 90 of file subr_vmem.c.

◆ VMEM_FLAGS

#define VMEM_FLAGS

Value:

(M_NOWAIT | M_WAITOK | M_USE_RESERVE | M_NOVM | \

M_BESTFIT | M_FIRSTFIT | M_NEXTFIT)

Definition at line 92 of file subr_vmem.c.

◆ VMEM_HASHSIZE_MAX

#define VMEM_HASHSIZE_MAX 131072

Definition at line 86 of file subr_vmem.c.

◆ VMEM_HASHSIZE_MIN

#define VMEM_HASHSIZE_MIN 16

Definition at line 85 of file subr_vmem.c.

◆ VMEM_LOCK

#define VMEM_LOCK ( vm ) mtx_lock(&vm->vm_lock)

◆ VMEM_LOCK_DESTROY

#define VMEM_LOCK_DESTROY ( vm ) mtx_destroy(&vm->vm_lock)

◆ VMEM_LOCK_INIT

#define VMEM_LOCK_INIT	(	vm,
		name
	)	mtx_init(&vm->vm_lock, (name), NULL, MTX_DEF)

◆ VMEM_MAXORDER

#define VMEM_MAXORDER (VMEM_OPTVALUE - 1 + sizeof(vmem_size_t) * NBBY - VMEM_OPTORDER)

Definition at line 82 of file subr_vmem.c.

◆ VMEM_NAME_MAX

#define VMEM_NAME_MAX 16

Definition at line 119 of file subr_vmem.c.

◆ VMEM_OPTORDER

#define VMEM_OPTORDER 5

Definition at line 80 of file subr_vmem.c.

◆ VMEM_OPTVALUE

#define VMEM_OPTVALUE (1 << VMEM_OPTORDER)

Definition at line 81 of file subr_vmem.c.

◆ vmem_printf

#define vmem_printf printf

Definition at line 1766 of file subr_vmem.c.

◆ VMEM_QCACHE_IDX_MAX

#define VMEM_QCACHE_IDX_MAX 16

Definition at line 88 of file subr_vmem.c.

◆ VMEM_TRYLOCK

#define VMEM_TRYLOCK ( vm ) mtx_trylock(&vm->vm_lock)

◆ VMEM_UNLOCK

#define VMEM_UNLOCK ( vm ) mtx_unlock(&vm->vm_lock)

Typedef Documentation

◆ bt_t

typedef struct vmem_btag bt_t

Definition at line 104 of file subr_vmem.c.

◆ qcache_t

typedef struct qcache qcache_t

Definition at line 116 of file subr_vmem.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ _bt_fill()

static __noinline int _bt_fill	(	vmem_t *	vm,
		int	flags
	)

static

Definition at line 270 of file subr_vmem.c.

References bt, BT_FLAGS, bt_freelist, BT_MAXALLOC, flags, VMEM_ASSERT_LOCKED, VMEM_LOCK, and VMEM_UNLOCK.

Referenced by bt_fill().

Here is the caller graph for this function:

◆ bt_alloc()

static bt_t * bt_alloc ( vmem_t * vm )

static

Definition at line 322 of file subr_vmem.c.

References bt, bt_freelist, and VMEM_ASSERT_LOCKED.

Referenced by vmem_add1(), and vmem_clip().

Here is the caller graph for this function:

◆ bt_fill()

static int bt_fill	(	vmem_t *	vm,
		int	flags
	)

inlinestatic

Definition at line 311 of file subr_vmem.c.

References _bt_fill(), BT_MAXALLOC, and flags.

Referenced by vmem_add(), vmem_xalloc(), and vmem_xalloc_nextfit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_free()

static void bt_free	(	vmem_t *	vm,
		bt_t *	bt
	)

inlinestatic

Definition at line 361 of file subr_vmem.c.

References bt, bt_freelist, and VMEM_ASSERT_LOCKED.

Referenced by bt_remseg().

Here is the caller graph for this function:

◆ bt_freehead_toalloc()

static struct vmem_freelist * bt_freehead_toalloc	(	vmem_t *	vm,
		vmem_size_t	size,
		int	strat
	)

static

Definition at line 424 of file subr_vmem.c.

References ORDER2SIZE, SIZE2ORDER, and VMEM_MAXORDER.

Referenced by vmem_xalloc().

Here is the caller graph for this function:

◆ bt_freehead_tofree()

static struct vmem_freelist * bt_freehead_tofree	(	vmem_t *	vm,
		vmem_size_t	size
	)

static

Definition at line 402 of file subr_vmem.c.

References SIZE2ORDER, and VMEM_MAXORDER.

Referenced by bt_insfree().

Here is the caller graph for this function:

◆ bt_freetrim()

static void bt_freetrim	(	vmem_t *	vm,
		int	freelimit
	)

static

Definition at line 340 of file subr_vmem.c.

References bt, bt_freelist, LIST_HEAD(), VMEM_ASSERT_LOCKED, and VMEM_UNLOCK.

Referenced by vmem_destroy1(), vmem_try_release(), and vmem_xfree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_hashhead()

static struct vmem_hashlist * bt_hashhead	(	vmem_t *	vm,
		vmem_addr_t	addr
	)

static

Definition at line 445 of file subr_vmem.c.

References addr.

Referenced by bt_insbusy(), and bt_lookupbusy().

Here is the caller graph for this function:

◆ bt_insbusy()

static void bt_insbusy	(	vmem_t *	vm,
		bt_t *	bt
	)

static

Definition at line 485 of file subr_vmem.c.

References bt, bt_hashhead(), bt_hashlist, BT_TYPE_BUSY, and VMEM_ASSERT_LOCKED.

Referenced by vmem_clip(), and vmem_rehash().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_insfree()

static void bt_insfree	(	vmem_t *	vm,
		bt_t *	bt
	)

static

Definition at line 533 of file subr_vmem.c.

References bt, bt_freehead_tofree(), and bt_freelist.

Referenced by vmem_add1(), vmem_clip(), and vmem_xfree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_insseg()

static void bt_insseg	(	vmem_t *	vm,
		bt_t *	bt,
		bt_t *	prev
	)

static

Definition at line 510 of file subr_vmem.c.

References bt.

Referenced by vmem_clip().

Here is the caller graph for this function:

◆ bt_insseg_tail()

static void bt_insseg_tail	(	vmem_t *	vm,
		bt_t *	bt
	)

static

Definition at line 517 of file subr_vmem.c.

References bt.

Referenced by vmem_add1().

Here is the caller graph for this function:

◆ bt_isfree()

static bool bt_isfree ( bt_t * bt )

static

Definition at line 259 of file subr_vmem.c.

References bt, and BT_TYPE_FREE.

Referenced by vmem_add1().

Here is the caller graph for this function:

◆ bt_lookupbusy()

static bt_t * bt_lookupbusy	(	vmem_t *	vm,
		vmem_addr_t	addr
	)

static

Definition at line 457 of file subr_vmem.c.

References addr, bt, bt_hashhead(), bt_hashlist, and VMEM_ASSERT_LOCKED.

Referenced by vmem_xfree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_rembusy()

static void bt_rembusy	(	vmem_t *	vm,
		bt_t *	bt
	)

static

Definition at line 474 of file subr_vmem.c.

References bt, bt_hashlist, and VMEM_ASSERT_LOCKED.

Referenced by vmem_rehash(), and vmem_xfree().

Here is the caller graph for this function:

◆ bt_remfree()

static void bt_remfree	(	vmem_t *vm	__unused,
		bt_t *	bt
	)

static

Definition at line 524 of file subr_vmem.c.

References bt, bt_freelist, and BT_TYPE_FREE.

Referenced by vmem_add1(), vmem_clip(), vmem_try_release(), vmem_xalloc_nextfit(), and vmem_xfree().

Here is the caller graph for this function:

◆ bt_remseg()

static void bt_remseg	(	vmem_t *	vm,
		bt_t *	bt
	)

static

Definition at line 501 of file subr_vmem.c.

References bt, bt_free(), and BT_TYPE_CURSOR.

Referenced by vmem_destroy1(), vmem_try_release(), vmem_xalloc_nextfit(), and vmem_xfree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ bt_restore()

static void bt_restore ( vmem_t * vm )

static

Definition at line 383 of file subr_vmem.c.

References BT_MAXALLOC.

Referenced by vmem_import(), and vmem_try_fetch().

Here is the caller graph for this function:

◆ bt_save()

static void bt_save ( vmem_t * vm )

static

Definition at line 375 of file subr_vmem.c.

References BT_MAXALLOC.

Referenced by vmem_import(), and vmem_try_fetch().

Here is the caller graph for this function:

◆ LIST_HEAD() [1/3]

static LIST_HEAD ( vmem )

static

Definition at line 195 of file subr_vmem.c.

◆ LIST_HEAD() [2/3]

LIST_HEAD	(	vmem_freelist	,
		vmem_btag
	)

Referenced by bt_freetrim().

Here is the caller graph for this function:

◆ LIST_HEAD() [3/3]

LIST_HEAD	(	vmem_hashlist	,
		vmem_btag
	)

◆ MALLOC_DEFINE()

MALLOC_DEFINE	(	M_VMEM	,
		"vmem"	,
		"vmem internal structures"
	)

◆ qc_destroy()

static void qc_destroy ( vmem_t * vm )

static

Definition at line 608 of file subr_vmem.c.

Referenced by vmem_destroy1().

Here is the caller graph for this function:

◆ qc_drain()

static void qc_drain ( vmem_t * vm )

static

Definition at line 619 of file subr_vmem.c.

Referenced by vmem_try_fetch().

Here is the caller graph for this function:

◆ qc_import()

static int qc_import	(	void *	arg,
		void **	store,
		int	cnt,
		int	domain,
		int	flags
	)

static

Definition at line 550 of file subr_vmem.c.

References addr, flags, qcache::qc_size, qcache::qc_vmem, and vmem_xalloc().

Referenced by qc_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ qc_init()

static void qc_init	(	vmem_t *	vm,
		vmem_size_t	qcache_max
	)

static

Definition at line 583 of file subr_vmem.c.

References qcache::qc_cache, qc_import(), qcache::qc_name, qc_release(), qcache::qc_size, qcache::qc_vmem, snprintf(), and VMEM_QCACHE_IDX_MAX.

Referenced by vmem_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ qc_release()

static void qc_release	(	void *	arg,
		void **	store,
		int	cnt
	)

static

Definition at line 572 of file subr_vmem.c.

References qcache::qc_size, qcache::qc_vmem, and vmem_xfree().

Referenced by qc_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ SYSINIT()

SYSINIT	(	vfs	,
		SI_SUB_CONFIGURE	,
		SI_ORDER_ANY	,
		vmem_start_callout	,
		NULL
	)

◆ TAILQ_HEAD()

TAILQ_HEAD	(	vmem_seglist	,
		vmem_btag
	)

◆ vmem_add()

int vmem_add	(	vmem_t *	vm,
		vmem_addr_t	addr,
		vmem_size_t	size,
		int	flags
	)

Definition at line 1526 of file subr_vmem.c.

References addr, bt_fill(), BT_TYPE_SPAN_STATIC, flags, vmem_add1(), VMEM_FLAGS, VMEM_LOCK, and VMEM_UNLOCK.

Referenced by vmem_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_add1()

static void vmem_add1	(	vmem_t *	vm,
		vmem_addr_t	addr,
		vmem_size_t	size,
		int	type
	)

static

Definition at line 828 of file subr_vmem.c.

References addr, bt_alloc(), bt_insfree(), bt_insseg_tail(), bt_isfree(), bt_remfree(), vmem_btag::bt_size, vmem_btag::bt_start, vmem_btag::bt_type, BT_TYPE_FREE, BT_TYPE_SPAN, BT_TYPE_SPAN_STATIC, type, and VMEM_ASSERT_LOCKED.

Referenced by vmem_add(), and vmem_import().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_alloc()

int vmem_alloc	(	vmem_t *	vm,
		vmem_size_t	size,
		int	flags,
		vmem_addr_t *	addrp
	)

Definition at line 1326 of file subr_vmem.c.

References flags, qcache::qc_cache, VMEM_FITMASK, VMEM_FLAGS, and vmem_xalloc().

Referenced by bufkva_alloc().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_bt_alloc()

static void * vmem_bt_alloc	(	uma_zone_t	zone,
		vm_size_t	bytes,
		int	domain,
		uint8_t *	pflag,
		int	wait
	)

static

Definition at line 660 of file subr_vmem.c.

References addr, domain, vmem_bt_lock, vmem_xalloc(), and vmem_xfree().

Referenced by vmem_startup().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_clip()

static void vmem_clip	(	vmem_t *	vm,
		bt_t *	bt,
		vmem_addr_t	start,
		vmem_size_t	size
	)

static

Definition at line 996 of file subr_vmem.c.

References bt, bt_alloc(), bt_insbusy(), bt_insfree(), bt_insseg(), bt_remfree(), vmem_btag::bt_size, vmem_btag::bt_start, vmem_btag::bt_type, BT_TYPE_BUSY, BT_TYPE_FREE, start, and VMEM_ASSERT_LOCKED.

Referenced by vmem_xalloc(), and vmem_xalloc_nextfit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_create()

vmem_t * vmem_create	(	const char *	name,
		vmem_addr_t	base,
		vmem_size_t	size,
		vmem_size_t	quantum,
		vmem_size_t	qcache_max,
		int	flags
	)

Definition at line 1289 of file subr_vmem.c.

References flags, name, and vmem_init().

Here is the call graph for this function:

◆ vmem_destroy()

void vmem_destroy ( vmem_t * vm )

Definition at line 1305 of file subr_vmem.c.

References vmem_destroy1(), and vmem_list_lock.

Here is the call graph for this function:

◆ vmem_destroy1()

static void vmem_destroy1 ( vmem_t * vm )

static

Definition at line 877 of file subr_vmem.c.

References bt, bt_freetrim(), bt_remseg(), free(), qc_destroy(), VMEM_CONDVAR_DESTROY, VMEM_LOCK, and VMEM_LOCK_DESTROY.

Referenced by vmem_destroy(), and vmem_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_fit()

static int vmem_fit	(	const bt_t *	bt,
		vmem_size_t	size,
		vmem_size_t	align,
		vmem_size_t	phase,
		vmem_size_t	nocross,
		vmem_addr_t	minaddr,
		vmem_addr_t	maxaddr,
		vmem_addr_t *	addrp
	)

static

Definition at line 946 of file subr_vmem.c.

References bt, BT_END, phase, start, VMEM_ALIGNUP, and VMEM_CROSS_P.

Referenced by vmem_xalloc(), and vmem_xalloc_nextfit().

Here is the caller graph for this function:

◆ vmem_free()

void vmem_free	(	vmem_t *	vm,
		vmem_addr_t	addr,
		vmem_size_t	size
	)

Definition at line 1466 of file subr_vmem.c.

References addr, qcache::qc_cache, and vmem_xfree().

Referenced by biodone(), and bufkva_free().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_import()

static int vmem_import	(	vmem_t *	vm,
		vmem_size_t	size,
		vmem_size_t	align,
		int	flags
	)

static

Definition at line 907 of file subr_vmem.c.

References addr, bt_restore(), bt_save(), BT_TYPE_SPAN, flags, vmem_add1(), VMEM_LOCK, and VMEM_UNLOCK.

Referenced by vmem_try_fetch().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_init()

vmem_t * vmem_init	(	vmem_t *	vm,
		const char *	name,
		vmem_addr_t	base,
		vmem_size_t	size,
		vmem_size_t	quantum,
		vmem_size_t	qcache_max,
		int	flags
	)

Definition at line 1236 of file subr_vmem.c.

References BT_TYPE_CURSOR, flags, name, qc_init(), vmem_add(), VMEM_CONDVAR_INIT, vmem_destroy1(), VMEM_HASHSIZE_MIN, vmem_list_lock, VMEM_LOCK_INIT, and VMEM_MAXORDER.

Referenced by vmem_create().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_periodic()

static void vmem_periodic	(	void *	unused,
		int	pending
	)

static

Definition at line 778 of file subr_vmem.c.

References VMEM_CONDVAR_BROADCAST, VMEM_HASHSIZE_MAX, VMEM_HASHSIZE_MIN, vmem_list_lock, VMEM_LOCK, vmem_periodic_ch, vmem_periodic_interval, vmem_periodic_kick(), vmem_rehash(), and VMEM_UNLOCK.

Referenced by vmem_start_callout().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_periodic_kick()

static void vmem_periodic_kick ( void * dummy )

static

Definition at line 771 of file subr_vmem.c.

References taskqueue_enqueue(), and vmem_periodic_wk.

Referenced by vmem_periodic(), and vmem_start_callout().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_rehash()

static int vmem_rehash	(	vmem_t *	vm,
		vmem_size_t	newhashsize
	)

static

Definition at line 730 of file subr_vmem.c.

References bt, bt_insbusy(), bt_rembusy(), free(), malloc(), VMEM_LOCK, and VMEM_UNLOCK.

Referenced by vmem_periodic().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_roundup_size()

vmem_size_t vmem_roundup_size	(	vmem_t *	vm,
		vmem_size_t	size
	)

Definition at line 1316 of file subr_vmem.c.

Referenced by vmem_xalloc(), and vmem_xfree().

Here is the caller graph for this function:

◆ vmem_set_import()

void vmem_set_import	(	vmem_t *	vm,
		vmem_import_t *	importfn,
		vmem_release_t *	releasefn,
		void *	arg,
		vmem_size_t	import_quantum
	)

Definition at line 1201 of file subr_vmem.c.

References VMEM_LOCK, and VMEM_UNLOCK.

◆ vmem_set_limit()

void vmem_set_limit	(	vmem_t *	vm,
		vmem_size_t	limit
	)

Definition at line 1215 of file subr_vmem.c.

References VMEM_LOCK, and VMEM_UNLOCK.

◆ vmem_set_reclaim()

void vmem_set_reclaim	(	vmem_t *	vm,
		vmem_reclaim_t *	reclaimfn
	)

Definition at line 1224 of file subr_vmem.c.

References VMEM_LOCK, and VMEM_UNLOCK.

Referenced by bufinit().

Here is the caller graph for this function:

◆ vmem_size()

vmem_size_t vmem_size	(	vmem_t *	vm,
		int	typemask
	)

Definition at line 1545 of file subr_vmem.c.

References ORDER2SIZE, panic(), VMEM_LOCK, VMEM_MAXORDER, and VMEM_UNLOCK.

Here is the call graph for this function:

◆ vmem_start_callout()

static void vmem_start_callout ( void * unused )

static

Definition at line 816 of file subr_vmem.c.

References callout_init(), hz, vmem_periodic(), vmem_periodic_ch, vmem_periodic_interval, vmem_periodic_kick(), and vmem_periodic_wk.

Here is the call graph for this function:

◆ vmem_startup()

void vmem_startup ( void )

Definition at line 702 of file subr_vmem.c.

References BT_MAXALLOC, mp_ncpus, vmem_bt_alloc(), vmem_bt_lock, and vmem_list_lock.

Here is the call graph for this function:

◆ vmem_try_fetch()

static int vmem_try_fetch	(	vmem_t *	vm,
		const vmem_size_t	size,
		vmem_size_t	align,
		int	flags
	)

static

Definition at line 1038 of file subr_vmem.c.

References bt_restore(), bt_save(), flags, qc_drain(), VMEM_ASSERT_LOCKED, VMEM_CONDVAR_WAIT, vmem_import(), VMEM_LOCK, and VMEM_UNLOCK.

Referenced by vmem_xalloc(), and vmem_xalloc_nextfit().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_try_release()

static int vmem_try_release	(	vmem_t *	vm,
		struct vmem_btag *	bt,
		const bool	remfree
	)

static

Definition at line 1079 of file subr_vmem.c.

References bt, bt_freetrim(), BT_MAXFREE, bt_remfree(), bt_remseg(), vmem_btag::bt_size, vmem_btag::bt_start, vmem_btag::bt_type, BT_TYPE_FREE, BT_TYPE_SPAN, and VMEM_CONDVAR_BROADCAST.

Referenced by vmem_xalloc_nextfit(), and vmem_xfree().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_xalloc()

int vmem_xalloc	(	vmem_t *	vm,
		const vmem_size_t	size0,
		vmem_size_t	align,
		const vmem_size_t	phase,
		const vmem_size_t	nocross,
		const vmem_addr_t	minaddr,
		const vmem_addr_t	maxaddr,
		int	flags,
		vmem_addr_t *	addrp
	)

Definition at line 1355 of file subr_vmem.c.

References bt, bt_fill(), bt_freehead_toalloc(), bt_freelist, flags, panic(), phase, vmem_clip(), VMEM_CROSS_P, vmem_fit(), VMEM_FITMASK, VMEM_FLAGS, VMEM_LOCK, VMEM_MAXORDER, vmem_roundup_size(), vmem_try_fetch(), VMEM_UNLOCK, and vmem_xalloc_nextfit().

Referenced by qc_import(), vmem_alloc(), and vmem_bt_alloc().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_xalloc_nextfit()

static int vmem_xalloc_nextfit	(	vmem_t *	vm,
		const vmem_size_t	size,
		vmem_size_t	align,
		const vmem_size_t	phase,
		const vmem_size_t	nocross,
		int	flags,
		vmem_addr_t *	addrp
	)

static

Definition at line 1113 of file subr_vmem.c.

References bt, bt_fill(), bt_remfree(), bt_remseg(), vmem_btag::bt_size, vmem_btag::bt_start, vmem_btag::bt_type, BT_TYPE_FREE, flags, phase, vmem_clip(), vmem_fit(), VMEM_LOCK, vmem_try_fetch(), vmem_try_release(), and VMEM_UNLOCK.

Referenced by vmem_xalloc().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ vmem_xfree()

void vmem_xfree	(	vmem_t *	vm,
		vmem_addr_t	addr,
		vmem_size_t size	__unused
	)

Definition at line 1480 of file subr_vmem.c.

References addr, bt, BT_END, bt_freetrim(), bt_insfree(), bt_lookupbusy(), BT_MAXFREE, bt_rembusy(), bt_remfree(), bt_remseg(), vmem_btag::bt_size, vmem_btag::bt_start, vmem_btag::bt_type, BT_TYPE_BUSY, BT_TYPE_FREE, VMEM_CONDVAR_BROADCAST, VMEM_LOCK, vmem_roundup_size(), and vmem_try_release().

Referenced by qc_release(), vmem_bt_alloc(), and vmem_free().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ vmem_bt_lock

struct mtx_padalign __exclusive_cache_line vmem_bt_lock

static

Definition at line 631 of file subr_vmem.c.

Referenced by vmem_bt_alloc(), and vmem_startup().

◆ vmem_list_lock

struct mtx_padalign __exclusive_cache_line vmem_list_lock

static

Definition at line 194 of file subr_vmem.c.

Referenced by vmem_destroy(), vmem_init(), vmem_periodic(), and vmem_startup().

◆ vmem_periodic_ch

struct callout vmem_periodic_ch

static

Definition at line 190 of file subr_vmem.c.

Referenced by vmem_periodic(), and vmem_start_callout().

◆ vmem_periodic_interval

int vmem_periodic_interval

static

Definition at line 191 of file subr_vmem.c.

Referenced by vmem_periodic(), and vmem_start_callout().

◆ vmem_periodic_wk

struct task vmem_periodic_wk

static

Definition at line 192 of file subr_vmem.c.

Referenced by vmem_periodic_kick(), and vmem_start_callout().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ BT_END

◆ BT_FLAGS

◆ bt_freelist

◆ bt_hashlist

◆ BT_ISSPAN_P

◆ BT_MAXALLOC

◆ BT_MAXFREE

◆ BT_TYPE_BUSY

◆ BT_TYPE_CURSOR

◆ BT_TYPE_FREE

◆ BT_TYPE_SPAN

◆ BT_TYPE_SPAN_STATIC

◆ ORDER2SIZE

◆ QC_NAME_MAX

◆ QC_POOL_TO_QCACHE

◆ SIZE2ORDER

◆ VMEM_ALIGNUP

◆ VMEM_ASSERT_LOCKED

◆ VMEM_CONDVAR_BROADCAST

◆ VMEM_CONDVAR_DESTROY

◆ VMEM_CONDVAR_INIT

◆ VMEM_CONDVAR_WAIT

◆ VMEM_CROSS_P

◆ VMEM_FITMASK

◆ VMEM_FLAGS

◆ VMEM_HASHSIZE_MAX

◆ VMEM_HASHSIZE_MIN

◆ VMEM_LOCK

◆ VMEM_LOCK_DESTROY

◆ VMEM_LOCK_INIT

◆ VMEM_MAXORDER

◆ VMEM_NAME_MAX

◆ VMEM_OPTORDER

◆ VMEM_OPTVALUE

◆ vmem_printf

◆ VMEM_QCACHE_IDX_MAX

◆ VMEM_TRYLOCK

◆ VMEM_UNLOCK

Typedef Documentation

◆ bt_t

◆ qcache_t

Function Documentation

◆ __FBSDID()

◆ _bt_fill()

◆ bt_alloc()

◆ bt_fill()

◆ bt_free()

◆ bt_freehead_toalloc()

◆ bt_freehead_tofree()

◆ bt_freetrim()

◆ bt_hashhead()

◆ bt_insbusy()

◆ bt_insfree()

◆ bt_insseg()

◆ bt_insseg_tail()

◆ bt_isfree()

◆ bt_lookupbusy()

◆ bt_rembusy()

◆ bt_remfree()

◆ bt_remseg()

◆ bt_restore()

◆ bt_save()

◆ LIST_HEAD() [1/3]

◆ LIST_HEAD() [2/3]

◆ LIST_HEAD() [3/3]

◆ MALLOC_DEFINE()

◆ qc_destroy()

◆ qc_drain()

◆ qc_import()

◆ qc_init()

◆ qc_release()

◆ SYSINIT()

◆ TAILQ_HEAD()

◆ vmem_add()