#include <sys/cdefs.h>#include "opt_ddb.h"#include "opt_param.h"#include "opt_vm.h"#include <sys/param.h>#include <sys/systm.h>#include <sys/asan.h>#include <sys/bitset.h>#include <sys/domainset.h>#include <sys/eventhandler.h>#include <sys/kernel.h>#include <sys/types.h>#include <sys/limits.h>#include <sys/queue.h>#include <sys/malloc.h>#include <sys/ktr.h>#include <sys/lock.h>#include <sys/msan.h>#include <sys/mutex.h>#include <sys/proc.h>#include <sys/random.h>#include <sys/rwlock.h>#include <sys/sbuf.h>#include <sys/sched.h>#include <sys/sleepqueue.h>#include <sys/smp.h>#include <sys/smr.h>#include <sys/sysctl.h>#include <sys/taskqueue.h>#include <sys/vmmeter.h>#include <vm/vm.h>#include <vm/vm_param.h>#include <vm/vm_domainset.h>#include <vm/vm_object.h>#include <vm/vm_page.h>#include <vm/vm_pageout.h>#include <vm/vm_phys.h>#include <vm/vm_pagequeue.h>#include <vm/vm_map.h>#include <vm/vm_kern.h>#include <vm/vm_extern.h>#include <vm/vm_dumpset.h>#include <vm/uma.h>#include <vm/uma_int.h>#include <vm/uma_dbg.h>#include <ddb/ddb.h>#include <machine/md_var.h>
Go to the source code of this file.
Data Structures | |
| struct | uma_zctor_args |
| struct | uma_kctor_args |
| struct | uma_bucket_zone |
| struct | keg_layout_result |
| struct | uma_zone_count |
| struct | uma_reclaim_args |
Macros | |
| #define | UMA_ALWAYS_CTORDTOR 0 |
| #define | SLAB_BITSETS 1 |
| #define | SLABZONE_SIZE(setsize) (sizeof(struct uma_hash_slab) + BITSET_SIZE(setsize) * SLAB_BITSETS) |
| #define | SLABZONE0_SETSIZE (PAGE_SIZE / 16) |
| #define | SLABZONE1_SETSIZE SLAB_MAX_SETSIZE |
| #define | SLABZONE0_SIZE SLABZONE_SIZE(SLABZONE0_SETSIZE) |
| #define | SLABZONE1_SIZE SLABZONE_SIZE(SLABZONE1_SETSIZE) |
| #define | UMA_TIMEOUT 20 /* Seconds for callout interval. */ |
| #define | BUCKET_SIZE(n) (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) |
| #define | BUCKET_MAX BUCKET_SIZE(256) |
| #define | UMA_FIXPT_SHIFT 31 |
| #define | UMA_FRAC_FIXPT(n, d) ((uint32_t)(((uint64_t)(n) << UMA_FIXPT_SHIFT) / (d))) |
| #define | UMA_FIXPT_PCT(f) ((u_int)(((uint64_t)100 * (f)) >> UMA_FIXPT_SHIFT)) |
| #define | UMA_PCT_FIXPT(pct) UMA_FRAC_FIXPT((pct), 100) |
| #define | UMA_MIN_EFF UMA_PCT_FIXPT(100 - UMA_MAX_WASTE) |
Enumerations | |
| enum | zfreeskip { SKIP_NONE = 0 , SKIP_CNT = 0x00000001 , SKIP_DTOR = 0x00010000 , SKIP_FINI = 0x00020000 } |
Functions | |
| __FBSDID ("$FreeBSD$") | |
| static | MALLOC_DEFINE (M_UMAHASH, "UMAHash", "UMA Hash Buckets") |
| static | MALLOC_DEFINE (M_UMA, "UMA", "UMA Misc") |
| static | LIST_HEAD (uma_keg) |
| void | uma_startup1 (vm_offset_t) |
| void | uma_startup2 (void) |
| static void * | noobj_alloc (uma_zone_t, vm_size_t, int, uint8_t *, int) |
| static void * | page_alloc (uma_zone_t, vm_size_t, int, uint8_t *, int) |
| static void * | pcpu_page_alloc (uma_zone_t, vm_size_t, int, uint8_t *, int) |
| static void * | startup_alloc (uma_zone_t, vm_size_t, int, uint8_t *, int) |
| static void * | contig_alloc (uma_zone_t, vm_size_t, int, uint8_t *, int) |
| static void | page_free (void *, vm_size_t, uint8_t) |
| static void | pcpu_page_free (void *, vm_size_t, uint8_t) |
| static uma_slab_t | keg_alloc_slab (uma_keg_t, uma_zone_t, int, int, int) |
| static void | cache_drain (uma_zone_t) |
| static void | bucket_drain (uma_zone_t, uma_bucket_t) |
| static void | bucket_cache_reclaim (uma_zone_t zone, bool, int) |
| static bool | bucket_cache_reclaim_domain (uma_zone_t, bool, bool, int) |
| static int | keg_ctor (void *, int, void *, int) |
| static void | keg_dtor (void *, int, void *) |
| static void | keg_drain (uma_keg_t keg, int domain) |
| static int | zone_ctor (void *, int, void *, int) |
| static void | zone_dtor (void *, int, void *) |
| static void | item_dtor (uma_zone_t zone, void *item, int size, void *udata, enum zfreeskip skip) |
| static int | zero_init (void *, int, int) |
| static void | zone_free_bucket (uma_zone_t zone, uma_bucket_t bucket, void *udata, int itemdomain, bool ws) |
| static void | zone_foreach (void(*zfunc)(uma_zone_t, void *), void *) |
| static void | zone_foreach_unlocked (void(*zfunc)(uma_zone_t, void *), void *) |
| static void | zone_timeout (uma_zone_t zone, void *) |
| static int | hash_alloc (struct uma_hash *, u_int) |
| static int | hash_expand (struct uma_hash *, struct uma_hash *) |
| static void | hash_free (struct uma_hash *hash) |
| static void | uma_timeout (void *) |
| static void | uma_shutdown (void) |
| static void * | zone_alloc_item (uma_zone_t, void *, int, int) |
| static void | zone_free_item (uma_zone_t, void *, void *, enum zfreeskip) |
| static int | zone_alloc_limit (uma_zone_t zone, int count, int flags) |
| static void | zone_free_limit (uma_zone_t zone, int count) |
| static void | bucket_enable (void) |
| static void | bucket_init (void) |
| static uma_bucket_t | bucket_alloc (uma_zone_t zone, void *, int) |
| static void | bucket_free (uma_zone_t zone, uma_bucket_t, void *) |
| static void | bucket_zone_drain (int domain) |
| static uma_bucket_t | zone_alloc_bucket (uma_zone_t, void *, int, int) |
| static void * | slab_alloc_item (uma_keg_t keg, uma_slab_t slab) |
| static void | slab_free_item (uma_zone_t zone, uma_slab_t slab, void *item) |
| static size_t | slab_sizeof (int nitems) |
| static uma_keg_t | uma_kcreate (uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, int align, uint32_t flags) |
| static int | zone_import (void *, void **, int, int, int) |
| static void | zone_release (void *, void **, int) |
| static bool | cache_alloc (uma_zone_t, uma_cache_t, void *, int) |
| static bool | cache_free (uma_zone_t, uma_cache_t, void *, int) |
| static int | sysctl_vm_zone_count (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_vm_zone_stats (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_handle_uma_zone_allocs (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_handle_uma_zone_frees (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_handle_uma_zone_flags (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_handle_uma_slab_efficiency (SYSCTL_HANDLER_ARGS) |
| static int | sysctl_handle_uma_zone_items (SYSCTL_HANDLER_ARGS) |
| static uint64_t | uma_zone_get_allocs (uma_zone_t zone) |
| static | SYSCTL_NODE (_vm, OID_AUTO, debug, CTLFLAG_RD|CTLFLAG_MPSAFE, 0, "Memory allocation debugging") |
| SYSCTL_NODE (_vm, OID_AUTO, uma, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "Universal Memory Allocator") | |
| SYSCTL_PROC (_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_INT, 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones") | |
| SYSCTL_PROC (_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_STRUCT, 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats") | |
| SYSCTL_INT (_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0, "Warn when UMA zones becomes full") | |
| TUNABLE_INT ("vm.debug.uma_multipage_slabs", &multipage_slabs) | |
| SYSCTL_INT (_vm_debug, OID_AUTO, uma_multipage_slabs, CTLFLAG_RDTUN|CTLFLAG_NOFETCH, &multipage_slabs, 0, "UMA may choose larger slab sizes for better efficiency") | |
| static uma_zone_t | slabzone (int ipers) |
| static struct uma_bucket_zone * | bucket_zone_lookup (int entries) |
| static int | bucket_select (int size) |
| static void | kasan_mark_item_valid (uma_zone_t zone __unused, void *item __unused) |
| static void | kasan_mark_item_invalid (uma_zone_t zone __unused, void *item __unused) |
| static void | kasan_mark_slab_valid (uma_keg_t keg __unused, void *mem __unused) |
| static void | kasan_mark_slab_invalid (uma_keg_t keg __unused, void *mem __unused) |
| static void | kmsan_mark_item_uninitialized (uma_zone_t zone __unused, void *item __unused) |
| static uma_zone_domain_t | zone_domain_lock (uma_zone_t zone, int domain) |
| static __noinline int | zone_domain_lowest (uma_zone_t zone, int pref) |
| static __noinline int | zone_domain_highest (uma_zone_t zone, int pref) |
| static void | zone_domain_imax_set (uma_zone_domain_t zdom, int nitems) |
| static uma_bucket_t | zone_fetch_bucket (uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) |
| static void | zone_put_bucket (uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, const bool ws) |
| static void * | cache_bucket_pop (uma_cache_t cache, uma_cache_bucket_t bucket) |
| static void | cache_bucket_push (uma_cache_t cache, uma_cache_bucket_t bucket, void *item) |
| static uma_bucket_t | cache_bucket_unload (uma_cache_bucket_t bucket) |
| static uma_bucket_t | cache_bucket_unload_alloc (uma_cache_t cache) |
| static uma_bucket_t | cache_bucket_unload_free (uma_cache_t cache) |
| static uma_bucket_t | cache_bucket_unload_cross (uma_cache_t cache) |
| static void | cache_bucket_load (uma_cache_bucket_t bucket, uma_bucket_t b) |
| static void | cache_bucket_load_alloc (uma_cache_t cache, uma_bucket_t b) |
| static void | cache_bucket_load_free (uma_cache_t cache, uma_bucket_t b) |
| static void | cache_bucket_copy (uma_cache_bucket_t b1, uma_cache_bucket_t b2) |
| static void | cache_bucket_swap (uma_cache_bucket_t b1, uma_cache_bucket_t b2) |
| static uma_bucket_t | cache_fetch_bucket (uma_zone_t zone, uma_cache_t cache, int domain) |
| static void | zone_log_warning (uma_zone_t zone) |
| static void | zone_maxaction (uma_zone_t zone) |
| static void | zone_domain_update_wss (uma_zone_domain_t zdom) |
| static void | cache_shrink (uma_zone_t zone, void *unused) |
| static void | cache_drain_safe_cpu (uma_zone_t zone, void *unused) |
| static void | pcpu_cache_drain_safe (uma_zone_t zone) |
| static void | keg_free_slab (uma_keg_t keg, uma_slab_t slab, int start) |
| static void | keg_drain_domain (uma_keg_t keg, int domain) |
| static void | zone_reclaim (uma_zone_t zone, int domain, int waitok, bool drain) |
| static void | startup_free (void *mem, vm_size_t bytes) |
| static u_int | slab_ipers_hdr (u_int size, u_int rsize, u_int slabsize, bool hdr) |
| static void | keg_layout_one (uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, struct keg_layout_result *kl) |
| static void | keg_layout (uma_keg_t keg) |
| static void | zone_kva_available (uma_zone_t zone, void *unused) |
| static void | zone_alloc_counters (uma_zone_t zone, void *unused) |
| static void | zone_alloc_sysctl (uma_zone_t zone, void *unused) |
| static void | zone_count (uma_zone_t zone, void *arg) |
| static void | zone_update_caches (uma_zone_t zone) |
| static void | zone_foreach_unlocked (void(*zfunc)(uma_zone_t, void *arg), void *arg) |
| static void | zone_foreach (void(*zfunc)(uma_zone_t, void *arg), void *arg) |
| void | vm_radix_reserve_kva (void) |
| static void | uma_startup_pcpu (void *arg __unused) |
| SYSINIT (uma_startup_pcpu, SI_SUB_COUNTER, SI_ORDER_ANY, uma_startup_pcpu, NULL) | |
| static void | uma_startup3 (void *arg __unused) |
| SYSINIT (uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL) | |
| void | uma_set_align (int align) |
| uma_zone_t | uma_zcreate (const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, int align, uint32_t flags) |
| uma_zone_t | uma_zsecond_create (const char *name, uma_ctor ctor, uma_dtor dtor, uma_init zinit, uma_fini zfini, uma_zone_t primary) |
| uma_zone_t | uma_zcache_create (const char *name, int size, uma_ctor ctor, uma_dtor dtor, uma_init zinit, uma_fini zfini, uma_import zimport, uma_release zrelease, void *arg, int flags) |
| void | uma_zdestroy (uma_zone_t zone) |
| void | uma_zwait (uma_zone_t zone) |
| void * | uma_zalloc_pcpu_arg (uma_zone_t zone, void *udata, int flags) |
| void | uma_zfree_pcpu_arg (uma_zone_t zone, void *pcpu_item, void *udata) |
| static void * | item_ctor (uma_zone_t zone, int uz_flags, int size, void *udata, int flags, void *item) |
| static void * | cache_alloc_item (uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, void *udata, int flags) |
| static __noinline void * | cache_alloc_retry (uma_zone_t zone, uma_cache_t cache, void *udata, int flags) |
| void * | uma_zalloc_smr (uma_zone_t zone, int flags) |
| void * | uma_zalloc_arg (uma_zone_t zone, void *udata, int flags) |
| void * | uma_zalloc_domain (uma_zone_t zone, void *udata, int domain, int flags) |
| static uma_slab_t | keg_first_slab (uma_keg_t keg, int domain, bool rr) |
| static uma_slab_t | keg_fetch_free_slab (uma_keg_t keg, int domain, bool rr, int flags) |
| static uma_slab_t | keg_fetch_slab (uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) |
| static int | zone_alloc_limit_hard (uma_zone_t zone, int count, int flags) |
| void | uma_zfree_smr (uma_zone_t zone, void *item) |
| void | uma_zfree_arg (uma_zone_t zone, void *item, void *udata) |
| int | uma_zone_set_max (uma_zone_t zone, int nitems) |
| void | uma_zone_set_maxcache (uma_zone_t zone, int nitems) |
| int | uma_zone_get_max (uma_zone_t zone) |
| void | uma_zone_set_warning (uma_zone_t zone, const char *warning) |
| void | uma_zone_set_maxaction (uma_zone_t zone, uma_maxaction_t maxaction) |
| int | uma_zone_get_cur (uma_zone_t zone) |
| static uint64_t | uma_zone_get_frees (uma_zone_t zone) |
| void | uma_zone_set_init (uma_zone_t zone, uma_init uminit) |
| void | uma_zone_set_fini (uma_zone_t zone, uma_fini fini) |
| void | uma_zone_set_zinit (uma_zone_t zone, uma_init zinit) |
| void | uma_zone_set_zfini (uma_zone_t zone, uma_fini zfini) |
| void | uma_zone_set_freef (uma_zone_t zone, uma_free freef) |
| void | uma_zone_set_allocf (uma_zone_t zone, uma_alloc allocf) |
| void | uma_zone_set_smr (uma_zone_t zone, smr_t smr) |
| smr_t | uma_zone_get_smr (uma_zone_t zone) |
| void | uma_zone_reserve (uma_zone_t zone, int items) |
| int | uma_zone_reserve_kva (uma_zone_t zone, int count) |
| void | uma_prealloc (uma_zone_t zone, int items) |
| size_t | uma_zone_memory (uma_zone_t zone) |
| static void | uma_reclaim_domain_cb (uma_zone_t zone, void *arg) |
| void | uma_reclaim (int req) |
| void | uma_reclaim_domain (int req, int domain) |
| void | uma_reclaim_wakeup (void) |
| void | uma_reclaim_worker (void *arg __unused) |
| void | uma_zone_reclaim (uma_zone_t zone, int req) |
| void | uma_zone_reclaim_domain (uma_zone_t zone, int req, int domain) |
| int | uma_zone_exhausted (uma_zone_t zone) |
| unsigned long | uma_limit (void) |
| void | uma_set_limit (unsigned long limit) |
| unsigned long | uma_size (void) |
| long | uma_avail (void) |
| static void | uma_vm_zone_stats (struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf, struct uma_percpu_stat *ups, bool internal) |
| int | sysctl_handle_uma_zone_max (SYSCTL_HANDLER_ARGS) |
| int | sysctl_handle_uma_zone_cur (SYSCTL_HANDLER_ARGS) |
Variables | |
| static uma_zone_t | kegs |
| static uma_zone_t | zones |
| static uma_zone_t | slabzones [2] |
| static uma_zone_t | hashzone |
| int | uma_align_cache = 64 - 1 |
| static int | bucketdisable = 1 |
| booted = BOOT_COLD | |
| static struct callout | uma_callout |
| struct uma_bucket_zone | bucket_zones [] |
| static int | zone_warnings = 1 |
| static int | multipage_slabs = 1 |
| static volatile int | uma_reclaim_needed |
Macro Definition Documentation
◆ BUCKET_MAX
| #define BUCKET_MAX BUCKET_SIZE(256) |
Definition at line 255 of file uma_core.c.
◆ BUCKET_SIZE
| #define BUCKET_SIZE | ( | n | ) | (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) |
Definition at line 252 of file uma_core.c.
◆ SLAB_BITSETS
| #define SLAB_BITSETS 1 |
Definition at line 129 of file uma_core.c.
◆ SLABZONE0_SETSIZE
| #define SLABZONE0_SETSIZE (PAGE_SIZE / 16) |
Definition at line 142 of file uma_core.c.
◆ SLABZONE0_SIZE
| #define SLABZONE0_SIZE SLABZONE_SIZE(SLABZONE0_SETSIZE) |
Definition at line 144 of file uma_core.c.
◆ SLABZONE1_SETSIZE
| #define SLABZONE1_SETSIZE SLAB_MAX_SETSIZE |
Definition at line 143 of file uma_core.c.
◆ SLABZONE1_SIZE
| #define SLABZONE1_SIZE SLABZONE_SIZE(SLABZONE1_SETSIZE) |
Definition at line 145 of file uma_core.c.
◆ SLABZONE_SIZE
| #define SLABZONE_SIZE | ( | setsize | ) | (sizeof(struct uma_hash_slab) + BITSET_SIZE(setsize) * SLAB_BITSETS) |
Definition at line 140 of file uma_core.c.
◆ UMA_ALWAYS_CTORDTOR
| #define UMA_ALWAYS_CTORDTOR 0 |
Definition at line 113 of file uma_core.c.
◆ UMA_FIXPT_PCT
| #define UMA_FIXPT_PCT | ( | f | ) | ((u_int)(((uint64_t)100 * (f)) >> UMA_FIXPT_SHIFT)) |
Definition at line 2175 of file uma_core.c.
◆ UMA_FIXPT_SHIFT
| #define UMA_FIXPT_SHIFT 31 |
Definition at line 2172 of file uma_core.c.
◆ UMA_FRAC_FIXPT
| #define UMA_FRAC_FIXPT | ( | n, | |
| d | |||
| ) | ((uint32_t)(((uint64_t)(n) << UMA_FIXPT_SHIFT) / (d))) |
Definition at line 2173 of file uma_core.c.
◆ UMA_MIN_EFF
| #define UMA_MIN_EFF UMA_PCT_FIXPT(100 - UMA_MAX_WASTE) |
Definition at line 2178 of file uma_core.c.
◆ UMA_PCT_FIXPT
| #define UMA_PCT_FIXPT | ( | pct | ) | UMA_FRAC_FIXPT((pct), 100) |
Definition at line 2177 of file uma_core.c.
◆ UMA_TIMEOUT
| #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ |
Definition at line 211 of file uma_core.c.
Enumeration Type Documentation
◆ zfreeskip
| enum zfreeskip |
| Enumerator | |
|---|---|
| SKIP_NONE | |
| SKIP_CNT | |
| SKIP_DTOR | |
| SKIP_FINI | |
Definition at line 274 of file uma_core.c.
Function Documentation
◆ __FBSDID()
| __FBSDID | ( | "$FreeBSD$" | ) |
◆ bucket_alloc()
|
static |
Definition at line 466 of file uma_core.c.
References booted, bucket_zone_lookup(), uma_bucket::ub_bucket, uma_bucket::ub_cnt, uma_bucket::ub_entries, uma_bucket::ub_seq, uma_bucket_zone::ubz_entries, uma_bucket_zone::ubz_zone, uma_zalloc_arg(), UMA_ZFLAG_BUCKET, UMA_ZONE_VM, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_max, uma_zone::uz_flags, and uma_zone::uz_name.
Referenced by cache_free(), and zone_alloc_bucket().
◆ bucket_cache_reclaim()
|
static |
Definition at line 1573 of file uma_core.c.
References bucket_cache_reclaim_domain(), UMA_ANYDOMAIN, UMA_ZONE_ROUNDROBIN, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_min, uma_zone::uz_flags, and vm_ndomains.
Referenced by cache_drain(), and zone_reclaim().
◆ bucket_cache_reclaim_domain()
|
static |
Definition at line 1518 of file uma_core.c.
References bucket_free(), uma_bucket::ub_cnt, UMA_TIMEOUT, UMA_ZONE_SMR, uma_zone::uz_flags, uma_zone_domain::uzd_buckets, uma_zone_domain::uzd_cross, uma_zone_domain::uzd_limin, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_timin, uma_zone_domain::uzd_wss, ZDOM_GET, ZDOM_LOCK, ZDOM_UNLOCK, ZONE_CROSS_LOCK, ZONE_CROSS_UNLOCK, zone_domain_update_wss(), and zone_fetch_bucket().
Referenced by bucket_cache_reclaim(), and zone_timeout().
◆ bucket_drain()
|
static |
Definition at line 1346 of file uma_core.c.
References item_dtor(), kasan_mark_item_invalid(), kasan_mark_item_valid(), SKIP_NONE, uma_bucket::ub_bucket, uma_bucket::ub_cnt, uma_bucket::ub_seq, UMA_ZONE_SMR, uma_zone::uz_arg, uma_zone::uz_fini, uma_zone::uz_flags, uma_zone::uz_max_items, uma_zone::uz_release, uma_zone::uz_size, uma_zone::uz_smr, and zone_free_limit().
Referenced by bucket_free(), and cache_free().
◆ bucket_enable()
|
static |
Definition at line 404 of file uma_core.c.
References booted, and bucketdisable.
Referenced by uma_reclaim_domain(), uma_startup2(), and uma_timeout().
◆ bucket_free()
|
static |
Definition at line 516 of file uma_core.c.
References bucket_drain(), bucket_zone_lookup(), uma_bucket::ub_cnt, uma_bucket::ub_entries, uma_bucket::ub_seq, uma_bucket_zone::ubz_zone, UMA_ZFLAG_BUCKET, uma_zfree_arg(), and uma_zone::uz_flags.
Referenced by bucket_cache_reclaim_domain(), cache_alloc(), cache_drain(), cache_free(), zone_alloc_bucket(), and zone_put_bucket().
◆ bucket_init()
|
static |
Definition at line 418 of file uma_core.c.
References bucket_zones, uma_bucket_zone::ubz_entries, uma_bucket_zone::ubz_name, uma_bucket_zone::ubz_zone, UMA_ALIGN_PTR, uma_zcreate(), UMA_ZFLAG_BUCKET, UMA_ZONE_FIRSTTOUCH, and UMA_ZONE_MTXCLASS.
Referenced by uma_startup1().
◆ bucket_select()
|
static |
Definition at line 450 of file uma_core.c.
References bucket_zones, uma_bucket_zone::ubz_entries, and uma_bucket_zone::ubz_maxsize.
Referenced by zone_ctor().
◆ bucket_zone_drain()
|
static |
Definition at line 534 of file uma_core.c.
References bucket_zones, uma_bucket_zone::ubz_entries, uma_bucket_zone::ubz_zone, UMA_RECLAIM_DRAIN, and uma_zone_reclaim_domain().
Referenced by uma_reclaim_domain().
◆ bucket_zone_lookup()
|
static |
Definition at line 438 of file uma_core.c.
References bucket_zones, and uma_bucket_zone::ubz_entries.
Referenced by bucket_alloc(), and bucket_free().
◆ cache_alloc()
|
static |
Definition at line 3685 of file uma_core.c.
References bucket_free(), bucketdisable, cache_bucket_load_alloc(), cache_bucket_swap(), cache_bucket_unload_alloc(), cache_fetch_bucket(), cache_uz_flags(), uma_zctor_args::flags, uma_bucket::ub_cnt, uma_cache::uc_allocbucket, uma_cache::uc_freebucket, uma_cache_bucket::ucb_bucket, uma_cache_bucket::ucb_cnt, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_ROUNDROBIN, UMA_ZONE_SMR, uma_zone::uz_bucket_size, uma_zone::uz_cpu, uma_zone::uz_name, uzd_imax, VM_DOMAIN_EMPTY, ZDOM_GET, zone_alloc_bucket(), zone_domain_highest(), and zone_put_bucket().
Referenced by cache_alloc_retry().
◆ cache_alloc_item()
|
inlinestatic |
Definition at line 3570 of file uma_core.c.
References cache_bucket_pop(), cache_uz_flags(), cache_uz_size(), uma_zctor_args::flags, item_ctor(), and uma_zctor_args::size.
Referenced by cache_alloc_retry(), uma_zalloc_arg(), and uma_zalloc_smr().
◆ cache_alloc_retry()
|
static |
Definition at line 3584 of file uma_core.c.
References cache_alloc(), cache_alloc_item(), uma_zctor_args::flags, uma_cache::uc_allocbucket, uma_cache_bucket::ucb_cnt, UMA_ANYDOMAIN, UMA_ZONE_FIRSTTOUCH, uma_zone::uz_cpu, uma_zone::uz_flags, and zone_alloc_item().
Referenced by uma_zalloc_arg(), and uma_zalloc_smr().
◆ cache_bucket_copy()
|
inlinestatic |
Definition at line 1023 of file uma_core.c.
References uma_cache_bucket::ucb_bucket, uma_cache_bucket::ucb_cnt, and uma_cache_bucket::ucb_entries.
Referenced by cache_bucket_swap().
◆ cache_bucket_load()
|
inlinestatic |
Definition at line 984 of file uma_core.c.
References uma_bucket::ub_cnt, uma_bucket::ub_entries, uma_bucket::ub_seq, uma_cache_bucket::ucb_bucket, uma_cache_bucket::ucb_cnt, and uma_cache_bucket::ucb_entries.
Referenced by cache_bucket_load_alloc(), and cache_bucket_load_free().
◆ cache_bucket_load_alloc()
|
inlinestatic |
Definition at line 997 of file uma_core.c.
References cache_bucket_load(), and uma_cache::uc_allocbucket.
Referenced by cache_alloc().
◆ cache_bucket_load_free()
|
inlinestatic |
Definition at line 1004 of file uma_core.c.
References cache_bucket_load(), and uma_cache::uc_freebucket.
Referenced by cache_free().
◆ cache_bucket_pop()
|
inlinestatic |
Definition at line 909 of file uma_core.c.
References uma_bucket::ub_bucket, uma_cache::uc_allocs, uma_cache_bucket::ucb_bucket, and uma_cache_bucket::ucb_cnt.
Referenced by cache_alloc_item().
◆ cache_bucket_push()
|
inlinestatic |
Definition at line 928 of file uma_core.c.
References uma_bucket::ub_bucket, uma_cache::uc_frees, uma_cache_bucket::ucb_bucket, and uma_cache_bucket::ucb_cnt.
Referenced by uma_zfree_arg(), and uma_zfree_smr().
◆ cache_bucket_swap()
|
inlinestatic |
Definition at line 1035 of file uma_core.c.
References cache_bucket_copy().
Referenced by cache_alloc(), and uma_zfree_arg().
◆ cache_bucket_unload()
|
inlinestatic |
Definition at line 944 of file uma_core.c.
References uma_bucket::ub_cnt, uma_bucket::ub_entries, uma_cache_bucket::ucb_bucket, uma_cache_bucket::ucb_cnt, and uma_cache_bucket::ucb_entries.
Referenced by cache_bucket_unload_alloc(), cache_bucket_unload_cross(), cache_bucket_unload_free(), and cache_free().
◆ cache_bucket_unload_alloc()
|
inlinestatic |
Definition at line 960 of file uma_core.c.
References cache_bucket_unload(), and uma_cache::uc_allocbucket.
Referenced by cache_alloc(), cache_drain(), and cache_drain_safe_cpu().
◆ cache_bucket_unload_cross()
|
inlinestatic |
Definition at line 974 of file uma_core.c.
References cache_bucket_unload(), and uma_cache::uc_crossbucket.
Referenced by cache_drain(), and cache_drain_safe_cpu().
◆ cache_bucket_unload_free()
|
inlinestatic |
Definition at line 967 of file uma_core.c.
References cache_bucket_unload(), and uma_cache::uc_freebucket.
Referenced by cache_drain(), and cache_drain_safe_cpu().
◆ cache_drain()
|
static |
Definition at line 1390 of file uma_core.c.
References bucket_cache_reclaim(), bucket_free(), cache_bucket_unload_alloc(), cache_bucket_unload_cross(), cache_bucket_unload_free(), uma_bucket::ub_seq, UMA_ANYDOMAIN, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, and uma_zone::uz_smr.
Referenced by zone_dtor().
◆ cache_drain_safe_cpu()
|
static |
Definition at line 1441 of file uma_core.c.
References cache_bucket_unload_alloc(), cache_bucket_unload_cross(), cache_bucket_unload_free(), UMA_ZFLAG_INTERNAL, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, vm_ndomains, and zone_free_bucket().
Referenced by pcpu_cache_drain_safe().
◆ cache_fetch_bucket()
|
static |
Definition at line 1050 of file uma_core.c.
References cache_uz_flags(), UMA_ZONE_SMR, uma_zone::uz_smr, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_seq, ZDOM_GET, ZDOM_UNLOCK, zone_domain_lock(), and zone_fetch_bucket().
Referenced by cache_alloc().
◆ cache_free()
|
static |
Definition at line 4642 of file uma_core.c.
References bucket_alloc(), bucket_drain(), bucket_free(), bucketdisable, cache_bucket_load_free(), cache_bucket_unload(), cache_uz_flags(), uma_bucket::ub_cnt, uma_bucket::ub_entries, uma_bucket::ub_seq, uma_cache::uc_crossbucket, uma_cache::uc_freebucket, uma_cache_bucket::ucb_bucket, uma_cache_bucket::ucb_cnt, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_SMR, uma_zone::uz_bucket_size, uma_zone::uz_cpu, uma_zone::uz_flags, uma_zone::uz_smr, uma_zone::uz_xdomain, and zone_free_bucket().
Referenced by uma_zfree_arg(), and uma_zfree_smr().
◆ cache_shrink()
|
static |
Definition at line 1428 of file uma_core.c.
References UMA_ZFLAG_INTERNAL, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_min, uma_zone::uz_flags, ZONE_LOCK, and ZONE_UNLOCK.
Referenced by pcpu_cache_drain_safe().
◆ contig_alloc()
|
static |
Definition at line 2068 of file uma_core.c.
References kmem_alloc_contig_domainset(), and UMA_SLAB_KERNEL.
Referenced by keg_ctor(), and zone_kva_available().
◆ hash_alloc()
|
static |
Definition at line 1246 of file uma_core.c.
References hashzone, uma_hash::uh_hashmask, uma_hash::uh_hashsize, uma_hash::uh_slab_hash, UMA_ANYDOMAIN, UMA_HASH_SIZE_INIT, and zone_alloc_item().
Referenced by keg_ctor(), and zone_timeout().
◆ hash_expand()
Definition at line 1285 of file uma_core.c.
References uma_hash::uh_hashsize, uma_hash::uh_slab_hash, uma_hash_slab::uhs_data, and UMA_HASH.
Referenced by zone_timeout().
◆ hash_free()
|
static |
Definition at line 1325 of file uma_core.c.
References hashzone, SKIP_NONE, uma_hash::uh_hashsize, uma_hash::uh_slab_hash, UMA_HASH_SIZE_INIT, and zone_free_item().
Referenced by keg_dtor(), and zone_timeout().
◆ item_ctor()
|
inlinestatic |
Definition at line 3421 of file uma_core.c.
References uma_zctor_args::flags, kasan_mark_item_valid(), kmsan_mark_item_uninitialized(), uma_zctor_args::size, SKIP_CNT, SKIP_DTOR, trash_ctor(), UMA_ZFLAG_CTORDTOR, UMA_ZFLAG_TRASH, uma_zone::uz_ctor, uma_zone::uz_fails, and zone_free_item().
Referenced by cache_alloc_item(), uma_zalloc_domain(), and zone_alloc_item().
◆ item_dtor()
|
inlinestatic |
Definition at line 3457 of file uma_core.c.
References kasan_mark_item_invalid(), uma_zctor_args::size, SKIP_DTOR, SKIP_NONE, trash_dtor(), UMA_ZFLAG_TRASH, UMA_ZONE_MALLOC, uma_zone::uz_dtor, and uma_zone::uz_flags.
Referenced by bucket_drain(), uma_zfree_arg(), zone_fetch_bucket(), and zone_free_item().
◆ kasan_mark_item_invalid()
|
static |
Definition at line 621 of file uma_core.c.
Referenced by bucket_drain(), item_dtor(), zone_alloc_bucket(), zone_alloc_item(), and zone_free_item().
◆ kasan_mark_item_valid()
|
static |
Definition at line 616 of file uma_core.c.
Referenced by bucket_drain(), item_ctor(), zone_alloc_bucket(), zone_alloc_item(), and zone_free_item().
◆ kasan_mark_slab_invalid()
|
static |
Definition at line 631 of file uma_core.c.
Referenced by keg_alloc_slab().
◆ kasan_mark_slab_valid()
|
static |
Definition at line 626 of file uma_core.c.
Referenced by keg_free_slab().
◆ keg_alloc_slab()
|
static |
Definition at line 1754 of file uma_core.c.
References kasan_mark_slab_invalid(), keg_free_slab(), KEG_LOCK, SKIP_NONE, slab_item(), slab_tohashslab(), slabzone(), uma_domain::ud_free_items, uma_domain::ud_pages, uma_domain::ud_part_slab, uma_hash_slab::uhs_data, uma_hash_slab::uhs_slab, uma_keg::uk_allocf, uma_keg::uk_domain, uma_keg::uk_flags, uma_keg::uk_hash, uma_keg::uk_init, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_pgoff, uma_keg::uk_ppera, uma_keg::uk_size, UMA_HASH_INSERT, uma_total_inc(), UMA_ZFLAG_HASH, UMA_ZFLAG_OFFPAGE, UMA_ZFLAG_VTOSLAB, UMA_ZONE_MALLOC, UMA_ZONE_NODUMP, uma_slab::us_domain, uma_slab::us_flags, uma_slab::us_free, uma_slab::us_freecount, vm_ndomains, vsetzoneslab(), zone_alloc_item(), and zone_free_item().
Referenced by keg_fetch_slab(), and uma_prealloc().
◆ keg_ctor()
|
static |
Definition at line 2427 of file uma_core.c.
References uma_kctor_args::align, booted, contig_alloc(), uma_kctor_args::fini, uma_kctor_args::flags, hash_alloc(), keg_layout(), KEG_LOCK_INIT, page_alloc(), page_free(), pcpu_page_alloc(), pcpu_page_free(), uma_kctor_args::size, slab_sizeof(), startup_alloc(), uma_keg::uk_align, uma_keg::uk_allocf, uma_keg::uk_dr, uma_keg::uk_fini, uma_keg::uk_flags, uma_keg::uk_freef, uma_keg::uk_hash, uma_keg::uk_init, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_pgoff, uma_keg::uk_ppera, uma_keg::uk_reserve, uma_keg::uk_rsize, uma_keg::uk_size, uma_small_alloc(), uma_small_free(), UMA_ZFLAG_CACHE, UMA_ZFLAG_HASH, UMA_ZFLAG_OFFPAGE, UMA_ZFLAG_VTOSLAB, UMA_ZONE_CONTIG, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_MALLOC, UMA_ZONE_MTXCLASS, UMA_ZONE_NOTPAGE, UMA_ZONE_PCPU, UMA_ZONE_ROUNDROBIN, UMA_ZONE_ZINIT, uma_kctor_args::uminit, uma_zone::uz_name, vm_ndomains, zero_init(), and uma_kctor_args::zone.
Referenced by uma_startup1(), and zone_ctor().
◆ keg_drain()
|
static |
Definition at line 1697 of file uma_core.c.
References keg_drain_domain(), uma_keg::uk_flags, UMA_ANYDOMAIN, UMA_ZONE_NOFREE, and vm_ndomains.
Referenced by zone_reclaim(), and zone_timeout().
◆ keg_drain_domain()
|
static |
Definition at line 1634 of file uma_core.c.
References keg_free_slab(), KEG_LOCK, KEG_UNLOCK, uma_keg::uk_domain, uma_keg::uk_flags, uma_keg::uk_hash, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_ppera, uma_keg::uk_reserve, UMA_HASH_REMOVE, and UMA_ZFLAG_HASH.
Referenced by keg_drain().
◆ keg_dtor()
|
static |
Definition at line 2969 of file uma_core.c.
References hash_free(), KEG_LOCK_FINI, uma_domain::ud_free_items, uma_domain::ud_pages, uma_keg::uk_domain, uma_keg::uk_hash, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_ppera, and vm_ndomains.
Referenced by uma_startup1().
◆ keg_fetch_free_slab()
|
static |
Definition at line 3880 of file uma_core.c.
References uma_zctor_args::flags, uma_zctor_args::keg, keg_first_slab(), KEG_LOCK, KEG_UNLOCK, uma_domain::ud_free_items, uma_keg::uk_domain, uma_keg::uk_flags, uma_keg::uk_reserve, and UMA_ZFLAG_HASH.
Referenced by keg_fetch_slab().
◆ keg_fetch_slab()
|
static |
Definition at line 3900 of file uma_core.c.
References keg_alloc_slab(), keg_fetch_free_slab(), uma_keg::uk_dr, UMA_ANYDOMAIN, vm_domainset_iter_policy(), vm_domainset_iter_policy_ref_init(), vm_wait_domain(), and vm_wait_doms().
Referenced by zone_import().
◆ keg_first_slab()
|
static |
Definition at line 3846 of file uma_core.c.
References uma_zctor_args::keg, KEG_LOCK_ASSERT, uma_domain::ud_free_slab, uma_domain::ud_free_slabs, uma_domain::ud_part_slab, uma_keg::uk_domain, and vm_ndomains.
Referenced by keg_fetch_free_slab().
◆ keg_free_slab()
|
static |
Definition at line 1594 of file uma_core.c.
References kasan_mark_slab_valid(), SKIP_NONE, slab_data(), slab_item(), slab_tohashslab(), slabzone(), trash_fini(), uma_keg::uk_fini, uma_keg::uk_flags, uma_keg::uk_freef, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_ppera, uma_keg::uk_size, uma_total_dec(), UMA_ZFLAG_OFFPAGE, uma_slab::us_flags, and zone_free_item().
Referenced by keg_alloc_slab(), and keg_drain_domain().
◆ keg_layout()
|
static |
Definition at line 2256 of file uma_core.c.
References keg_layout_result::eff, keg_layout_result::format, keg_layout_result::ipers, keg_layout_one(), multipage_slabs, PRINT_UMA_ZFLAGS, SLAB_MAX_SETSIZE, keg_layout_result::slabsize, uma_keg::uk_align, uma_keg::uk_flags, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_ppera, uma_keg::uk_rsize, uma_keg::uk_size, UMA_CACHESPREAD_MAX_SIZE, UMA_FIXPT_PCT, UMA_MIN_EFF, UMA_SMALLEST_UNIT, UMA_ZFLAG_HASH, UMA_ZFLAG_INTERNAL, UMA_ZFLAG_OFFPAGE, UMA_ZFLAG_VTOSLAB, UMA_ZONE_CACHESPREAD, UMA_ZONE_CONTIG, UMA_ZONE_NOTOUCH, UMA_ZONE_NOTPAGE, UMA_ZONE_PCPU, and UMA_ZONE_VM.
Referenced by keg_ctor().
◆ keg_layout_one()
|
static |
Definition at line 2220 of file uma_core.c.
References keg_layout_result::eff, keg_layout_result::format, keg_layout_result::ipers, slab_ipers_hdr(), keg_layout_result::slabsize, slabzone(), uma_keg::uk_rsize, uma_keg::uk_size, UMA_FRAC_FIXPT, UMA_ZFLAG_INTERNAL, UMA_ZFLAG_OFFPAGE, and uma_zone::uz_keg.
Referenced by keg_layout().
◆ kmsan_mark_item_uninitialized()
|
inlinestatic |
Definition at line 685 of file uma_core.c.
Referenced by item_ctor().
◆ LIST_HEAD()
|
static |
Definition at line 166 of file uma_core.c.
◆ MALLOC_DEFINE() [1/2]
|
static |
◆ MALLOC_DEFINE() [2/2]
|
static |
◆ noobj_alloc()
|
static |
Definition at line 2014 of file uma_core.c.
References pmap_qenter(), TAILQ_HEAD(), uma_keg::uk_kva, uma_keg::uk_offset, UMA_SLAB_PRIV, uma_zone::uz_keg, VM_ALLOC_INTERRUPT, VM_ALLOC_WAITOK, VM_ALLOC_WIRED, vm_page_alloc_noobj_domain(), vm_page_free(), and vm_page_unwire_noq().
Referenced by uma_zone_reserve_kva().
◆ page_alloc()
|
static |
Definition at line 1937 of file uma_core.c.
References kmem_malloc_domainset(), and UMA_SLAB_KERNEL.
Referenced by keg_ctor(), and zone_kva_available().
◆ page_free()
|
static |
Definition at line 2089 of file uma_core.c.
References kmem_free(), startup_free(), UMA_SLAB_BOOT, and UMA_SLAB_KERNEL.
Referenced by keg_ctor().
◆ pcpu_cache_drain_safe()
|
static |
Definition at line 1485 of file uma_core.c.
References cache_drain_safe_cpu(), cache_shrink(), and zone_foreach().
Referenced by uma_reclaim_domain(), and uma_zone_reclaim_domain().
◆ pcpu_page_alloc()
|
static |
Definition at line 1949 of file uma_core.c.
References kva_alloc(), pmap_qenter(), UMA_SLAB_KERNEL, VM_ALLOC_SYSTEM, VM_ALLOC_WIRED, VM_DOMAIN_EMPTY, vm_page_alloc_noobj(), vm_page_alloc_noobj_domain(), vm_page_free(), and vm_page_unwire_noq().
Referenced by keg_ctor(), and zone_kva_available().
◆ pcpu_page_free()
|
static |
Definition at line 2115 of file uma_core.c.
References kva_free(), PHYS_TO_VM_PAGE(), pmap_qremove(), startup_free(), UMA_SLAB_BOOT, vm_page_free(), and vm_page_unwire_noq().
Referenced by keg_ctor().
◆ slab_alloc_item()
|
static |
Definition at line 3978 of file uma_core.c.
References KEG_LOCK_ASSERT, slab_item(), uma_domain::ud_free_items, uma_domain::ud_full_slab, uma_keg::uk_domain, uma_keg::uk_ipers, uma_slab::us_domain, uma_slab::us_free, and uma_slab::us_freecount.
Referenced by zone_import().
◆ slab_free_item()
|
static |
Definition at line 4731 of file uma_core.c.
References KEG_LOCK_ASSERT, slab_item_index(), uma_domain::ud_free_items, uma_domain::ud_free_slab, uma_domain::ud_free_slabs, uma_domain::ud_part_slab, uma_keg::uk_domain, uma_keg::uk_ipers, uma_slab::us_domain, uma_slab::us_free, uma_slab::us_freecount, and uma_zone::uz_keg.
Referenced by zone_release().
◆ slab_ipers_hdr()
|
static |
Definition at line 2186 of file uma_core.c.
References SLAB_MAX_SETSIZE, and slab_sizeof().
Referenced by keg_layout_one().
◆ slab_sizeof()
|
static |
Definition at line 2164 of file uma_core.c.
References SLAB_BITSETS, and UMA_ALIGN_PTR.
Referenced by keg_ctor(), and slab_ipers_hdr().
◆ slabzone()
|
inlinestatic |
Definition at line 394 of file uma_core.c.
References SLABZONE0_SETSIZE, and slabzones.
Referenced by keg_alloc_slab(), keg_free_slab(), keg_layout_one(), and sysctl_handle_uma_slab_efficiency().
◆ startup_alloc()
|
static |
Definition at line 1869 of file uma_core.c.
References dump_add_page, pmap_map(), UMA_SLAB_BOOT, VM_ALLOC_WIRED, vm_page_alloc_noobj_contig_domain(), VM_PAGE_TO_PHYS, VM_PROT_READ, and VM_PROT_WRITE.
Referenced by keg_ctor(), uma_startup1(), and zone_kva_available().
◆ startup_free()
|
static |
Definition at line 1901 of file uma_core.c.
References dump_drop_page, PHYS_TO_VM_PAGE(), pmap_remove(), vm_page_free(), VM_PAGE_TO_PHYS, and vm_page_unwire_noq().
Referenced by page_free(), and pcpu_page_free().
◆ sysctl_handle_uma_slab_efficiency()
|
static |
Definition at line 5600 of file uma_core.c.
References slabzone(), uma_keg::uk_align, uma_keg::uk_flags, uma_keg::uk_ipers, uma_keg::uk_ppera, uma_keg::uk_rsize, uma_keg::uk_size, UMA_ZFLAG_OFFPAGE, UMA_ZONE_PCPU, and uma_zone::uz_keg.
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_allocs()
|
static |
Definition at line 5562 of file uma_core.c.
References uma_zone_get_allocs().
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_cur()
| int sysctl_handle_uma_zone_cur | ( | SYSCTL_HANDLER_ARGS | ) |
Definition at line 5544 of file uma_core.c.
References uma_zone_get_cur().
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_flags()
|
static |
Definition at line 5582 of file uma_core.c.
References PRINT_UMA_ZFLAGS.
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_frees()
|
static |
Definition at line 5572 of file uma_core.c.
References uma_zone_get_frees().
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_items()
|
static |
Definition at line 5621 of file uma_core.c.
References uma_zone::uz_items, and UZ_ITEMS_COUNT.
Referenced by zone_alloc_sysctl().
◆ sysctl_handle_uma_zone_max()
| int sysctl_handle_uma_zone_max | ( | SYSCTL_HANDLER_ARGS | ) |
Definition at line 5528 of file uma_core.c.
References uma_zone_get_max(), and uma_zone_set_max().
◆ SYSCTL_INT() [1/2]
| SYSCTL_INT | ( | _vm | , |
| OID_AUTO | , | ||
| zone_warnings | , | ||
| CTLFLAG_RWTUN | , | ||
| & | zone_warnings, | ||
| 0 | , | ||
| "Warn when UMA zones becomes full" | |||
| ) |
◆ SYSCTL_INT() [2/2]
| SYSCTL_INT | ( | _vm_debug | , |
| OID_AUTO | , | ||
| uma_multipage_slabs | , | ||
| CTLFLAG_RDTUN| | CTLFLAG_NOFETCH, | ||
| & | multipage_slabs, | ||
| 0 | , | ||
| "UMA may choose larger slab sizes for better efficiency" | |||
| ) |
◆ SYSCTL_NODE() [1/2]
|
static |
◆ SYSCTL_NODE() [2/2]
| SYSCTL_NODE | ( | _vm | , |
| OID_AUTO | , | ||
| uma | , | ||
| CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
| 0 | , | ||
| "Universal Memory Allocator" | |||
| ) |
◆ SYSCTL_PROC() [1/2]
| SYSCTL_PROC | ( | _vm | , |
| OID_AUTO | , | ||
| zone_count | , | ||
| CTLFLAG_RD|CTLFLAG_MPSAFE| | CTLTYPE_INT, | ||
| 0 | , | ||
| 0 | , | ||
| sysctl_vm_zone_count | , | ||
| "I" | , | ||
| "Number of UMA zones" | |||
| ) |
◆ SYSCTL_PROC() [2/2]
| SYSCTL_PROC | ( | _vm | , |
| OID_AUTO | , | ||
| zone_stats | , | ||
| CTLFLAG_RD|CTLFLAG_MPSAFE| | CTLTYPE_STRUCT, | ||
| 0 | , | ||
| 0 | , | ||
| sysctl_vm_zone_stats | , | ||
| " | s, | ||
| struct uma_type_header" | , | ||
| "Zone Stats" | |||
| ) |
◆ sysctl_vm_zone_count()
|
static |
Definition at line 5384 of file uma_core.c.
References uma_reclaim_args::req.
◆ sysctl_vm_zone_stats()
|
static |
Definition at line 5435 of file uma_core.c.
References UMA_STREAM_VERSION, uma_vm_zone_stats(), UMA_ZFLAG_INTERNAL, UMA_ZONE_SECONDARY, uma_stream_header::ush_count, uma_stream_header::ush_maxcpus, uma_stream_header::ush_version, uma_type_header::uth_align, uma_type_header::uth_keg_free, uma_type_header::uth_limit, UTH_MAX_NAME, uma_type_header::uth_maxpages, uma_type_header::uth_name, uma_type_header::uth_pages, uma_type_header::uth_rsize, uma_type_header::uth_size, uma_type_header::uth_zone_flags, UTH_ZONE_SECONDARY, uma_zone::uz_flags, uma_zone::uz_items, UZ_ITEMS_COUNT, uma_zone::uz_max_items, uma_zone::uz_name, uma_zone::uz_size, and vm_ndomains.
◆ SYSINIT() [1/2]
| SYSINIT | ( | uma_startup3 | , |
| SI_SUB_VM_CONF | , | ||
| SI_ORDER_SECOND | , | ||
| uma_startup3 | , | ||
| NULL | |||
| ) |
◆ SYSINIT() [2/2]
| SYSINIT | ( | uma_startup_pcpu | , |
| SI_SUB_COUNTER | , | ||
| SI_ORDER_ANY | , | ||
| uma_startup_pcpu | , | ||
| NULL | |||
| ) |
◆ TUNABLE_INT()
| TUNABLE_INT | ( | "vm.debug.uma_multipage_slabs" | , |
| & | multipage_slabs | ||
| ) |
◆ uma_avail()
| long uma_avail | ( | void | ) |
Definition at line 5332 of file uma_core.c.
References uma_kmem_limit, and uma_size().
◆ uma_kcreate()
|
static |
Definition at line 3225 of file uma_core.c.
References uma_kctor_args::align, uma_kctor_args::fini, uma_kctor_args::flags, kegs, uma_kctor_args::size, UMA_ALIGN_CACHE, uma_align_cache, UMA_ANYDOMAIN, uma_kctor_args::uminit, uma_kctor_args::zone, and zone_alloc_item().
Referenced by zone_ctor().
◆ uma_limit()
| unsigned long uma_limit | ( | void | ) |
Definition at line 5311 of file uma_core.c.
References uma_kmem_limit.
◆ uma_prealloc()
| void uma_prealloc | ( | uma_zone_t | zone, |
| int | items | ||
| ) |
Definition at line 5133 of file uma_core.c.
References keg_alloc_slab(), KEG_GET, KEG_UNLOCK, uma_keg::uk_domain, uma_keg::uk_dr, uma_keg::uk_ipers, uma_slab::us_domain, vm_domainset_iter_policy(), vm_domainset_iter_policy_ref_init(), and vm_wait_doms().
Referenced by _vm_map_unlock(), pbuf_prealloc(), pbuf_zsecond_create(), and vm_map_startup().
◆ uma_reclaim()
| void uma_reclaim | ( | int | req | ) |
Definition at line 5207 of file uma_core.c.
References uma_reclaim_args::req, UMA_ANYDOMAIN, and uma_reclaim_domain().
Referenced by debug_uma_reclaim(), uma_reclaim_wakeup(), uma_reclaim_worker(), and vm_pageout_lowmem().
◆ uma_reclaim_domain()
| void uma_reclaim_domain | ( | int | req, |
| int | domain | ||
| ) |
Definition at line 5213 of file uma_core.c.
References bucket_enable(), bucket_zone_drain(), uma_reclaim_args::domain, pcpu_cache_drain_safe(), uma_reclaim_args::req, slabzones, uma_reclaim_domain_cb(), UMA_RECLAIM_DRAIN, UMA_RECLAIM_DRAIN_CPU, UMA_RECLAIM_TRIM, uma_zone_reclaim_domain(), and zone_foreach().
Referenced by debug_uma_reclaim_domain(), and uma_reclaim().
◆ uma_reclaim_domain_cb()
|
static |
Definition at line 5196 of file uma_core.c.
References uma_reclaim_args::domain, uma_reclaim_args::req, uma_zone_reclaim_domain(), UMA_ZONE_UNMANAGED, and uma_zone::uz_flags.
Referenced by uma_reclaim_domain().
◆ uma_reclaim_wakeup()
| void uma_reclaim_wakeup | ( | void | ) |
Definition at line 5251 of file uma_core.c.
References uma_reclaim(), and uma_reclaim_needed.
Referenced by vm_pageout_lowmem().
◆ uma_reclaim_worker()
| void uma_reclaim_worker | ( | void *arg | __unused | ) |
Definition at line 5259 of file uma_core.c.
References uma_reclaim(), UMA_RECLAIM_DRAIN_CPU, uma_reclaim_needed, and VM_LOW_KMEM.
◆ uma_set_align()
| void uma_set_align | ( | int | align | ) |
Definition at line 3242 of file uma_core.c.
References uma_kctor_args::align, UMA_ALIGN_CACHE, and uma_align_cache.
◆ uma_set_limit()
| void uma_set_limit | ( | unsigned long | limit | ) |
Definition at line 5318 of file uma_core.c.
References uma_kmem_limit.
◆ uma_shutdown()
|
static |
Definition at line 3218 of file uma_core.c.
References booted.
Referenced by uma_startup3().
◆ uma_size()
| unsigned long uma_size | ( | void | ) |
Definition at line 5325 of file uma_core.c.
References uma_kmem_total.
Referenced by uma_avail().
◆ uma_startup1()
| void uma_startup1 | ( | vm_offset_t | virtual_avail | ) |
Definition at line 3078 of file uma_core.c.
References uma_zctor_args::align, bucket_init(), uma_zctor_args::ctor, uma_zctor_args::dtor, uma_zctor_args::fini, uma_zctor_args::flags, hashzone, uma_zctor_args::keg, keg_ctor(), keg_dtor(), kegs, uma_zctor_args::name, uma_zctor_args::size, SLABZONE0_SIZE, SLABZONE1_SIZE, slabzones, startup_alloc(), UMA_ALIGN_PTR, UMA_HASH_SIZE_INIT, UMA_SUPER_ALIGN, uma_zcreate(), UMA_ZFLAG_INTERNAL, uma_zctor_args::uminit, vm_ndomains, zero_init(), zone_ctor(), zone_dtor(), and zones.
Referenced by vm_mem_init().
◆ uma_startup2()
| void uma_startup2 | ( | void | ) |
Definition at line 3162 of file uma_core.c.
References booted, bucket_enable(), kernel_map, MAP_NOFAULT, vm_map_insert(), vm_map_lock, vm_map_unlock, VM_PROT_RW, vm_radix_reserve_kva(), zone_foreach_unlocked(), and zone_kva_available().
Referenced by kmem_init().
◆ uma_startup3()
|
static |
Definition at line 3199 of file uma_core.c.
References booted, uma_callout, uma_shutdown(), UMA_TIMEOUT, uma_timeout(), zone_alloc_sysctl(), and zone_foreach_unlocked().
◆ uma_startup_pcpu()
|
static |
Definition at line 3187 of file uma_core.c.
References booted, zone_alloc_counters(), and zone_foreach_unlocked().
◆ uma_timeout()
|
static |
Definition at line 1110 of file uma_core.c.
References bucket_enable(), uma_callout, UMA_TIMEOUT, uma_timeout(), zone_foreach(), and zone_timeout().
Referenced by uma_startup3(), and uma_timeout().
◆ uma_vm_zone_stats()
|
static |
Definition at line 5404 of file uma_core.c.
References uma_cache::uc_allocbucket, uma_cache::uc_allocs, uma_cache::uc_crossbucket, uma_cache::uc_freebucket, uma_cache::uc_frees, uma_cache_bucket::ucb_cnt, uma_percpu_stat::ups_allocs, uma_percpu_stat::ups_cache_free, uma_percpu_stat::ups_frees, uma_type_header::uth_allocs, uma_type_header::uth_fails, uma_type_header::uth_frees, uma_type_header::uth_sleeps, uma_type_header::uth_xdomain, uma_type_header::uth_zone_free, uma_zone::uz_allocs, uma_zone::uz_cpu, uma_zone::uz_fails, uma_zone::uz_frees, uma_zone::uz_sleeps, uma_zone::uz_xdomain, uma_zone_domain::uzd_nitems, vm_ndomains, and ZDOM_GET.
Referenced by sysctl_vm_zone_stats().
◆ uma_zalloc_arg()
| void * uma_zalloc_arg | ( | uma_zone_t | zone, |
| void * | udata, | ||
| int | flags | ||
| ) |
Definition at line 3637 of file uma_core.c.
References cache_alloc_item(), cache_alloc_retry(), uma_zctor_args::flags, uma_cache::uc_allocbucket, uma_cache_bucket::ucb_cnt, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, and uma_zone::uz_name.
Referenced by bucket_alloc(), uma_zalloc_domain(), and uma_zalloc_pcpu_arg().
◆ uma_zalloc_domain()
| void * uma_zalloc_domain | ( | uma_zone_t | zone, |
| void * | udata, | ||
| int | domain, | ||
| int | flags | ||
| ) |
Definition at line 3776 of file uma_core.c.
References uma_zctor_args::flags, item_ctor(), uma_bucket::ub_bucket, uma_bucket::ub_cnt, uma_zalloc_arg(), UMA_ZONE_FIRSTTOUCH, UMA_ZONE_SMR, uma_zone::uz_allocs, uma_zone::uz_flags, uma_zone::uz_name, uma_zone::uz_size, vm_ndomains, ZDOM_UNLOCK, zone_alloc_item(), zone_domain_lock(), zone_fetch_bucket(), and zone_put_bucket().
◆ uma_zalloc_pcpu_arg()
| void * uma_zalloc_pcpu_arg | ( | uma_zone_t | zone, |
| void * | udata, | ||
| int | flags | ||
| ) |
Definition at line 3377 of file uma_core.c.
References uma_zctor_args::flags, uma_zalloc_arg(), UMA_ZONE_PCPU, uma_zone::uz_flags, and uma_zone::uz_size.
◆ uma_zalloc_smr()
| void * uma_zalloc_smr | ( | uma_zone_t | zone, |
| int | flags | ||
| ) |
Definition at line 3610 of file uma_core.c.
References cache_alloc_item(), cache_alloc_retry(), uma_zctor_args::flags, uma_cache::uc_allocbucket, uma_cache_bucket::ucb_cnt, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, and uma_zone::uz_name.
Referenced by uma_zwait(), and vm_radix_node_get().
◆ uma_zcache_create()
| uma_zone_t uma_zcache_create | ( | const char * | name, |
| int | size, | ||
| uma_ctor | ctor, | ||
| uma_dtor | dtor, | ||
| uma_init | zinit, | ||
| uma_fini | zfini, | ||
| uma_import | zimport, | ||
| uma_release | zrelease, | ||
| void * | arg, | ||
| int | flags | ||
| ) |
Definition at line 3325 of file uma_core.c.
References uma_zctor_args::align, uma_zctor_args::arg, uma_zctor_args::ctor, uma_zctor_args::dtor, uma_zctor_args::fini, uma_zctor_args::flags, uma_zctor_args::import, uma_zctor_args::name, uma_zctor_args::release, uma_zctor_args::size, UMA_ANYDOMAIN, UMA_ZFLAG_CACHE, uma_zctor_args::uminit, zone_alloc_item(), and zones.
Referenced by kstack_cache_init(), and vm_page_init_cache_zones().
◆ uma_zcreate()
| uma_zone_t uma_zcreate | ( | const char * | name, |
| size_t | size, | ||
| uma_ctor | ctor, | ||
| uma_dtor | dtor, | ||
| uma_init | uminit, | ||
| uma_fini | fini, | ||
| int | align, | ||
| uint32_t | flags | ||
| ) |
Definition at line 3251 of file uma_core.c.
References uma_zctor_args::align, uma_zctor_args::ctor, uma_zctor_args::dtor, uma_zctor_args::fini, uma_zctor_args::flags, uma_zctor_args::keg, uma_zctor_args::name, uma_zctor_args::size, trash_fini(), trash_init(), UMA_ANYDOMAIN, UMA_ZONE_NOFREE, UMA_ZONE_NOTOUCH, UMA_ZONE_ZINIT, uma_zctor_args::uminit, zone_alloc_item(), and zones.
Referenced by bucket_init(), swap_pager_swap_init(), uma_startup1(), vm_map_startup(), vm_object_init(), vm_pager_bufferinit(), and vm_radix_zinit().
◆ uma_zdestroy()
| void uma_zdestroy | ( | uma_zone_t | zone | ) |
Definition at line 3349 of file uma_core.c.
References booted, SKIP_NONE, uma_zone::uz_fini, uma_zone::uz_release, zone_free_item(), zone_release(), and zones.
◆ uma_zfree_arg()
| void uma_zfree_arg | ( | uma_zone_t | zone, |
| void * | item, | ||
| void * | udata | ||
| ) |
Definition at line 4417 of file uma_core.c.
References cache_bucket_push(), cache_bucket_swap(), cache_free(), cache_uz_flags(), cache_uz_size(), item_dtor(), SKIP_DTOR, SKIP_NONE, uma_cache::uc_allocbucket, uma_cache::uc_crossbucket, uma_cache::uc_freebucket, uma_cache_bucket::ucb_cnt, uma_cache_bucket::ucb_entries, UMA_ALWAYS_CTORDTOR, UMA_ZFLAG_CTORDTOR, UMA_ZFLAG_LIMIT, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, uma_zone::uz_name, uma_zone::uz_sleepers, and zone_free_item().
Referenced by bucket_free(), and uma_zfree_pcpu_arg().
◆ uma_zfree_pcpu_arg()
| void uma_zfree_pcpu_arg | ( | uma_zone_t | zone, |
| void * | pcpu_item, | ||
| void * | udata | ||
| ) |
Definition at line 3404 of file uma_core.c.
References uma_zfree_arg(), UMA_ZONE_PCPU, and uma_zone::uz_flags.
◆ uma_zfree_smr()
| void uma_zfree_smr | ( | uma_zone_t | zone, |
| void * | item | ||
| ) |
Definition at line 4366 of file uma_core.c.
References cache_bucket_push(), cache_free(), cache_uz_flags(), SKIP_NONE, uma_cache::uc_crossbucket, uma_cache::uc_freebucket, uma_cache_bucket::ucb_cnt, uma_cache_bucket::ucb_entries, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_SMR, uma_zone::uz_cpu, uma_zone::uz_flags, uma_zone::uz_name, uma_zone::uz_smr, and zone_free_item().
Referenced by uma_zwait(), and vm_radix_node_put().
◆ uma_zone_exhausted()
| int uma_zone_exhausted | ( | uma_zone_t | zone | ) |
Definition at line 5304 of file uma_core.c.
References uma_zone::uz_sleepers.
Referenced by swp_pager_meta_build().
◆ uma_zone_get_allocs()
|
static |
Definition at line 4950 of file uma_core.c.
References uma_cache::uc_allocs, uma_zone::uz_allocs, and uma_zone::uz_cpu.
Referenced by sysctl_handle_uma_zone_allocs().
◆ uma_zone_get_cur()
| int uma_zone_get_cur | ( | uma_zone_t | zone | ) |
Definition at line 4933 of file uma_core.c.
References uma_cache::uc_allocs, uma_cache::uc_frees, uma_zone::uz_allocs, uma_zone::uz_cpu, and uma_zone::uz_frees.
Referenced by sysctl_handle_uma_zone_cur().
◆ uma_zone_get_frees()
|
static |
Definition at line 4965 of file uma_core.c.
References uma_cache::uc_frees, uma_zone::uz_cpu, and uma_zone::uz_frees.
Referenced by sysctl_handle_uma_zone_frees().
◆ uma_zone_get_max()
| int uma_zone_get_max | ( | uma_zone_t | zone | ) |
Definition at line 4904 of file uma_core.c.
References uma_zone::uz_max_items.
Referenced by swap_pager_swap_init(), and sysctl_handle_uma_zone_max().
◆ uma_zone_get_smr()
| smr_t uma_zone_get_smr | ( | uma_zone_t | zone | ) |
Definition at line 5073 of file uma_core.c.
References uma_zone::uz_smr.
Referenced by vm_radix_zinit().
◆ uma_zone_memory()
| size_t uma_zone_memory | ( | uma_zone_t | zone | ) |
Definition at line 5173 of file uma_core.c.
References uma_domain::ud_pages, uma_keg::uk_domain, UMA_ZFLAG_CACHE, uma_zone::uz_flags, uma_zone::uz_keg, uma_zone::uz_size, vm_ndomains, and ZDOM_GET.
◆ uma_zone_reclaim()
| void uma_zone_reclaim | ( | uma_zone_t | zone, |
| int | req | ||
| ) |
Definition at line 5278 of file uma_core.c.
References uma_reclaim_args::req, UMA_ANYDOMAIN, and uma_zone_reclaim_domain().
◆ uma_zone_reclaim_domain()
| void uma_zone_reclaim_domain | ( | uma_zone_t | zone, |
| int | req, | ||
| int | domain | ||
| ) |
Definition at line 5284 of file uma_core.c.
References uma_reclaim_args::domain, pcpu_cache_drain_safe(), uma_reclaim_args::req, UMA_RECLAIM_DRAIN, UMA_RECLAIM_DRAIN_CPU, UMA_RECLAIM_TRIM, and zone_reclaim().
Referenced by bucket_zone_drain(), uma_reclaim_domain(), uma_reclaim_domain_cb(), and uma_zone_reclaim().
◆ uma_zone_reserve()
| void uma_zone_reserve | ( | uma_zone_t | zone, |
| int | items | ||
| ) |
Definition at line 5081 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, and uma_keg::uk_reserve.
Referenced by vm_map_startup().
◆ uma_zone_reserve_kva()
| int uma_zone_reserve_kva | ( | uma_zone_t | zone, |
| int | count | ||
| ) |
Definition at line 5092 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, kva_alloc(), noobj_alloc(), uma_keg::uk_allocf, uma_keg::uk_flags, uma_keg::uk_ipers, uma_keg::uk_kva, uma_keg::uk_offset, uma_keg::uk_ppera, uma_small_alloc(), UMA_ZFLAG_LIMIT, UMA_ZONE_NOFREE, uma_zone::uz_flags, uma_zone::uz_max_items, ZONE_ASSERT_COLD, and zone_update_caches().
Referenced by swap_pager_swap_init(), and vm_radix_reserve_kva().
◆ uma_zone_set_allocf()
| void uma_zone_set_allocf | ( | uma_zone_t | zone, |
| uma_alloc | allocf | ||
| ) |
Definition at line 5048 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, and uma_keg::uk_allocf.
Referenced by vm_map_startup().
◆ uma_zone_set_fini()
| void uma_zone_set_fini | ( | uma_zone_t | zone, |
| uma_fini | fini | ||
| ) |
Definition at line 5008 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, and uma_keg::uk_fini.
◆ uma_zone_set_freef()
| void uma_zone_set_freef | ( | uma_zone_t | zone, |
| uma_free | freef | ||
| ) |
Definition at line 5037 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, and uma_keg::uk_freef.
Referenced by vm_map_startup().
◆ uma_zone_set_init()
| void uma_zone_set_init | ( | uma_zone_t | zone, |
| uma_init | uminit | ||
| ) |
Definition at line 4997 of file uma_core.c.
References KEG_ASSERT_COLD, KEG_GET, and uma_keg::uk_init.
◆ uma_zone_set_max()
| int uma_zone_set_max | ( | uma_zone_t | zone, |
| int | nitems | ||
| ) |
Definition at line 4841 of file uma_core.c.
References UMA_ZFLAG_LIMIT, uma_zone_set_maxcache(), uma_zone::uz_flags, uma_zone::uz_max_items, ZONE_LOCK, ZONE_UNLOCK, and zone_update_caches().
Referenced by pbuf_zsecond_create(), sysctl_handle_uma_zone_max(), and vm_pager_bufferinit().
◆ uma_zone_set_maxaction()
| void uma_zone_set_maxaction | ( | uma_zone_t | zone, |
| uma_maxaction_t | maxaction | ||
| ) |
Definition at line 4924 of file uma_core.c.
References uma_zone::uz_maxaction, and ZONE_ASSERT_COLD.
◆ uma_zone_set_maxcache()
| void uma_zone_set_maxcache | ( | uma_zone_t | zone, |
| int | nitems | ||
| ) |
Definition at line 4868 of file uma_core.c.
References BUCKET_MAX, UMA_ZONE_FIRSTTOUCH, uma_zone::uz_bucket_max, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_max, uma_zone::uz_bucket_size_min, uma_zone::uz_flags, vm_ndomains, ZONE_LOCK, and ZONE_UNLOCK.
Referenced by kstack_cache_init(), sysctl_kstack_cache_size(), uma_zone_set_max(), and vm_page_init_cache_zones().
◆ uma_zone_set_smr()
| void uma_zone_set_smr | ( | uma_zone_t | zone, |
| smr_t | smr | ||
| ) |
Definition at line 5059 of file uma_core.c.
References UMA_ZONE_SMR, uma_zone::uz_flags, uma_zone::uz_name, uma_zone::uz_smr, ZONE_ASSERT_COLD, and zone_update_caches().
◆ uma_zone_set_warning()
| void uma_zone_set_warning | ( | uma_zone_t | zone, |
| const char * | warning | ||
| ) |
Definition at line 4915 of file uma_core.c.
References uma_zone::uz_warning, and ZONE_ASSERT_COLD.
◆ uma_zone_set_zfini()
| void uma_zone_set_zfini | ( | uma_zone_t | zone, |
| uma_fini | zfini | ||
| ) |
Definition at line 5028 of file uma_core.c.
References uma_zone::uz_fini, and ZONE_ASSERT_COLD.
◆ uma_zone_set_zinit()
| void uma_zone_set_zinit | ( | uma_zone_t | zone, |
| uma_init | zinit | ||
| ) |
Definition at line 5019 of file uma_core.c.
References uma_zone::uz_init, and ZONE_ASSERT_COLD.
◆ uma_zsecond_create()
| uma_zone_t uma_zsecond_create | ( | const char * | name, |
| uma_ctor | ctor, | ||
| uma_dtor | dtor, | ||
| uma_init | zinit, | ||
| uma_fini | zfini, | ||
| uma_zone_t | primary | ||
| ) |
Definition at line 3297 of file uma_core.c.
References uma_zctor_args::align, uma_zctor_args::ctor, uma_zctor_args::dtor, uma_zctor_args::fini, uma_zctor_args::flags, uma_zctor_args::keg, uma_zctor_args::name, uma_zctor_args::size, uma_keg::uk_align, uma_keg::uk_flags, uma_keg::uk_size, UMA_ANYDOMAIN, UMA_ZONE_SECONDARY, uma_zctor_args::uminit, uma_zone::uz_keg, zone_alloc_item(), and zones.
Referenced by pbuf_zsecond_create().
◆ uma_zwait()
| void uma_zwait | ( | uma_zone_t | zone | ) |
Definition at line 3365 of file uma_core.c.
References uma_zalloc(), uma_zalloc_pcpu(), uma_zalloc_smr(), uma_zfree(), uma_zfree_pcpu(), uma_zfree_smr(), UMA_ZONE_PCPU, UMA_ZONE_SMR, and uma_zone::uz_flags.
Referenced by swp_pager_meta_build(), and vm_radix_wait().
◆ vm_radix_reserve_kva()
| void vm_radix_reserve_kva | ( | void | ) |
Definition at line 354 of file vm_radix.c.
References uma_zone_reserve_kva(), vm_cnt, and vm_radix_node_zone.
Referenced by uma_startup2().
◆ zero_init()
|
static |
Definition at line 2145 of file uma_core.c.
Referenced by keg_ctor(), and uma_startup1().
◆ zone_alloc_bucket()
|
static |
Definition at line 4219 of file uma_core.c.
References bucket_alloc(), bucket_free(), kasan_mark_item_invalid(), kasan_mark_item_valid(), uma_bucket::ub_bucket, uma_bucket::ub_cnt, uma_bucket::ub_entries, UMA_ANYDOMAIN, UMA_ZONE_ROUNDROBIN, uma_zone::uz_arg, uma_zone::uz_bucket_size, uma_zone::uz_fails, uma_zone::uz_flags, uma_zone::uz_import, uma_zone::uz_init, uma_zone::uz_max_items, uma_zone::uz_name, uma_zone::uz_release, uma_zone::uz_size, VM_DOMAIN_EMPTY, zone_alloc_limit(), and zone_free_limit().
Referenced by cache_alloc().
◆ zone_alloc_counters()
|
static |
Definition at line 2572 of file uma_core.c.
References uma_zone::uz_allocs, uma_zone::uz_fails, uma_zone::uz_frees, uma_zone::uz_xdomain, and uma_kctor_args::zone.
Referenced by uma_startup_pcpu(), and zone_ctor().
◆ zone_alloc_item()
|
static |
Definition at line 4309 of file uma_core.c.
References item_ctor(), kasan_mark_item_invalid(), kasan_mark_item_valid(), SKIP_CNT, SKIP_FINI, UMA_ANYDOMAIN, uma_zone::uz_allocs, uma_zone::uz_arg, uma_zone::uz_fails, uma_zone::uz_flags, uma_zone::uz_import, uma_zone::uz_init, uma_zone::uz_max_items, uma_zone::uz_name, uma_zone::uz_size, VM_DOMAIN_EMPTY, zone_alloc_limit(), zone_free_item(), and zone_free_limit().
Referenced by cache_alloc_retry(), hash_alloc(), keg_alloc_slab(), uma_kcreate(), uma_zalloc_domain(), uma_zcache_create(), uma_zcreate(), and uma_zsecond_create().
◆ zone_alloc_limit()
|
static |
Definition at line 4163 of file uma_core.c.
References uma_zone::uz_items, uma_zone::uz_max_items, zone_alloc_limit_hard(), and zone_free_limit().
Referenced by zone_alloc_bucket(), and zone_alloc_item().
◆ zone_alloc_limit_hard()
|
static |
Definition at line 4066 of file uma_core.c.
References uma_zone::uz_items, UZ_ITEMS_COUNT, UZ_ITEMS_SLEEPER, UZ_ITEMS_SLEEPERS, UZ_ITEMS_SLEEPERS_MAX, uma_zone::uz_max_items, uma_zone::uz_sleepers, uma_zone::uz_sleeps, zone_free_limit(), zone_log_warning(), and zone_maxaction().
Referenced by zone_alloc_limit().
◆ zone_alloc_sysctl()
|
static |
Definition at line 2582 of file uma_core.c.
References sysctl_handle_uma_slab_efficiency(), sysctl_handle_uma_zone_allocs(), sysctl_handle_uma_zone_cur(), sysctl_handle_uma_zone_flags(), sysctl_handle_uma_zone_frees(), sysctl_handle_uma_zone_items(), uma_domain::ud_free_items, uma_domain::ud_free_slabs, uma_domain::ud_pages, uma_keg::uk_align, uma_keg::uk_domain, uma_keg::uk_ipers, uma_keg::uk_name, uma_keg::uk_ppera, uma_keg::uk_reserve, uma_keg::uk_rsize, UMA_ZFLAG_CACHE, UMA_ZFLAG_HASH, uma_zone::uz_bucket_max, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_max, uma_zone::uz_ctlname, uma_zone::uz_fails, uma_zone::uz_flags, uma_zone::uz_keg, uma_zone::uz_max_items, uma_zone::uz_name, uma_zone::uz_namecnt, uma_zone::uz_oid, uma_zone::uz_size, uma_zone::uz_sleepers, uma_zone::uz_sleeps, uma_zone::uz_xdomain, uma_zone_domain::uzd_bimin, uma_zone_domain::uzd_imax, uma_zone_domain::uzd_imin, uma_zone_domain::uzd_limin, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_timin, uma_zone_domain::uzd_wss, VM_DOMAIN, vm_ndomains, vmd_name, and ZDOM_GET.
Referenced by uma_startup3(), and zone_ctor().
◆ zone_count()
|
static |
Definition at line 2769 of file uma_core.c.
References uma_zone_count::count, uma_zone_count::name, uma_zone::uz_name, and uma_zone::uz_namecnt.
Referenced by zone_ctor().
◆ zone_ctor()
|
static |
Definition at line 2802 of file uma_core.c.
References uma_zctor_args::align, uma_kctor_args::align, uma_zctor_args::arg, booted, BUCKET_MAX, bucket_select(), uma_zone_count::count, uma_zctor_args::fini, uma_kctor_args::fini, uma_zctor_args::flags, uma_kctor_args::flags, uma_zctor_args::keg, keg_ctor(), uma_zone_count::name, uma_zctor_args::size, uma_kctor_args::size, trash_fini(), trash_init(), uma_keg::uk_flags, uma_keg::uk_size, uma_kcreate(), UMA_ZFLAG_CACHE, UMA_ZFLAG_CTORDTOR, UMA_ZFLAG_INHERIT, UMA_ZFLAG_INTERNAL, UMA_ZFLAG_TRASH, UMA_ZONE_FIRSTTOUCH, UMA_ZONE_INHERIT, UMA_ZONE_MAXBUCKET, UMA_ZONE_MTXCLASS, UMA_ZONE_NOBUCKET, UMA_ZONE_NOFREE, UMA_ZONE_NOKASAN, UMA_ZONE_ROUNDROBIN, UMA_ZONE_SECONDARY, UMA_ZONE_SMR, uma_zctor_args::uminit, uma_kctor_args::uminit, uma_zone::uz_allocs, uma_zone::uz_arg, uma_zone::uz_bucket_max, uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_max, uma_zone::uz_bucket_size_min, uma_zone::uz_ctor, uma_zone::uz_dtor, uma_zone::uz_fails, uma_zone::uz_fini, uma_zone::uz_flags, uma_zone::uz_frees, uma_zone::uz_import, uma_zone::uz_init, uma_zone::uz_keg, uma_zone::uz_name, uma_zone::uz_namecnt, uma_zone::uz_ratecheck, uma_zone::uz_release, uma_zone::uz_size, uma_zone::uz_sleeps, uma_zone::uz_smr, uma_zone::uz_warning, uma_zone_domain::uzd_buckets, vm_ndomains, ZDOM_GET, ZDOM_LOCK_INIT, uma_kctor_args::zone, zone_alloc_counters(), zone_alloc_sysctl(), zone_count(), ZONE_CROSS_LOCK_INIT, zone_foreach(), zone_import(), ZONE_LOCK, zone_release(), ZONE_UNLOCK, and zone_update_caches().
Referenced by uma_startup1().
◆ zone_domain_highest()
|
static |
Definition at line 743 of file uma_core.c.
References BUCKET_MAX, uzd_nitems, vm_ndomains, and ZDOM_GET.
Referenced by cache_alloc().
◆ zone_domain_imax_set()
|
static |
Definition at line 769 of file uma_core.c.
References uma_zone_domain::uzd_bimin, uma_zone_domain::uzd_imax, and uma_zone_domain::uzd_wss.
Referenced by zone_put_bucket().
◆ zone_domain_lock()
|
static |
Definition at line 694 of file uma_core.c.
References uma_zone::uz_bucket_size, uma_zone::uz_bucket_size_max, ZDOM_GET, ZDOM_LOCK, and ZDOM_OWNED.
Referenced by cache_fetch_bucket(), uma_zalloc_domain(), and zone_put_bucket().
◆ zone_domain_lowest()
|
static |
Definition at line 715 of file uma_core.c.
References vm_ndomains, and ZDOM_GET.
Referenced by zone_free_bucket().
◆ zone_domain_update_wss()
|
static |
Definition at line 1124 of file uma_core.c.
References uma_zone_domain::uzd_bimin, uma_zone_domain::uzd_imax, uma_zone_domain::uzd_imin, uma_zone_domain::uzd_limin, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_timin, uma_zone_domain::uzd_wss, and ZDOM_LOCK_ASSERT.
Referenced by bucket_cache_reclaim_domain().
◆ zone_dtor()
|
static |
Definition at line 2998 of file uma_core.c.
References cache_drain(), kegs, SKIP_NONE, uma_keg::uk_reserve, UMA_ANYDOMAIN, UMA_ZFLAG_CACHE, UMA_ZFLAG_INTERNAL, UMA_ZONE_SECONDARY, uma_zone::uz_allocs, uma_zone::uz_ctlname, uma_zone::uz_fails, uma_zone::uz_flags, uma_zone::uz_frees, uma_zone::uz_keg, uma_zone::uz_oid, uma_zone::uz_xdomain, vm_ndomains, ZDOM_GET, ZDOM_LOCK_FINI, uma_kctor_args::zone, ZONE_CROSS_LOCK_FINI, zone_free_item(), and zone_reclaim().
Referenced by uma_startup1().
◆ zone_fetch_bucket()
|
static |
Definition at line 793 of file uma_core.c.
References item_dtor(), SKIP_NONE, uma_bucket::ub_bucket, uma_bucket::ub_cnt, uma_bucket::ub_seq, UMA_ALWAYS_CTORDTOR, UMA_ZONE_SMR, uma_zone::uz_dtor, uma_zone::uz_flags, uma_zone::uz_size, uma_zone::uz_smr, uma_zone_domain::uzd_bimin, uma_zone_domain::uzd_buckets, uma_zone_domain::uzd_imax, uma_zone_domain::uzd_imin, uma_zone_domain::uzd_limin, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_seq, uma_zone_domain::uzd_timin, ZDOM_LOCK_ASSERT, and ZDOM_UNLOCK.
Referenced by bucket_cache_reclaim_domain(), cache_fetch_bucket(), and uma_zalloc_domain().
◆ zone_foreach() [1/2]
|
static |
Referenced by pcpu_cache_drain_safe(), uma_reclaim_domain(), uma_timeout(), and zone_ctor().
◆ zone_foreach() [2/2]
|
static |
Definition at line 3065 of file uma_core.c.
References zone_foreach_unlocked().
◆ zone_foreach_unlocked() [1/2]
|
static |
Referenced by uma_startup2(), uma_startup3(), uma_startup_pcpu(), and zone_foreach().
◆ zone_foreach_unlocked() [2/2]
|
static |
Definition at line 3041 of file uma_core.c.
References uma_kctor_args::zone.
◆ zone_free_bucket()
|
static |
Definition at line 4603 of file uma_core.c.
References UMA_ZONE_FIRSTTOUCH, UMA_ZONE_ROUNDROBIN, uma_zone::uz_flags, uma_zone::uz_name, vm_ndomains, zone_domain_lowest(), and zone_put_bucket().
Referenced by cache_drain_safe_cpu(), and cache_free().
◆ zone_free_item()
|
static |
Definition at line 4808 of file uma_core.c.
References item_dtor(), kasan_mark_item_invalid(), kasan_mark_item_valid(), SKIP_CNT, SKIP_NONE, UMA_ZONE_SMR, uma_zone::uz_arg, uma_zone::uz_fini, uma_zone::uz_flags, uma_zone::uz_frees, uma_zone::uz_max_items, uma_zone::uz_release, uma_zone::uz_size, uma_zone::uz_smr, and zone_free_limit().
Referenced by hash_free(), item_ctor(), keg_alloc_slab(), keg_free_slab(), uma_zdestroy(), uma_zfree_arg(), uma_zfree_smr(), zone_alloc_item(), and zone_dtor().
◆ zone_free_limit()
|
static |
Definition at line 4196 of file uma_core.c.
References uma_zone::uz_items, UZ_ITEMS_COUNT, UZ_ITEMS_SLEEPERS, and uma_zone::uz_max_items.
Referenced by bucket_drain(), zone_alloc_bucket(), zone_alloc_item(), zone_alloc_limit(), zone_alloc_limit_hard(), and zone_free_item().
◆ zone_import()
|
static |
Definition at line 4007 of file uma_core.c.
References keg_fetch_slab(), KEG_UNLOCK, slab_alloc_item(), uma_domain::ud_free_items, uma_keg::uk_domain, uma_keg::uk_reserve, UMA_ZONE_ROUNDROBIN, uma_slab::us_domain, uma_slab::us_freecount, uma_zone::uz_flags, uma_zone::uz_keg, and vm_ndomains.
Referenced by zone_ctor().
◆ zone_kva_available()
|
static |
Definition at line 2551 of file uma_core.c.
References contig_alloc(), KEG_GET, page_alloc(), pcpu_page_alloc(), startup_alloc(), uma_keg::uk_allocf, uma_keg::uk_flags, uma_keg::uk_ppera, UMA_ZFLAG_CACHE, UMA_ZONE_CONTIG, UMA_ZONE_PCPU, uma_zone::uz_flags, and uma_kctor_args::zone.
Referenced by uma_startup2().
◆ zone_log_warning()
|
static |
Definition at line 1080 of file uma_core.c.
References uma_zone::uz_name, uma_zone::uz_ratecheck, uma_zone::uz_warning, and zone_warnings.
Referenced by zone_alloc_limit_hard().
◆ zone_maxaction()
|
inlinestatic |
Definition at line 1092 of file uma_core.c.
References uma_zone::uz_maxaction.
Referenced by zone_alloc_limit_hard().
◆ zone_put_bucket()
|
static |
Definition at line 860 of file uma_core.c.
References bucket_free(), uma_bucket::ub_cnt, uma_bucket::ub_seq, uma_zone::uz_bucket_max, uma_zone_domain::uzd_bimin, uma_zone_domain::uzd_buckets, uma_zone_domain::uzd_imax, uma_zone_domain::uzd_imin, uma_zone_domain::uzd_limin, uma_zone_domain::uzd_nitems, uma_zone_domain::uzd_seq, ZDOM_UNLOCK, zone_domain_imax_set(), and zone_domain_lock().
Referenced by cache_alloc(), uma_zalloc_domain(), and zone_free_bucket().
◆ zone_reclaim()
|
static |
Definition at line 1712 of file uma_core.c.
References bucket_cache_reclaim(), keg_drain(), UMA_ZFLAG_CACHE, uma_zone::uz_flags, uma_zone::uz_keg, uma_zone::uz_reclaimers, ZONE_LOCK, ZONE_LOCKPTR, and ZONE_UNLOCK.
Referenced by uma_zone_reclaim_domain(), and zone_dtor().
◆ zone_release()
|
static |
Definition at line 4761 of file uma_core.c.
References hash_sfind(), KEG_LOCK, KEG_LOCKPTR, slab_free_item(), uma_keg::uk_hash, uma_keg::uk_pgoff, UMA_ZFLAG_HASH, UMA_ZFLAG_VTOSLAB, uma_slab::us_domain, uma_zone::uz_flags, uma_zone::uz_keg, and vtoslab().
Referenced by uma_zdestroy(), and zone_ctor().
◆ zone_timeout()
|
static |
Definition at line 1172 of file uma_core.c.
References bucket_cache_reclaim_domain(), hash_alloc(), hash_expand(), hash_free(), keg_drain(), KEG_LOCK, KEG_UNLOCK, uma_domain::ud_pages, uma_hash::uh_hashsize, uma_keg::uk_domain, uma_keg::uk_hash, uma_keg::uk_ppera, UMA_ZFLAG_CACHE, UMA_ZFLAG_HASH, UMA_ZONE_UNMANAGED, uma_zone::uz_flags, uma_zone::uz_keg, and vm_ndomains.
Referenced by uma_timeout().
◆ zone_update_caches()
|
static |
Definition at line 2785 of file uma_core.c.
References cache_set_uz_flags(), cache_set_uz_size(), uma_zone::uz_cpu, uma_zone::uz_flags, and uma_zone::uz_size.
Referenced by uma_zone_reserve_kva(), uma_zone_set_max(), uma_zone_set_smr(), and zone_ctor().
Variable Documentation
◆ booted
| booted = BOOT_COLD |
Definition at line 204 of file uma_core.c.
Referenced by bucket_alloc(), bucket_enable(), keg_ctor(), uma_shutdown(), uma_startup2(), uma_startup3(), uma_startup_pcpu(), uma_zdestroy(), and zone_ctor().
◆ bucket_zones
| struct uma_bucket_zone bucket_zones[] |
Definition at line 257 of file uma_core.c.
Referenced by bucket_init(), bucket_select(), bucket_zone_drain(), and bucket_zone_lookup().
◆ bucketdisable
|
static |
Definition at line 163 of file uma_core.c.
Referenced by bucket_enable(), cache_alloc(), and cache_free().
◆ hashzone
|
static |
Definition at line 152 of file uma_core.c.
Referenced by hash_alloc(), hash_free(), and uma_startup1().
◆ kegs
|
static |
Definition at line 119 of file uma_core.c.
Referenced by uma_kcreate(), uma_startup1(), and zone_dtor().
◆ multipage_slabs
|
static |
Definition at line 384 of file uma_core.c.
Referenced by keg_layout().
◆ slabzones
|
static |
Definition at line 146 of file uma_core.c.
Referenced by slabzone(), uma_reclaim_domain(), and uma_startup1().
◆ uma_align_cache
| int uma_align_cache = 64 - 1 |
Definition at line 155 of file uma_core.c.
Referenced by uma_kcreate(), and uma_set_align().
◆ uma_callout
|
static |
Definition at line 210 of file uma_core.c.
Referenced by uma_startup3(), and uma_timeout().
◆ uma_reclaim_needed
|
static |
Definition at line 5248 of file uma_core.c.
Referenced by uma_reclaim_wakeup(), and uma_reclaim_worker().
◆ zone_warnings
|
static |
Definition at line 380 of file uma_core.c.
Referenced by zone_log_warning().
◆ zones
|
static |
Definition at line 120 of file uma_core.c.
Referenced by uma_startup1(), uma_zcache_create(), uma_zcreate(), uma_zdestroy(), and uma_zsecond_create().
