65#include "opt_kstack_pages.h"
66#include "opt_kstack_max_pages.h"
67#include "opt_kstack_usage_prof.h"
72#include <sys/domainset.h>
73#include <sys/limits.h>
75#include <sys/malloc.h>
80#include <sys/refcount.h>
81#include <sys/resourcevar.h>
82#include <sys/rwlock.h>
84#include <sys/sf_buf.h>
87#include <sys/vmmeter.h>
90#include <sys/sysctl.h>
91#include <sys/kernel.h>
93#include <sys/unistd.h>
109#include <machine/cpu.h>
124 vm_offset_t saddr, eaddr;
128 (
"illegal ``rw'' argument to kernacc (%x)\n", rw));
131 (vm_offset_t)addr + len < (vm_offset_t)addr)
135 saddr = trunc_page((vm_offset_t)addr);
136 eaddr = round_page((vm_offset_t)addr + len);
160 (
"illegal ``rw'' argument to useracc (%x)\n", rw));
162 map = &curproc->p_vmspace->vm_map;
163 if ((vm_offset_t)addr + len >
vm_map_max(map) ||
164 (vm_offset_t)addr + len < (vm_offset_t)addr) {
169 round_page((vm_offset_t)addr + len), prot);
177 vm_offset_t end, last, start;
181 last = (vm_offset_t)addr + len;
182 start = trunc_page((vm_offset_t)addr);
183 end = round_page(last);
184 if (last < (vm_offset_t)addr || end < (vm_offset_t)addr)
186 npages = atop(end - start);
189 error =
vm_map_wire(&curproc->p_vmspace->vm_map, start, end,
192 curthread->td_vslock_sz += len;
208 MPASS(curthread->td_vslock_sz >= len);
209 curthread->td_vslock_sz -= len;
211 trunc_page((vm_offset_t)addr), round_page((vm_offset_t)addr + len),
245 return (sf_buf_alloc(m, SFB_CPUPRIVATE));
279 error = sysctl_handle_int(oidp, arg1, arg2, req);
301 ks =
kva_alloc((pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
303 printf(
"%s: kstack allocation failed\n", __func__);
307 if (KSTACK_GUARD_PAGES != 0) {
309 ks += KSTACK_GUARD_PAGES * PAGE_SIZE;
316 for (i = 0; i < pages; i++)
330 pindex = atop(ks - VM_MIN_KERNEL_ADDRESS);
334 for (i = 0; i < pages; i++) {
337 panic(
"%s: kstack already missing?", __func__);
343 kasan_mark((
void *)ks, ptoa(pages), ptoa(pages), 0);
344 kva_free(ks - (KSTACK_GUARD_PAGES * PAGE_SIZE),
345 (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE);
358 pages = kstack_pages;
377 td->td_kstack_pages = pages;
378 kasan_mark((
void *)ks, ptoa(pages), ptoa(pages), 0);
379 kmsan_mark((
void *)ks, ptoa(pages), KMSAN_STATE_UNINIT);
392 pages = td->td_kstack_pages;
395 td->td_kstack_pages = 0;
396 kasan_mark((
void *)ks, 0, ptoa(pages), KASAN_KSTACK_FREED);
397 if (pages == kstack_pages)
409 int npages,
int req_class)
414 pindex = atop(ks - VM_MIN_KERNEL_ADDRESS);
417 for (n = 0; n < npages;) {
435 struct domainset *ds;
441 ds = DOMAINSET_PREF(domain);
443 for (i = 0; i < cnt; i++) {
445 if (store[i] == NULL)
457 for (i = 0; i < cnt; i++) {
458 ks = (vm_offset_t)store[i];
467 atop(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS));
469 kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL,
477#ifdef KSTACK_USAGE_PROF
481static int max_kstack_used;
483SYSCTL_INT(_debug, OID_AUTO, max_kstack_used, CTLFLAG_RD,
485 "Maximum stack depth used by a thread in kernel");
488intr_prof_stack_use(
struct thread *td,
struct trapframe *frame)
490 vm_offset_t stack_top;
499 if (TRAPF_USERMODE(frame))
502 stack_top = td->td_kstack + td->td_kstack_pages * PAGE_SIZE;
503 current = (vm_offset_t)(uintptr_t)&stack_top;
509 if (stack_top <= current || current < td->td_kstack)
512 used = stack_top - current;
514 prev_used = max_kstack_used;
515 if (prev_used >= used)
517 if (atomic_cmpset_int(&max_kstack_used, prev_used, used))
532vm_forkproc(
struct thread *td,
struct proc *p2,
struct thread *td2,
533 struct vmspace *vm2,
int flags)
535 struct proc *p1 = td->td_proc;
536 struct domainset *dset;
539 if ((flags & RFPROC) == 0) {
545 if ((flags & RFMEM) == 0) {
550 cpu_fork(td, p2, td2, flags);
555 p2->p_vmspace = p1->p_vmspace;
556 refcount_acquire(&p1->p_vmspace->vm_refcnt);
558 dset = td2->td_domain.dr_policy;
559 while (vm_page_count_severe_set(&dset->ds_mask)) {
563 if ((flags & RFMEM) == 0) {
565 if (p1->p_vmspace->vm_shm)
573 cpu_fork(td, p2, td2, flags);
SYSCTL_INT(_vm_memguard, OID_AUTO, options, CTLFLAG_RWTUN, &memguard_options, 0, "MemGuard options:\n" "\t0x001 - add guard pages around each allocation\n" "\t0x002 - always use MemGuard for allocations over a page\n" "\t0x004 - guard uma(9) zones with UMA_ZONE_NOFREE flag")
void pmap_qenter(vm_offset_t, vm_page_t *, int)
void pmap_sync_icache(pmap_t, vm_offset_t, vm_size_t)
void pmap_qremove(vm_offset_t, int)
struct domainset_ref domain
static __inline void uma_zfree(uma_zone_t zone, void *item)
static __inline void * uma_zalloc(uma_zone_t zone, int flags)
#define UMA_ZONE_FIRSTTOUCH
void uma_zone_set_maxcache(uma_zone_t zone, int nitems)
uma_zone_t uma_zcache_create(const char *name, int size, uma_ctor ctor, uma_dtor dtor, uma_init zinit, uma_fini zfini, uma_import zimport, uma_release zrelease, void *arg, int flags)
int vm_wait_doms(const domainset_t *, int mflags)
int vmspace_unshare(struct proc *)
void vmspace_exitfree(struct proc *)
vm_offset_t kva_alloc(vm_size_t)
void kva_free(vm_offset_t, vm_size_t)
int vm_forkproc(struct thread *td, struct proc *p2, struct thread *td2, struct vmspace *vm2, int flags)
static void kstack_cache_init(void *null)
void vsunlock(void *addr, size_t len)
static vm_page_t vm_imgact_hold_page(vm_object_t object, vm_ooffset_t offset)
vm_object_t kstack_object
SYSCTL_PROC(_vm, OID_AUTO, kstack_cache_size, CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &kstack_cache_size, 0, sysctl_kstack_cache_size, "IU", "Maximum number of cached kernel stacks")
static void kstack_release(void *arg, void **store, int cnt)
int vm_thread_new(struct thread *td, int pages)
static uma_zone_t kstack_cache
int useracc(void *addr, int len, int rw)
struct sf_buf * vm_imgact_map_page(vm_object_t object, vm_ooffset_t offset)
static int kstack_import(void *arg, void **store, int cnt, int domain, int flags)
void vm_waitproc(struct proc *p)
int kernacc(void *addr, int len, int rw)
static vm_offset_t vm_thread_stack_create(struct domainset *ds, int pages)
static int sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS)
void vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset_t sz)
int vslock(void *addr, size_t len)
static int kstack_cache_size
static void vm_thread_stack_dispose(vm_offset_t ks, int pages)
SYSINIT(vm_kstacks, SI_SUB_KMEM, SI_ORDER_ANY, kstack_cache_init, NULL)
void vm_imgact_unmap_page(struct sf_buf *sf)
void vm_thread_stack_back(struct domainset *ds, vm_offset_t ks, vm_page_t ma[], int npages, int req_class)
void vm_thread_dispose(struct thread *td)
int vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags)
boolean_t vm_map_check_protection(vm_map_t map, vm_offset_t start, vm_offset_t end, vm_prot_t protection)
int vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags)
#define VM_MAP_WIRE_SYSTEM
#define vm_map_unlock_read(map)
static __inline vm_offset_t vm_map_max(const struct vm_map *map)
#define vm_map_lock_read(map)
#define VM_MAP_WIRE_NOHOLES
vm_object_t vm_object_allocate(objtype_t type, vm_pindex_t size)
#define VM_OBJECT_WLOCK(object)
#define VM_OBJECT_WUNLOCK(object)
int vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags, vm_page_t *ma, int count)
int vm_page_grab_valid_unlocked(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
vm_page_t vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
void vm_page_valid(vm_page_t m)
bool vm_page_unwire_noq(vm_page_t m)
void vm_page_unwire(vm_page_t m, uint8_t nqueue)
void vm_page_free(vm_page_t m)
#define VM_ALLOC_WAITFAIL
#define vm_page_xbusy_claim(m)
u_long vm_page_max_user_wired