82#include <sys/disklabel.h>
83#include <sys/eventhandler.h>
85#include <sys/limits.h>
87#include <sys/kernel.h>
90#include <sys/malloc.h>
91#include <sys/pctrie.h>
95#include <sys/resource.h>
96#include <sys/resourcevar.h>
97#include <sys/rwlock.h>
99#include <sys/sysctl.h>
100#include <sys/sysproto.h>
101#include <sys/systm.h>
103#include <sys/unistd.h>
105#include <sys/vmmeter.h>
106#include <sys/vnode.h>
108#include <security/mac/mac_framework.h>
123#include <geom/geom.h>
129#ifndef MAX_PAGEOUT_CLUSTER
130#define MAX_PAGEOUT_CLUSTER 32
133#if !defined(SWB_NPAGES)
134#define SWB_NPAGES MAX_PAGEOUT_CLUSTER
137#define SWAP_META_PAGES PCTRIE_COUNT
153static struct swdevt *swdevhd;
156static struct sx swdev_syscall_lock;
158static __exclusive_cache_line u_long swap_reserved;
159static u_long swap_total;
160static int sysctl_page_shift(SYSCTL_HANDLER_ARGS);
162static SYSCTL_NODE(_vm_stats, OID_AUTO, swap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
165SYSCTL_PROC(_vm, OID_AUTO, swap_reserved, CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
166 &swap_reserved, 0, sysctl_page_shift,
"A",
167 "Amount of swap storage needed to back all allocated anonymous memory.");
168SYSCTL_PROC(_vm, OID_AUTO, swap_total, CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE,
169 &swap_total, 0, sysctl_page_shift,
"A",
170 "Total amount of available swap storage.");
172static int overcommit = 0;
174 "Configure virtual memory overcommit behavior. See tuning(7) "
176static unsigned long swzone;
177SYSCTL_ULONG(_vm, OID_AUTO, swzone, CTLFLAG_RD, &swzone, 0,
178 "Actual size of swap metadata zone");
179static unsigned long swap_maxpages;
180SYSCTL_ULONG(_vm, OID_AUTO, swap_maxpages, CTLFLAG_RD, &swap_maxpages, 0,
181 "Maximum amount of swap supported");
185 CTLFLAG_RD, &swap_free_deferred,
186 "Number of pages that deferred freeing swap space");
190 CTLFLAG_RD, &swap_free_completed,
191 "Number of deferred frees completed");
194#define SWAP_RESERVE_FORCE_ON (1 << 0)
195#define SWAP_RESERVE_RLIMIT_ON (1 << 1)
196#define SWAP_RESERVE_ALLOW_NONWIRED (1 << 2)
199sysctl_page_shift(SYSCTL_HANDLER_ARGS)
202 u_long value = *(u_long *)arg1;
204 newval = ((uint64_t)value) << PAGE_SHIFT;
205 return (sysctl_handle_64(oidp, &newval, 0, req));
214 uip = cred->cr_ruidinfo;
216 prev = atomic_fetchadd_long(&uip->ui_vmsize, pincr);
218 prev + pincr > lim_cur(curthread, RLIMIT_SWAP) &&
219 priv_check(curthread, PRIV_VM_SWAP_NORLIMIT) != 0) {
220 prev = atomic_fetchadd_long(&uip->ui_vmsize, -pincr);
221 KASSERT(prev >= pincr, (
"negative vmsize for uid = %d\n", uip->ui_uid));
235 uip = cred->cr_ruidinfo;
238 prev = atomic_fetchadd_long(&uip->ui_vmsize, -pdecr);
239 KASSERT(prev >= pdecr, (
"negative vmsize for uid = %d\n", uip->ui_uid));
241 atomic_subtract_long(&uip->ui_vmsize, pdecr);
250 uip = cred->cr_ruidinfo;
251 atomic_add_long(&uip->ui_vmsize, pincr);
264 u_long r, s, prev, pincr;
270 static struct timeval lastfail;
272 KASSERT((incr & PAGE_MASK) == 0, (
"%s: incr: %ju & PAGE_MASK", __func__,
276 if (RACCT_ENABLED()) {
278 error = racct_add(curproc, RACCT_SWAP, incr);
279 PROC_UNLOCK(curproc);
286 prev = atomic_fetchadd_long(&swap_reserved, pincr);
289 oc = atomic_load_int(&overcommit);
295 priv_check(curthread, PRIV_VM_SWAP_NOQUOTA) != 0) {
296 prev = atomic_fetchadd_long(&swap_reserved, -pincr);
297 KASSERT(prev >= pincr, (
"swap_reserved < incr on overcommit fail"));
302 prev = atomic_fetchadd_long(&swap_reserved, -pincr);
303 KASSERT(prev >= pincr, (
"swap_reserved < incr on overcommit fail"));
310 if (ppsratecheck(&lastfail, &curfail, 1)) {
311 printf(
"uid %d, pid %d: swap reservation for %jd bytes failed\n",
312 cred->cr_ruidinfo->ui_uid, curproc->p_pid, incr);
315 if (RACCT_ENABLED()) {
317 racct_sub(curproc, RACCT_SWAP, incr);
318 PROC_UNLOCK(curproc);
330 KASSERT((incr & PAGE_MASK) == 0, (
"%s: incr: %ju & PAGE_MASK", __func__,
334 if (RACCT_ENABLED()) {
336 racct_add_force(curproc, RACCT_SWAP, incr);
337 PROC_UNLOCK(curproc);
341 atomic_add_long(&swap_reserved, pincr);
351 cred = curproc->p_ucred;
353 PROC_UNLOCK(curproc);
364 KASSERT((decr & PAGE_MASK) == 0, (
"%s: decr: %ju & PAGE_MASK", __func__,
369 prev = atomic_fetchadd_long(&swap_reserved, -pdecr);
370 KASSERT(prev >= pdecr, (
"swap_reserved < decr"));
372 atomic_subtract_long(&swap_reserved, pdecr);
378 racct_sub_cred(cred, RACCT_SWAP, decr);
390SYSCTL_PROC(_vm, OID_AUTO, swap_async_max, CTLTYPE_INT | CTLFLAG_RW |
392 "Maximum running async swap ops");
394SYSCTL_PROC(_vm, OID_AUTO, swap_fragmentation, CTLTYPE_STRING | CTLFLAG_RD |
396 "Swap Fragmentation Info");
407#define NOBJLIST(handle) \
408 (&swap_pager_object_list[((int)(intptr_t)handle >> 4) & (NOBJLISTS-1)])
423 vm_prot_t prot, vm_ooffset_t offset,
struct ucred *);
436 vm_offset_t start, vm_offset_t end);
438 vm_offset_t start, vm_offset_t end);
465 "Maximum size of a swap block in pages");
472static int swaponvp(
struct thread *,
struct vnode *, u_long);
488 vm_pindex_t pindex, vm_pindex_t count);
496 *start = SWAPBLK_NONE;
504 if (*start + *num == addr) {
518 M_USE_RESERVE : 0)));
547 printf(
"swap_pager: out of swap space\n");
574 mtx_init(&
sw_dev_mtx,
"swapdev", NULL, MTX_DEF);
576 sx_init(&swdev_syscall_lock,
"swsysc");
622 mtx_init(&
swbuf_mtx,
"async swbuf mutex", NULL, MTX_DEF);
632 n = maxswzone != 0 ? maxswzone /
sizeof(
struct swblk) :
635 pctrie_zone_init, NULL, UMA_ALIGN_PTR, 0);
637 NULL, NULL, _Alignof(struct swblk) - 1, 0);
657 printf(
"Swap blk zone entries changed from %lu to %lu.\n",
661 swzone = n *
sizeof(
struct swblk);
663 printf(
"Cannot reserve swap pctrie zone, "
664 "reduce kern.maxswzone.\n");
669 vm_ooffset_t size, vm_ooffset_t offset)
677 object->un_pager.swp.writemappings = 0;
678 object->handle = handle;
681 object->charge = size;
688 vm_ooffset_t size, vm_ooffset_t offset)
720 vm_ooffset_t offset,
struct ucred *cred)
724 if (handle != NULL) {
733 if (
object == NULL) {
736 if (
object != NULL) {
738 object, pager_object_list);
764 KASSERT((object->
flags &
OBJ_DEAD) != 0, (
"dealloc of reachable obj"));
788 object->handle = NULL;
818 KASSERT(*io_npages >= 1,
819 (
"%s: npages not positive", __func__));
822 npages = imin(BLIST_MAX_ALLOC, mpages);
825 while (!TAILQ_EMPTY(&swtailq)) {
827 sp = TAILQ_FIRST(&swtailq);
829 blk = blist_alloc(sp->
sw_blist, &npages, mpages);
830 if (blk != SWAPBLK_NONE)
832 sp = TAILQ_NEXT(sp, sw_list);
840 if (blk != SWAPBLK_NONE) {
846 swdevhd = TAILQ_NEXT(sp, sw_list);
849 printf(
"swp_pager_getswapspace(%d): failed\n",
873 TAILQ_FOREACH(sp, &swtailq, sw_list) {
877 unmapped_buf_allowed) {
878 bp->b_data = unmapped_buf;
882 &bp->b_pages[0], bp->b_bcount / PAGE_SIZE);
888 panic(
"Swapdev not found");
906 TAILQ_FOREACH(sp, &swtailq, sw_list) {
924 panic(
"Swapdev not found");
938 error = sysctl_wire_old_buffer(req, 0);
941 sbuf_new_for_sysctl(&sbuf, NULL, 128, req);
943 TAILQ_FOREACH(sp, &swtailq, sw_list) {
944 if (vn_isdisk(sp->
sw_vp))
945 devname = devtoname(sp->
sw_vp->v_rdev);
948 sbuf_printf(&sbuf,
"\nFree space on device %s:\n", devname);
952 error = sbuf_finish(&sbuf);
987 daddr_t addr, blk, n_free, s_free;
993 for (i = 0; i < size; i += n) {
994 n = MIN(size - i, INT_MAX);
996 if (blk == SWAPBLK_NONE) {
1001 for (j = 0; j < n; ++j) {
1003 start + i + j, blk + j);
1004 if (addr != SWAPBLK_NONE)
1016 vm_pindex_t pindex, daddr_t addr)
1021 (
"%s: Srcobject not swappable", __func__));
1034 KASSERT(dstaddr == SWAPBLK_NONE,
1035 (
"Unexpected destination swapblk"));
1061 vm_pindex_t offset,
int destroysource)
1072 srcobject->
handle != NULL) {
1094 if (destroysource) {
1126 (
"%s: object not swappable", __func__));
1132 if (blk0 == SWAPBLK_NONE) {
1143 if (before != NULL) {
1148 if (blk != blk0 - i)
1157 if (after != NULL) {
1160 if (blk != blk0 + i)
1209 counter_u64_add(swap_free_deferred, 1);
1214 counter_u64_add(swap_free_completed, 1);
1221 KASSERT((m->object->flags &
OBJ_SWAP) != 0,
1222 (
"Free object not swappable"));
1224 sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,
1248 int *rbehind,
int *rahead)
1251 vm_page_t bm, mpred, msucc, p;
1254 int i, maxahead, maxbehind, reqcount;
1260 (
"%s: object not swappable", __func__));
1266 KASSERT(reqcount - 1 <= maxahead,
1267 (
"page count %d extends beyond swap block", reqcount));
1276 maxahead = reqcount - 1;
1283 if (rahead != NULL) {
1284 *rahead = imin(*rahead, maxahead - (reqcount - 1));
1285 pindex = ma[reqcount - 1]->pindex;
1286 msucc = TAILQ_NEXT(ma[reqcount - 1], listq);
1287 if (msucc != NULL && msucc->pindex - pindex - 1 < *rahead)
1288 *rahead = msucc->pindex - pindex - 1;
1290 if (rbehind != NULL) {
1291 *rbehind = imin(*rbehind, maxbehind);
1292 pindex = ma[0]->pindex;
1293 mpred = TAILQ_PREV(ma[0], pglist, listq);
1294 if (mpred != NULL && pindex - mpred->pindex - 1 < *rbehind)
1295 *rbehind = pindex - mpred->pindex - 1;
1299 for (i = 0; i < count; i++)
1305 if (rbehind != NULL) {
1306 for (i = 1; i <= *rbehind; i++) {
1316 if (rahead != NULL) {
1317 for (i = 0; i < *rahead; i++) {
1326 if (rbehind != NULL)
1333 pindex = bm->pindex;
1335 KASSERT(blk != SWAPBLK_NONE,
1336 (
"no swap blocking containing %p(%jx)",
object, (uintmax_t)pindex));
1340 MPASS((bp->b_flags & B_MAXPHYS) != 0);
1342 for (i = 0, p = bm; i < count; i++, p = TAILQ_NEXT(p, listq)) {
1343 MPASS(p->pindex == bm->pindex + i);
1347 bp->b_flags |= B_PAGING;
1348 bp->b_iocmd = BIO_READ;
1350 bp->b_rcred = crhold(thread0.td_ucred);
1351 bp->b_wcred = crhold(thread0.td_ucred);
1353 bp->b_bcount = PAGE_SIZE * count;
1354 bp->b_bufsize = PAGE_SIZE * count;
1355 bp->b_npages = count;
1356 bp->b_pgbefore = rbehind != NULL ? *rbehind : 0;
1357 bp->b_pgafter = rahead != NULL ? *rahead : 0;
1359 VM_CNT_INC(v_swapin);
1360 VM_CNT_ADD(v_swappgsin, count);
1385 VM_CNT_INC(v_intrans);
1387 "swread", hz * 20)) {
1389"swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
1390 bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
1398 for (i = 0; i < reqcount; i++)
1399 if (ma[i]->valid != VM_PAGE_BITS_ALL)
1414 int *rbehind,
int *rahead)
1445 panic(
"unhandled swap_pager_getpages() error %d", r);
1447 (iodone)(arg, ma, count, error);
1476 int flags,
int *rtvals)
1479 daddr_t addr, blk, n_free, s_free;
1484 KASSERT(count == 0 || ma[0]->
object ==
object,
1485 (
"%s: object mismatch %p/%p",
1486 __func__,
object, ma[0]->
object));
1495 KASSERT(addr == SWAPBLK_NONE,
1496 (
"unexpected object swap block"));
1509 for (i = 0; i < count; i += n) {
1525 if (blk == SWAPBLK_NONE) {
1531 for (j = 0; j < n; ++j)
1535 for (j = 0; j < n; ++j) {
1540 if (addr != SWAPBLK_NONE)
1543 MPASS(mreq->dirty == VM_PAGE_BITS_ALL);
1549 MPASS((bp->b_flags & B_MAXPHYS) != 0);
1551 bp->b_flags |= B_ASYNC;
1552 bp->b_flags |= B_PAGING;
1553 bp->b_iocmd = BIO_WRITE;
1555 bp->b_rcred = crhold(thread0.td_ucred);
1556 bp->b_wcred = crhold(thread0.td_ucred);
1557 bp->b_bcount = PAGE_SIZE * n;
1558 bp->b_bufsize = PAGE_SIZE * n;
1560 for (j = 0; j < n; j++)
1561 bp->b_pages[j] = ma[i + j];
1568 bp->b_dirtyend = bp->b_bcount;
1570 VM_CNT_INC(v_swapout);
1571 VM_CNT_ADD(v_swappgsout, bp->b_npages);
1580 for (j = 0; j < n; j++)
1600 bp->b_iodone = bdone;
1606 bwait(bp, PVM,
"swwrt");
1636 if (bp->b_ioflags & BIO_ERROR && bp->b_error != ENOMEM) {
1638 "swap_pager: I/O error - %s failed; blkno %ld,"
1639 "size %ld, error %d\n",
1640 ((bp->b_iocmd == BIO_READ) ?
"pagein" :
"pageout"),
1653 bp->b_data = bp->b_kvabase;
1656 object = bp->b_pages[0]->object;
1668 for (i = 0; i < bp->b_npages; ++i) {
1669 vm_page_t m = bp->b_pages[i];
1671 m->oflags &= ~VPO_SWAPINPROG;
1673 m->oflags &= ~VPO_SWAPSLEEP;
1680 if (bp->b_ioflags & BIO_ERROR) {
1687 if (bp->b_iocmd == BIO_READ) {
1700 MPASS(m->dirty == VM_PAGE_BITS_ALL);
1706 }
else if (bp->b_iocmd == BIO_READ) {
1716 KASSERT(!pmap_page_is_mapped(m),
1717 (
"swp_pager_async_iodone: page %p is mapped", m));
1718 KASSERT(m->dirty == 0,
1719 (
"swp_pager_async_iodone: page %p is dirty", m));
1722 if (i < bp->b_pgbefore ||
1723 i >= bp->b_npages - bp->b_pgafter)
1734 KASSERT(!pmap_page_is_write_mapped(m),
1735 (
"swp_pager_async_iodone: page %p is not write"
1747 if (
object != NULL) {
1759 bp->b_bufobj = NULL;
1764 if (bp->b_flags & B_ASYNC) {
1801 for (res = 0, pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
1805 if (sb->
d[i] != SWAPBLK_NONE)
1825 int i, nv, rahead, rv;
1828 (
"%s: Object not swappable", __func__));
1830 for (pi = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
1846 blk = sb->
d[i + nv];
1848 blk == SWAPBLK_NONE)
1880 }
while (--nv > 0 &&
1895 panic(
"%s: read from swap failed: %d",
1929 sx_assert(&swdev_syscall_lock, SA_XLOCKED);
1953 atomic_thread_fence_acq();
1972 if (retries > 100) {
1973 panic(
"swapoff: failed to locate %d swap blocks",
1976 pause(
"swpoff", hz / 20);
1979 EVENTHANDLER_INVOKE(swapoff, sp);
2003 for (i = start; i < limit; i++) {
2004 if (sb->
d[i] != SWAPBLK_NONE)
2038 static volatile int swblk_zone_exhausted, swpctrie_zone_exhausted;
2039 struct swblk *sb, *sb1;
2040 vm_pindex_t modpi, rdpi;
2041 daddr_t prev_swapblk;
2056 atomic_thread_fence_rel();
2060 object->un_pager.swp.writemappings = 0;
2063 (
"default pager %p with handle %p",
2064 object, object->
handle));
2070 if (swapblk == SWAPBLK_NONE)
2071 return (SWAPBLK_NONE);
2078 sb->
d[i] = SWAPBLK_NONE;
2079 if (atomic_cmpset_int(&swblk_zone_exhausted,
2081 printf(
"swblk zone ok\n");
2086 if (atomic_cmpset_int(&swblk_zone_exhausted,
2088 printf(
"swap blk zone exhausted, "
2089 "increase kern.maxswzone\n");
2091 pause(
"swzonxb", 10);
2106 error = SWAP_PCTRIE_INSERT(
2109 if (atomic_cmpset_int(&swpctrie_zone_exhausted,
2111 printf(
"swpctrie zone ok\n");
2116 if (atomic_cmpset_int(&swpctrie_zone_exhausted,
2118 printf(
"swap pctrie zone exhausted, "
2119 "increase kern.maxswzone\n");
2121 pause(
"swzonxp", 10);
2135 MPASS(sb->
p == rdpi);
2139 prev_swapblk = sb->
d[modpi];
2141 sb->
d[modpi] = swapblk;
2146 if (swapblk == SWAPBLK_NONE)
2148 return (prev_swapblk);
2160 vm_pindex_t pindex, vm_pindex_t count)
2163 daddr_t n_free, s_free;
2164 vm_pindex_t offset, last;
2165 int i, limit, start;
2173 last = pindex + count;
2177 if (sb == NULL || sb->
p >= last)
2179 start = pindex > sb->
p ? pindex - sb->
p : 0;
2182 for (i = start; i < limit; i++) {
2183 if (sb->
d[i] == SWAPBLK_NONE)
2185 if (dstobject == NULL ||
2187 sb->
p + i - offset, sb->
d[i])) {
2191 sb->
d[i] = SWAPBLK_NONE;
2230 daddr_t n_free, s_free;
2239 for (pindex = 0; (sb = SWAP_PCTRIE_LOOKUP_GE(
2243 if (sb->
d[i] == SWAPBLK_NONE)
2276 (
"Lookup object not swappable"));
2281 return (SWAPBLK_NONE);
2300 return (object->
size);
2305 return (object->
size);
2306 if (sb->
p < pindex) {
2308 if (sb->
d[i] != SWAPBLK_NONE)
2314 return (object->
size);
2317 if (sb->
d[i] != SWAPBLK_NONE)
2326 return (object->
size);
2334#ifndef _SYS_SYSPROTO_H_
2345 struct nameidata nd;
2348 error = priv_check(td, PRIV_SWAPON);
2352 sx_xlock(&swdev_syscall_lock);
2363 NDINIT(&nd, LOOKUP, ISOPEN | FOLLOW | LOCKLEAF | AUDITVNODE1,
2364 UIO_USERSPACE, uap->
name);
2369 NDFREE(&nd, NDF_ONLY_PNBUF);
2372 if (vn_isdisk_error(vp, &error)) {
2374 }
else if (vp->v_type == VREG &&
2375 (vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
2376 (error = VOP_GETATTR(vp, &attr, td->td_ucred)) == 0) {
2381 error =
swaponvp(td, vp, attr.va_size / DEV_BSIZE);
2389 sx_xunlock(&swdev_syscall_lock);
2403 if (swap_total > swap_maxpages / 2) {
2404 printf(
"warning: total configured swap (%lu pages) "
2405 "exceeds maximum recommended amount (%lu pages).\n",
2406 swap_total, swap_maxpages / 2);
2407 printf(
"warning: increase kern.maxswzone "
2408 "or reduce amount of swap.\n");
2425 nblks &= ~(ctodb(1) - 1);
2426 nblks = dbtoc(nblks);
2428 sp = malloc(
sizeof *sp, M_VMPGDATA, M_WAITOK | M_ZERO);
2429 sp->
sw_blist = blist_create(nblks, M_WAITOK);
2443 blist_free(sp->
sw_blist, howmany(BBSIZE, PAGE_SIZE),
2444 nblks - howmany(BBSIZE, PAGE_SIZE));
2448 TAILQ_FOREACH(tsp, &swtailq, sw_list) {
2449 if (tsp->
sw_end >= dvbase) {
2455 dvbase = tsp->
sw_end + 1;
2459 sp->
sw_end = dvbase + nblks;
2460 TAILQ_INSERT_TAIL(&swtailq, sp, sw_list);
2463 swap_total += nblks;
2467 EVENTHANDLER_INVOKE(swapon, sp);
2484 struct nameidata nd;
2488 error = priv_check(td, PRIV_SWAPOFF);
2491 if ((flags & ~(SWAPOFF_FORCE)) != 0)
2494 sx_xlock(&swdev_syscall_lock);
2496 NDINIT(&nd, LOOKUP, FOLLOW | AUDITVNODE1, name_seg, name);
2500 NDFREE(&nd, NDF_ONLY_PNBUF);
2504 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2505 if (sp->
sw_vp == vp)
2515 sx_xunlock(&swdev_syscall_lock);
2520#ifdef COMPAT_FREEBSD13
2522freebsd13_swapoff(
struct thread *td,
struct freebsd13_swapoff_args *uap)
2524 return (
kern_swapoff(td, uap->name, UIO_USERSPACE, 0));
2531 return (
kern_swapoff(td, uap->name, UIO_USERSPACE, uap->flags));
2542 sx_assert(&swdev_syscall_lock, SA_XLOCKED);
2544 (void) vn_lock(sp->
sw_vp, LK_EXCLUSIVE | LK_RETRY);
2545 error = mac_system_check_swapoff(cred, sp->
sw_vp);
2546 (void) VOP_UNLOCK(sp->
sw_vp);
2565 if ((flags & SWAPOFF_FORCE) == 0 &&
2575 swap_total -= nblks;
2586 TAILQ_REMOVE(&swtailq, sp, sw_list);
2588 if (nswapdev == 0) {
2596 free(sp, M_VMPGDATA);
2604 const char *devname;
2607 sx_xlock(&swdev_syscall_lock);
2610 TAILQ_FOREACH_SAFE(sp, &swtailq, sw_list, spt) {
2612 if (vn_isdisk(sp->
sw_vp))
2613 devname = devtoname(sp->
sw_vp->v_rdev);
2616 error =
swapoff_one(sp, thread0.td_ucred, SWAPOFF_FORCE);
2618 printf(
"Cannot remove swap device %s (error=%d), "
2619 "skipping.\n", devname, error);
2620 }
else if (bootverbose) {
2621 printf(
"Swap device %s removed.\n", devname);
2627 sx_xunlock(&swdev_syscall_lock);
2634 *total = swap_total;
2636 nswapdev * howmany(BBSIZE, PAGE_SIZE);
2643 const char *tmp_devname;
2649 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2659 if (devname != NULL) {
2660 if (vn_isdisk(sp->
sw_vp))
2661 tmp_devname = devtoname(sp->
sw_vp->v_rdev);
2663 tmp_devname =
"[file]";
2664 strncpy(devname, tmp_devname, len);
2673#if defined(COMPAT_FREEBSD11)
2674#define XSWDEV_VERSION_11 1
2684#if defined(__amd64__) && defined(COMPAT_FREEBSD32)
2687 u_int xsw_dev1, xsw_dev2;
2698#if defined(__amd64__) && defined(COMPAT_FREEBSD32)
2699 struct xswdev32 xs32;
2701#if defined(COMPAT_FREEBSD11)
2702 struct xswdev11 xs11;
2709 memset(&xs, 0,
sizeof(xs));
2713#if defined(__amd64__) && defined(COMPAT_FREEBSD32)
2714 if (req->oldlen ==
sizeof(xs32)) {
2715 memset(&xs32, 0,
sizeof(xs32));
2718 xs32.xsw_dev2 = xs.
xsw_dev >> 32;
2722 error = SYSCTL_OUT(req, &xs32,
sizeof(xs32));
2726#if defined(COMPAT_FREEBSD11)
2727 if (req->oldlen ==
sizeof(xs11)) {
2728 memset(&xs11, 0,
sizeof(xs11));
2729 xs11.xsw_version = XSWDEV_VERSION_11;
2734 error = SYSCTL_OUT(req, &xs11,
sizeof(xs11));
2738 error = SYSCTL_OUT(req, &xs,
sizeof(xs));
2743 "Number of swap devices");
2746 "Swap statistics by device");
2779 sb = SWAP_PCTRIE_LOOKUP_GE(
2781 if (sb == NULL || sb->
p >= e)
2784 if (sb->
p + i < e &&
2785 sb->
d[i] != SWAPBLK_NONE)
2806 .version = G_VERSION,
2815 struct g_consumer *cp;
2818 g_access(cp, -1, -1, 0);
2820 g_destroy_consumer(cp);
2845 if (cp->index == 0) {
2856 struct g_consumer *cp;
2858 bp = bp2->bio_caller2;
2860 bp->b_ioflags = bp2->bio_flags;
2862 bp->b_ioflags |= BIO_ERROR;
2863 bp->b_resid = bp->b_bcount - bp2->bio_completed;
2864 bp->b_error = bp2->bio_error;
2865 bp->b_caller1 = NULL;
2867 sp = bp2->bio_caller1;
2878 struct g_consumer *cp;
2884 bp->b_error = ENXIO;
2885 bp->b_ioflags |= BIO_ERROR;
2891 if (bp->b_iocmd == BIO_WRITE)
2894 bio = g_alloc_bio();
2899 bp->b_error = ENOMEM;
2900 bp->b_ioflags |= BIO_ERROR;
2901 printf(
"swap_pager: cannot allocate bio\n");
2906 bp->b_caller1 = bio;
2907 bio->bio_caller1 = sp;
2908 bio->bio_caller2 = bp;
2909 bio->bio_cmd = bp->b_iocmd;
2910 bio->bio_offset = (bp->b_blkno - sp->
sw_first) * PAGE_SIZE;
2911 bio->bio_length = bp->b_bcount;
2913 bio->bio_flags |= BIO_SWAP;
2914 if (!buf_mapped(bp)) {
2915 bio->bio_ma = bp->b_pages;
2916 bio->bio_data = unmapped_buf;
2917 bio->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
2918 bio->bio_ma_n = bp->b_npages;
2919 bio->bio_flags |= BIO_UNMAPPED;
2921 bio->bio_data = bp->b_data;
2924 g_io_request(bio, cp);
2935 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2936 if (sp->
sw_id == cp) {
2947 destroy = ((sp != NULL) && (cp->index == 0));
2958 struct g_consumer *cp;
2977 struct g_provider *pp;
2978 struct g_consumer *cp;
2979 static struct g_geom *gp;
2984 pp = g_dev_getprovider(dev);
2988 TAILQ_FOREACH(sp, &swtailq, sw_list) {
2990 if (cp != NULL && cp->provider == pp) {
2998 cp = g_new_consumer(gp);
3000 cp->flags |= G_CF_DIRECT_SEND | G_CF_DIRECT_RECEIVE;
3008 error = g_access(cp, 1, 1, 0);
3011 g_destroy_consumer(cp);
3014 nblks = pp->mediasize / DEV_BSIZE;
3017 (pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ?
SW_UNMAPPED : 0);
3026 ASSERT_VOP_ELOCKED(vp,
"swapongeom");
3027 if (vp->v_type != VCHR || VN_IS_DOOMED(vp)) {
3032 g_topology_unlock();
3050 bp->b_blkno = ctodb(bp->b_blkno - sp->
sw_first);
3054 if (bp->b_iocmd == BIO_WRITE) {
3055 vn_lock(vp2, LK_EXCLUSIVE | LK_RETRY);
3057 bufobj_wdrop(bp->b_bufobj);
3058 bufobj_wref(&vp2->v_bufobj);
3060 vn_lock(vp2, LK_SHARED | LK_RETRY);
3062 if (bp->b_bufobj != &vp2->v_bufobj)
3063 bp->b_bufobj = &vp2->v_bufobj;
3065 bp->b_iooffset = dbtob(bp->b_blkno);
3076 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
3077 VOP_CLOSE(vp, FREAD | FWRITE, td->td_ucred, td);
3082swaponvp(
struct thread *td,
struct vnode *vp, u_long nblks)
3087 ASSERT_VOP_ELOCKED(vp,
"swaponvp");
3091 TAILQ_FOREACH(sp, &swtailq, sw_list) {
3092 if (sp->
sw_id == vp) {
3100 error = mac_system_check_swapon(td->td_ucred, vp);
3103 error = VOP_OPEN(vp, FREAD | FWRITE, td->td_ucred, td, NULL);
3118 error = sysctl_handle_int(oidp, &
new, 0, req);
3119 if (error != 0 || req->newptr == NULL)
3122 if (
new > nswbuf / 2 ||
new < 1)
3156 (
"Splittable object with writecount"));
3157 object->un_pager.swp.writemappings += (vm_ooffset_t)end - start;
3168 (
"Splittable object with writecount"));
3169 object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start;
SYSCTL_ULONG(_vm_memguard, OID_AUTO, mapsize, CTLFLAG_RD, &memguard_mapsize, 0, "MemGuard private arena size")
void pmap_qenter(vm_offset_t, vm_page_t *, int)
void pmap_qremove(vm_offset_t, int)
daddr_t d[SWAP_META_PAGES]
sw_strategy_t * sw_strategy
union vm_map_object object
struct vm_object::@0::@4 swp
union vm_object::@0 un_pager
void uma_zwait(uma_zone_t zone)
int uma_zone_exhausted(uma_zone_t zone)
static __inline void uma_zfree(uma_zone_t zone, void *item)
int uma_zone_get_max(uma_zone_t zone)
static __inline void * uma_zalloc(uma_zone_t zone, int flags)
uma_zone_t uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, int align, uint32_t flags)
int uma_zone_reserve_kva(uma_zone_t zone, int nitems)
struct vm_object * vm_object
#define VM_MAP_ENTRY_FOREACH(it, map)
#define MAP_ENTRY_IS_SUB_MAP
struct vmmeter __read_mostly vm_cnt
u_int vm_free_count(void)
void vm_object_pip_wakeupn(vm_object_t object, short i)
struct mtx vm_object_list_mtx
void vm_object_pip_wait(vm_object_t object, const char *waitid)
static COUNTER_U64_DEFINE_EARLY(object_collapses)
SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD, &object_collapses, "VM object collapses")
void vm_object_pip_add(vm_object_t object, short i)
void vm_object_clear_flag(vm_object_t object, u_short bits)
vm_object_t vm_object_allocate(objtype_t type, vm_pindex_t size)
struct object_q vm_object_list
void vm_object_deallocate(vm_object_t object)
#define VM_OBJECT_RLOCK(object)
#define VM_OBJECT_SLEEP(object, wchan, pri, wmesg, timo)
#define VM_OBJECT_RUNLOCK(object)
#define VM_OBJECT_ASSERT_LOCKED(object)
#define VM_OBJECT_WLOCK(object)
static __inline void vm_object_set_flag(vm_object_t object, u_short bits)
#define VM_OBJECT_WOWNED(object)
#define VM_OBJECT_WUNLOCK(object)
#define VM_OBJECT_ASSERT_WLOCKED(object)
void vm_page_activate(vm_page_t m)
vm_page_t vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
void vm_page_launder(vm_page_t m)
vm_page_t vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
bool vm_page_busy_acquire(vm_page_t m, int allocflags)
void vm_page_readahead_finish(vm_page_t m)
void vm_page_valid(vm_page_t m)
vm_page_t vm_page_next(vm_page_t m)
void vm_page_invalid(vm_page_t m)
void vm_page_sunbusy(vm_page_t m)
void vm_page_deactivate_noreuse(vm_page_t m)
static void vm_page_aflag_set(vm_page_t m, uint16_t bits)
static bool vm_page_all_valid(vm_page_t m)
#define VM_PAGE_OBJECT_BUSY_ASSERT(m)
static __inline void vm_page_undirty(vm_page_t m)
#define VM_ALLOC_WAITFAIL
static void vm_page_aflag_clear(vm_page_t m, uint16_t bits)
#define vm_page_xunbusy(m)
static __inline void vm_page_dirty(vm_page_t m)
void vm_pageout_oom(int shortage)