FreeBSD virtual memory subsystem code
vm_object.c
Go to the documentation of this file.
1/*-
2 * SPDX-License-Identifier: (BSD-3-Clause AND MIT-CMU)
3 *
4 * Copyright (c) 1991, 1993
5 * The Regents of the University of California. All rights reserved.
6 *
7 * This code is derived from software contributed to Berkeley by
8 * The Mach Operating System project at Carnegie-Mellon University.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * from: @(#)vm_object.c 8.5 (Berkeley) 3/22/94
35 *
36 *
37 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
38 * All rights reserved.
39 *
40 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
41 *
42 * Permission to use, copy, modify and distribute this software and
43 * its documentation is hereby granted, provided that both the copyright
44 * notice and this permission notice appear in all copies of the
45 * software, derivative works or modified versions, and any portions
46 * thereof, and that both notices appear in supporting documentation.
47 *
48 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
49 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
50 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
51 *
52 * Carnegie Mellon requests users of this software to return to
53 *
54 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
55 * School of Computer Science
56 * Carnegie Mellon University
57 * Pittsburgh PA 15213-3890
58 *
59 * any improvements or extensions that they make and grant Carnegie the
60 * rights to redistribute these changes.
61 */
62
63/*
64 * Virtual memory object module.
65 */
66
67#include <sys/cdefs.h>
68__FBSDID("$FreeBSD$");
69
70#include "opt_vm.h"
71
72#include <sys/param.h>
73#include <sys/systm.h>
74#include <sys/blockcount.h>
75#include <sys/cpuset.h>
76#include <sys/limits.h>
77#include <sys/lock.h>
78#include <sys/mman.h>
79#include <sys/mount.h>
80#include <sys/kernel.h>
81#include <sys/pctrie.h>
82#include <sys/sysctl.h>
83#include <sys/mutex.h>
84#include <sys/proc.h> /* for curproc, pageproc */
85#include <sys/refcount.h>
86#include <sys/socket.h>
87#include <sys/resourcevar.h>
88#include <sys/refcount.h>
89#include <sys/rwlock.h>
90#include <sys/user.h>
91#include <sys/vnode.h>
92#include <sys/vmmeter.h>
93#include <sys/sx.h>
94
95#include <vm/vm.h>
96#include <vm/vm_param.h>
97#include <vm/pmap.h>
98#include <vm/vm_map.h>
99#include <vm/vm_object.h>
100#include <vm/vm_page.h>
101#include <vm/vm_pageout.h>
102#include <vm/vm_pager.h>
103#include <vm/vm_phys.h>
104#include <vm/vm_pagequeue.h>
105#include <vm/swap_pager.h>
106#include <vm/vm_kern.h>
107#include <vm/vm_extern.h>
108#include <vm/vm_radix.h>
109#include <vm/vm_reserv.h>
110#include <vm/uma.h>
111
112static int old_msync;
113SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
114 "Use old (insecure) msync behavior");
115
116static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p,
117 int pagerflags, int flags, boolean_t *allclean,
118 boolean_t *eio);
119static boolean_t vm_object_page_remove_write(vm_page_t p, int flags,
120 boolean_t *allclean);
121static void vm_object_backing_remove(vm_object_t object);
122
123/*
124 * Virtual memory objects maintain the actual data
125 * associated with allocated virtual memory. A given
126 * page of memory exists within exactly one object.
127 *
128 * An object is only deallocated when all "references"
129 * are given up. Only one "reference" to a given
130 * region of an object should be writeable.
131 *
132 * Associated with each object is a list of all resident
133 * memory pages belonging to that object; this list is
134 * maintained by the "vm_page" module, and locked by the object's
135 * lock.
136 *
137 * Each object also records a "pager" routine which is
138 * used to retrieve (and store) pages to the proper backing
139 * storage. In addition, objects may be backed by other
140 * objects from which they were virtual-copied.
141 *
142 * The only items within the object structure which are
143 * modified after time of creation are:
144 * reference count locked by object's lock
145 * pager routine locked by object's lock
146 *
147 */
148
149struct object_q vm_object_list;
150struct mtx vm_object_list_mtx; /* lock for object list and count */
151
152struct vm_object kernel_object_store;
153
154static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
155 "VM object stats");
156
157static COUNTER_U64_DEFINE_EARLY(object_collapses);
158SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD,
159 &object_collapses,
160 "VM object collapses");
161
162static COUNTER_U64_DEFINE_EARLY(object_bypasses);
163SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, bypasses, CTLFLAG_RD,
164 &object_bypasses,
165 "VM object bypasses");
166
167static COUNTER_U64_DEFINE_EARLY(object_collapse_waits);
168SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapse_waits, CTLFLAG_RD,
169 &object_collapse_waits,
170 "Number of sleeps for collapse");
171
172static uma_zone_t obj_zone;
173
174static int vm_object_zinit(void *mem, int size, int flags);
175
176#ifdef INVARIANTS
177static void vm_object_zdtor(void *mem, int size, void *arg);
178
179static void
180vm_object_zdtor(void *mem, int size, void *arg)
181{
182 vm_object_t object;
183
184 object = (vm_object_t)mem;
185 KASSERT(object->ref_count == 0,
186 ("object %p ref_count = %d", object, object->ref_count));
187 KASSERT(TAILQ_EMPTY(&object->memq),
188 ("object %p has resident pages in its memq", object));
189 KASSERT(vm_radix_is_empty(&object->rtree),
190 ("object %p has resident pages in its trie", object));
191#if VM_NRESERVLEVEL > 0
192 KASSERT(LIST_EMPTY(&object->rvq),
193 ("object %p has reservations",
194 object));
195#endif
196 KASSERT(!vm_object_busied(object),
197 ("object %p busy = %d", object, blockcount_read(&object->busy)));
198 KASSERT(object->resident_page_count == 0,
199 ("object %p resident_page_count = %d",
200 object, object->resident_page_count));
201 KASSERT(atomic_load_int(&object->shadow_count) == 0,
202 ("object %p shadow_count = %d",
203 object, atomic_load_int(&object->shadow_count)));
204 KASSERT(object->type == OBJT_DEAD,
205 ("object %p has non-dead type %d",
206 object, object->type));
207}
208#endif
209
210static int
211vm_object_zinit(void *mem, int size, int flags)
212{
213 vm_object_t object;
214
215 object = (vm_object_t)mem;
216 rw_init_flags(&object->lock, "vm object", RW_DUPOK | RW_NEW);
217
218 /* These are true for any object that has been freed */
219 object->type = OBJT_DEAD;
220 vm_radix_init(&object->rtree);
221 refcount_init(&object->ref_count, 0);
222 blockcount_init(&object->paging_in_progress);
223 blockcount_init(&object->busy);
224 object->resident_page_count = 0;
225 atomic_store_int(&object->shadow_count, 0);
226 object->flags = OBJ_DEAD;
227
228 mtx_lock(&vm_object_list_mtx);
229 TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
230 mtx_unlock(&vm_object_list_mtx);
231 return (0);
232}
233
234static void
235_vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags,
236 vm_object_t object, void *handle)
237{
238
239 TAILQ_INIT(&object->memq);
240 LIST_INIT(&object->shadow_head);
241
242 object->type = type;
243 object->flags = flags;
244 if ((flags & OBJ_SWAP) != 0)
245 pctrie_init(&object->un_pager.swp.swp_blks);
246
247 /*
248 * Ensure that swap_pager_swapoff() iteration over object_list
249 * sees up to date type and pctrie head if it observed
250 * non-dead object.
251 */
252 atomic_thread_fence_rel();
253
254 object->pg_color = 0;
255 object->size = size;
256 object->domain.dr_policy = NULL;
257 object->generation = 1;
258 object->cleangeneration = 1;
259 refcount_init(&object->ref_count, 1);
260 object->memattr = VM_MEMATTR_DEFAULT;
261 object->cred = NULL;
262 object->charge = 0;
263 object->handle = handle;
264 object->backing_object = NULL;
265 object->backing_object_offset = (vm_ooffset_t) 0;
266#if VM_NRESERVLEVEL > 0
267 LIST_INIT(&object->rvq);
268#endif
269 umtx_shm_object_init(object);
270}
271
272/*
273 * vm_object_init:
274 *
275 * Initialize the VM objects module.
276 */
277void
278vm_object_init(void)
279{
280 TAILQ_INIT(&vm_object_list);
281 mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF);
282
283 rw_init(&kernel_object->lock, "kernel vm object");
284 _vm_object_allocate(OBJT_PHYS, atop(VM_MAX_KERNEL_ADDRESS -
285 VM_MIN_KERNEL_ADDRESS), OBJ_UNMANAGED, kernel_object, NULL);
286#if VM_NRESERVLEVEL > 0
287 kernel_object->flags |= OBJ_COLORED;
288 kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS);
289#endif
290 kernel_object->un_pager.phys.ops = &default_phys_pg_ops;
291
292 /*
293 * The lock portion of struct vm_object must be type stable due
294 * to vm_pageout_fallback_object_lock locking a vm object
295 * without holding any references to it.
296 *
297 * paging_in_progress is valid always. Lockless references to
298 * the objects may acquire pip and then check OBJ_DEAD.
299 */
300 obj_zone = uma_zcreate("VM OBJECT", sizeof (struct vm_object), NULL,
301#ifdef INVARIANTS
302 vm_object_zdtor,
303#else
304 NULL,
305#endif
306 vm_object_zinit, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
307
308 vm_radix_zinit();
309}
310
311void
312vm_object_clear_flag(vm_object_t object, u_short bits)
313{
314
315 VM_OBJECT_ASSERT_WLOCKED(object);
316 object->flags &= ~bits;
317}
318
319/*
320 * Sets the default memory attribute for the specified object. Pages
321 * that are allocated to this object are by default assigned this memory
322 * attribute.
323 *
324 * Presently, this function must be called before any pages are allocated
325 * to the object. In the future, this requirement may be relaxed for
326 * "default" and "swap" objects.
327 */
328int
329vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
330{
331
332 VM_OBJECT_ASSERT_WLOCKED(object);
333
334 if (object->type == OBJT_DEAD)
335 return (KERN_INVALID_ARGUMENT);
336 if (!TAILQ_EMPTY(&object->memq))
337 return (KERN_FAILURE);
338
339 object->memattr = memattr;
340 return (KERN_SUCCESS);
341}
342
343void
344vm_object_pip_add(vm_object_t object, short i)
345{
346
347 if (i > 0)
348 blockcount_acquire(&object->paging_in_progress, i);
349}
350
351void
352vm_object_pip_wakeup(vm_object_t object)
353{
354
355 vm_object_pip_wakeupn(object, 1);
356}
357
358void
359vm_object_pip_wakeupn(vm_object_t object, short i)
360{
361
362 if (i > 0)
363 blockcount_release(&object->paging_in_progress, i);
364}
365
366/*
367 * Atomically drop the object lock and wait for pip to drain. This protects
368 * from sleep/wakeup races due to identity changes. The lock is not re-acquired
369 * on return.
370 */
371static void
372vm_object_pip_sleep(vm_object_t object, const char *waitid)
373{
374
375 (void)blockcount_sleep(&object->paging_in_progress, &object->lock,
376 waitid, PVM | PDROP);
377}
378
379void
380vm_object_pip_wait(vm_object_t object, const char *waitid)
381{
382
383 VM_OBJECT_ASSERT_WLOCKED(object);
384
385 blockcount_wait(&object->paging_in_progress, &object->lock, waitid,
386 PVM);
387}
388
389void
390vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
391{
392
393 VM_OBJECT_ASSERT_UNLOCKED(object);
394
395 blockcount_wait(&object->paging_in_progress, NULL, waitid, PVM);
396}
397
398/*
399 * vm_object_allocate:
400 *
401 * Returns a new object with the given size.
402 */
403vm_object_t
404vm_object_allocate(objtype_t type, vm_pindex_t size)
405{
406 vm_object_t object;
407 u_short flags;
408
409 switch (type) {
410 case OBJT_DEAD:
411 panic("vm_object_allocate: can't create OBJT_DEAD");
412 case OBJT_DEFAULT:
413 flags = OBJ_COLORED;
414 break;
415 case OBJT_SWAP:
416 flags = OBJ_COLORED | OBJ_SWAP;
417 break;
418 case OBJT_DEVICE:
419 case OBJT_SG:
420 flags = OBJ_FICTITIOUS | OBJ_UNMANAGED;
421 break;
422 case OBJT_MGTDEVICE:
423 flags = OBJ_FICTITIOUS;
424 break;
425 case OBJT_PHYS:
426 flags = OBJ_UNMANAGED;
427 break;
428 case OBJT_VNODE:
429 flags = 0;
430 break;
431 default:
432 panic("vm_object_allocate: type %d is undefined or dynamic",
433 type);
434 }
435 object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
436 _vm_object_allocate(type, size, flags, object, NULL);
437
438 return (object);
439}
440
441vm_object_t
442vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
443{
444 vm_object_t object;
445
446 MPASS(dyntype >= OBJT_FIRST_DYN /* && dyntype < nitems(pagertab) */);
447 object = (vm_object_t)uma_zalloc(obj_zone, M_WAITOK);
448 _vm_object_allocate(dyntype, size, flags, object, NULL);
449
450 return (object);
451}
452
453/*
454 * vm_object_allocate_anon:
455 *
456 * Returns a new default object of the given size and marked as
457 * anonymous memory for special split/collapse handling. Color
458 * to be initialized by the caller.
459 */
460vm_object_t
461vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object,
462 struct ucred *cred, vm_size_t charge)
463{
464 vm_object_t handle, object;
465
466 if (backing_object == NULL)
467 handle = NULL;
468 else if ((backing_object->flags & OBJ_ANON) != 0)
469 handle = backing_object->handle;
470 else
471 handle = backing_object;
472 object = uma_zalloc(obj_zone, M_WAITOK);
473 _vm_object_allocate(OBJT_DEFAULT, size, OBJ_ANON | OBJ_ONEMAPPING,
474 object, handle);
475 object->cred = cred;
476 object->charge = cred != NULL ? charge : 0;
477 return (object);
478}
479
480static void
481vm_object_reference_vnode(vm_object_t object)
482{
483 u_int old;
484
485 /*
486 * vnode objects need the lock for the first reference
487 * to serialize with vnode_object_deallocate().
488 */
489 if (!refcount_acquire_if_gt(&object->ref_count, 0)) {
490 VM_OBJECT_RLOCK(object);
491 old = refcount_acquire(&object->ref_count);
492 if (object->type == OBJT_VNODE && old == 0)
493 vref(object->handle);
494 VM_OBJECT_RUNLOCK(object);
495 }
496}
497
498/*
499 * vm_object_reference:
500 *
501 * Acquires a reference to the given object.
502 */
503void
504vm_object_reference(vm_object_t object)
505{
506
507 if (object == NULL)
508 return;
509
510 if (object->type == OBJT_VNODE)
511 vm_object_reference_vnode(object);
512 else
513 refcount_acquire(&object->ref_count);
514 KASSERT((object->flags & OBJ_DEAD) == 0,
515 ("vm_object_reference: Referenced dead object."));
516}
517
518/*
519 * vm_object_reference_locked:
520 *
521 * Gets another reference to the given object.
522 *
523 * The object must be locked.
524 */
525void
526vm_object_reference_locked(vm_object_t object)
527{
528 u_int old;
529
530 VM_OBJECT_ASSERT_LOCKED(object);
531 old = refcount_acquire(&object->ref_count);
532 if (object->type == OBJT_VNODE && old == 0)
533 vref(object->handle);
534 KASSERT((object->flags & OBJ_DEAD) == 0,
535 ("vm_object_reference: Referenced dead object."));
536}
537
538/*
539 * Handle deallocating an object of type OBJT_VNODE.
540 */
541static void
542vm_object_deallocate_vnode(vm_object_t object)
543{
544 struct vnode *vp = (struct vnode *) object->handle;
545 bool last;
546
547 KASSERT(object->type == OBJT_VNODE,
548 ("vm_object_deallocate_vnode: not a vnode object"));
549 KASSERT(vp != NULL, ("vm_object_deallocate_vnode: missing vp"));
550
551 /* Object lock to protect handle lookup. */
552 last = refcount_release(&object->ref_count);
553 VM_OBJECT_RUNLOCK(object);
554
555 if (!last)
556 return;
557
558 if (!umtx_shm_vnobj_persistent)
559 umtx_shm_object_terminated(object);
560
561 /* vrele may need the vnode lock. */
562 vrele(vp);
563}
564
565/*
566 * We dropped a reference on an object and discovered that it had a
567 * single remaining shadow. This is a sibling of the reference we
568 * dropped. Attempt to collapse the sibling and backing object.
569 */
570static vm_object_t
571vm_object_deallocate_anon(vm_object_t backing_object)
572{
573 vm_object_t object;
574
575 /* Fetch the final shadow. */
576 object = LIST_FIRST(&backing_object->shadow_head);
577 KASSERT(object != NULL &&
578 atomic_load_int(&backing_object->shadow_count) == 1,
579 ("vm_object_anon_deallocate: ref_count: %d, shadow_count: %d",
580 backing_object->ref_count,
581 atomic_load_int(&backing_object->shadow_count)));
582 KASSERT((object->flags & OBJ_ANON) != 0,
583 ("invalid shadow object %p", object));
584
585 if (!VM_OBJECT_TRYWLOCK(object)) {
586 /*
587 * Prevent object from disappearing since we do not have a
588 * reference.
589 */
590 vm_object_pip_add(object, 1);
591 VM_OBJECT_WUNLOCK(backing_object);
592 VM_OBJECT_WLOCK(object);
593 vm_object_pip_wakeup(object);
594 } else
595 VM_OBJECT_WUNLOCK(backing_object);
596
597 /*
598 * Check for a collapse/terminate race with the last reference holder.
599 */
600 if ((object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) != 0 ||
601 !refcount_acquire_if_not_zero(&object->ref_count)) {
602 VM_OBJECT_WUNLOCK(object);
603 return (NULL);
604 }
605 backing_object = object->backing_object;
606 if (backing_object != NULL && (backing_object->flags & OBJ_ANON) != 0)
607 vm_object_collapse(object);
608 VM_OBJECT_WUNLOCK(object);
609
610 return (object);
611}
612
613/*
614 * vm_object_deallocate:
615 *
616 * Release a reference to the specified object,
617 * gained either through a vm_object_allocate
618 * or a vm_object_reference call. When all references
619 * are gone, storage associated with this object
620 * may be relinquished.
621 *
622 * No object may be locked.
623 */
624void
625vm_object_deallocate(vm_object_t object)
626{
627 vm_object_t temp;
628 bool released;
629
630 while (object != NULL) {
631 /*
632 * If the reference count goes to 0 we start calling
633 * vm_object_terminate() on the object chain. A ref count
634 * of 1 may be a special case depending on the shadow count
635 * being 0 or 1. These cases require a write lock on the
636 * object.
637 */
638 if ((object->flags & OBJ_ANON) == 0)
639 released = refcount_release_if_gt(&object->ref_count, 1);
640 else
641 released = refcount_release_if_gt(&object->ref_count, 2);
642 if (released)
643 return;
644
645 if (object->type == OBJT_VNODE) {
646 VM_OBJECT_RLOCK(object);
647 if (object->type == OBJT_VNODE) {
648 vm_object_deallocate_vnode(object);
649 return;
650 }
651 VM_OBJECT_RUNLOCK(object);
652 }
653
654 VM_OBJECT_WLOCK(object);
655 KASSERT(object->ref_count > 0,
656 ("vm_object_deallocate: object deallocated too many times: %d",
657 object->type));
658
659 /*
660 * If this is not the final reference to an anonymous
661 * object we may need to collapse the shadow chain.
662 */
663 if (!refcount_release(&object->ref_count)) {
664 if (object->ref_count > 1 ||
665 atomic_load_int(&object->shadow_count) == 0) {
666 if ((object->flags & OBJ_ANON) != 0 &&
667 object->ref_count == 1)
668 vm_object_set_flag(object,
669 OBJ_ONEMAPPING);
670 VM_OBJECT_WUNLOCK(object);
671 return;
672 }
673
674 /* Handle collapsing last ref on anonymous objects. */
675 object = vm_object_deallocate_anon(object);
676 continue;
677 }
678
679 /*
680 * Handle the final reference to an object. We restart
681 * the loop with the backing object to avoid recursion.
682 */
683 umtx_shm_object_terminated(object);
684 temp = object->backing_object;
685 if (temp != NULL) {
686 KASSERT(object->type == OBJT_DEFAULT ||
687 object->type == OBJT_SWAP,
688 ("shadowed tmpfs v_object 2 %p", object));
689 vm_object_backing_remove(object);
690 }
691
692 KASSERT((object->flags & OBJ_DEAD) == 0,
693 ("vm_object_deallocate: Terminating dead object."));
694 vm_object_set_flag(object, OBJ_DEAD);
695 vm_object_terminate(object);
696 object = temp;
697 }
698}
699
700/*
701 * vm_object_destroy removes the object from the global object list
702 * and frees the space for the object.
703 */
704void
705vm_object_destroy(vm_object_t object)
706{
707
708 /*
709 * Release the allocation charge.
710 */
711 if (object->cred != NULL) {
712 swap_release_by_cred(object->charge, object->cred);
713 object->charge = 0;
714 crfree(object->cred);
715 object->cred = NULL;
716 }
717
718 /*
719 * Free the space for the object.
720 */
721 uma_zfree(obj_zone, object);
722}
723
724static void
725vm_object_sub_shadow(vm_object_t object)
726{
727 KASSERT(object->shadow_count >= 1,
728 ("object %p sub_shadow count zero", object));
729 atomic_subtract_int(&object->shadow_count, 1);
730}
731
732static void
733vm_object_backing_remove_locked(vm_object_t object)
734{
735 vm_object_t backing_object;
736
737 backing_object = object->backing_object;
738 VM_OBJECT_ASSERT_WLOCKED(object);
739 VM_OBJECT_ASSERT_WLOCKED(backing_object);
740
741 KASSERT((object->flags & OBJ_COLLAPSING) == 0,
742 ("vm_object_backing_remove: Removing collapsing object."));
743
744 vm_object_sub_shadow(backing_object);
745 if ((object->flags & OBJ_SHADOWLIST) != 0) {
746 LIST_REMOVE(object, shadow_list);
747 object->flags &= ~OBJ_SHADOWLIST;
748 }
749 object->backing_object = NULL;
750}
751
752static void
753vm_object_backing_remove(vm_object_t object)
754{
755 vm_object_t backing_object;
756
757 VM_OBJECT_ASSERT_WLOCKED(object);
758
759 backing_object = object->backing_object;
760 if ((object->flags & OBJ_SHADOWLIST) != 0) {
761 VM_OBJECT_WLOCK(backing_object);
762 vm_object_backing_remove_locked(object);
763 VM_OBJECT_WUNLOCK(backing_object);
764 } else {
765 object->backing_object = NULL;
766 vm_object_sub_shadow(backing_object);
767 }
768}
769
770static void
771vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
772{
773
774 VM_OBJECT_ASSERT_WLOCKED(object);
775
776 atomic_add_int(&backing_object->shadow_count, 1);
777 if ((backing_object->flags & OBJ_ANON) != 0) {
778 VM_OBJECT_ASSERT_WLOCKED(backing_object);
779 LIST_INSERT_HEAD(&backing_object->shadow_head, object,
780 shadow_list);
781 object->flags |= OBJ_SHADOWLIST;
782 }
783 object->backing_object = backing_object;
784}
785
786static void
787vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
788{
789
790 VM_OBJECT_ASSERT_WLOCKED(object);
791
792 if ((backing_object->flags & OBJ_ANON) != 0) {
793 VM_OBJECT_WLOCK(backing_object);
794 vm_object_backing_insert_locked(object, backing_object);
795 VM_OBJECT_WUNLOCK(backing_object);
796 } else {
797 object->backing_object = backing_object;
798 atomic_add_int(&backing_object->shadow_count, 1);
799 }
800}
801
802/*
803 * Insert an object into a backing_object's shadow list with an additional
804 * reference to the backing_object added.
805 */
806static void
807vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
808{
809
810 VM_OBJECT_ASSERT_WLOCKED(object);
811
812 if ((backing_object->flags & OBJ_ANON) != 0) {
813 VM_OBJECT_WLOCK(backing_object);
814 KASSERT((backing_object->flags & OBJ_DEAD) == 0,
815 ("shadowing dead anonymous object"));
816 vm_object_reference_locked(backing_object);
817 vm_object_backing_insert_locked(object, backing_object);
818 vm_object_clear_flag(backing_object, OBJ_ONEMAPPING);
819 VM_OBJECT_WUNLOCK(backing_object);
820 } else {
821 vm_object_reference(backing_object);
822 atomic_add_int(&backing_object->shadow_count, 1);
823 object->backing_object = backing_object;
824 }
825}
826
827/*
828 * Transfer a backing reference from backing_object to object.
829 */
830static void
831vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
832{
833 vm_object_t new_backing_object;
834
835 /*
836 * Note that the reference to backing_object->backing_object
837 * moves from within backing_object to within object.
838 */
839 vm_object_backing_remove_locked(object);
840 new_backing_object = backing_object->backing_object;
841 if (new_backing_object == NULL)
842 return;
843 if ((new_backing_object->flags & OBJ_ANON) != 0) {
844 VM_OBJECT_WLOCK(new_backing_object);
845 vm_object_backing_remove_locked(backing_object);
846 vm_object_backing_insert_locked(object, new_backing_object);
847 VM_OBJECT_WUNLOCK(new_backing_object);
848 } else {
849 /*
850 * shadow_count for new_backing_object is left
851 * unchanged, its reference provided by backing_object
852 * is replaced by object.
853 */
854 object->backing_object = new_backing_object;
855 backing_object->backing_object = NULL;
856 }
857}
858
859/*
860 * Wait for a concurrent collapse to settle.
861 */
862static void
863vm_object_collapse_wait(vm_object_t object)
864{
865
866 VM_OBJECT_ASSERT_WLOCKED(object);
867
868 while ((object->flags & OBJ_COLLAPSING) != 0) {
869 vm_object_pip_wait(object, "vmcolwait");
870 counter_u64_add(object_collapse_waits, 1);
871 }
872}
873
874/*
875 * Waits for a backing object to clear a pending collapse and returns
876 * it locked if it is an ANON object.
877 */
878static vm_object_t
879vm_object_backing_collapse_wait(vm_object_t object)
880{
881 vm_object_t backing_object;
882
883 VM_OBJECT_ASSERT_WLOCKED(object);
884
885 for (;;) {
886 backing_object = object->backing_object;
887 if (backing_object == NULL ||
888 (backing_object->flags & OBJ_ANON) == 0)
889 return (NULL);
890 VM_OBJECT_WLOCK(backing_object);
891 if ((backing_object->flags & (OBJ_DEAD | OBJ_COLLAPSING)) == 0)
892 break;
893 VM_OBJECT_WUNLOCK(object);
894 vm_object_pip_sleep(backing_object, "vmbckwait");
895 counter_u64_add(object_collapse_waits, 1);
896 VM_OBJECT_WLOCK(object);
897 }
898 return (backing_object);
899}
900
901/*
902 * vm_object_terminate_pages removes any remaining pageable pages
903 * from the object and resets the object to an empty state.
904 */
905static void
906vm_object_terminate_pages(vm_object_t object)
907{
908 vm_page_t p, p_next;
909
910 VM_OBJECT_ASSERT_WLOCKED(object);
911
912 /*
913 * Free any remaining pageable pages. This also removes them from the
914 * paging queues. However, don't free wired pages, just remove them
915 * from the object. Rather than incrementally removing each page from
916 * the object, the page and object are reset to any empty state.
917 */
918 TAILQ_FOREACH_SAFE(p, &object->memq, listq, p_next) {
919 vm_page_assert_unbusied(p);
920 KASSERT(p->object == object &&
921 (p->ref_count & VPRC_OBJREF) != 0,
922 ("vm_object_terminate_pages: page %p is inconsistent", p));
923
924 p->object = NULL;
925 if (vm_page_drop(p, VPRC_OBJREF) == VPRC_OBJREF) {
926 VM_CNT_INC(v_pfree);
927 vm_page_free(p);
928 }
929 }
930
931 /*
932 * If the object contained any pages, then reset it to an empty state.
933 * None of the object's fields, including "resident_page_count", were
934 * modified by the preceding loop.
935 */
936 if (object->resident_page_count != 0) {
937 vm_radix_reclaim_allnodes(&object->rtree);
938 TAILQ_INIT(&object->memq);
939 object->resident_page_count = 0;
940 if (object->type == OBJT_VNODE)
941 vdrop(object->handle);
942 }
943}
944
945/*
946 * vm_object_terminate actually destroys the specified object, freeing
947 * up all previously used resources.
948 *
949 * The object must be locked.
950 * This routine may block.
951 */
952void
953vm_object_terminate(vm_object_t object)
954{
955
956 VM_OBJECT_ASSERT_WLOCKED(object);
957 KASSERT((object->flags & OBJ_DEAD) != 0,
958 ("terminating non-dead obj %p", object));
959 KASSERT((object->flags & OBJ_COLLAPSING) == 0,
960 ("terminating collapsing obj %p", object));
961 KASSERT(object->backing_object == NULL,
962 ("terminating shadow obj %p", object));
963
964 /*
965 * Wait for the pageout daemon and other current users to be
966 * done with the object. Note that new paging_in_progress
967 * users can come after this wait, but they must check
968 * OBJ_DEAD flag set (without unlocking the object), and avoid
969 * the object being terminated.
970 */
971 vm_object_pip_wait(object, "objtrm");
972
973 KASSERT(object->ref_count == 0,
974 ("vm_object_terminate: object with references, ref_count=%d",
975 object->ref_count));
976
977 if ((object->flags & OBJ_PG_DTOR) == 0)
978 vm_object_terminate_pages(object);
979
980#if VM_NRESERVLEVEL > 0
981 if (__predict_false(!LIST_EMPTY(&object->rvq)))
982 vm_reserv_break_all(object);
983#endif
984
985 KASSERT(object->cred == NULL || object->type == OBJT_DEFAULT ||
986 (object->flags & OBJ_SWAP) != 0,
987 ("%s: non-swap obj %p has cred", __func__, object));
988
989 /*
990 * Let the pager know object is dead.
991 */
992 vm_pager_deallocate(object);
993 VM_OBJECT_WUNLOCK(object);
994
995 vm_object_destroy(object);
996}
997
998/*
999 * Make the page read-only so that we can clear the object flags. However, if
1000 * this is a nosync mmap then the object is likely to stay dirty so do not
1001 * mess with the page and do not clear the object flags. Returns TRUE if the
1002 * page should be flushed, and FALSE otherwise.
1003 */
1004static boolean_t
1005vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
1006{
1007
1008 vm_page_assert_busied(p);
1009
1010 /*
1011 * If we have been asked to skip nosync pages and this is a
1012 * nosync page, skip it. Note that the object flags were not
1013 * cleared in this case so we do not have to set them.
1014 */
1015 if ((flags & OBJPC_NOSYNC) != 0 && (p->a.flags & PGA_NOSYNC) != 0) {
1016 *allclean = FALSE;
1017 return (FALSE);
1018 } else {
1019 pmap_remove_write(p);
1020 return (p->dirty != 0);
1021 }
1022}
1023
1024/*
1025 * vm_object_page_clean
1026 *
1027 * Clean all dirty pages in the specified range of object. Leaves page
1028 * on whatever queue it is currently on. If NOSYNC is set then do not
1029 * write out pages with PGA_NOSYNC set (originally comes from MAP_NOSYNC),
1030 * leaving the object dirty.
1031 *
1032 * For swap objects backing tmpfs regular files, do not flush anything,
1033 * but remove write protection on the mapped pages to update mtime through
1034 * mmaped writes.
1035 *
1036 * When stuffing pages asynchronously, allow clustering. XXX we need a
1037 * synchronous clustering mode implementation.
1038 *
1039 * Odd semantics: if start == end, we clean everything.
1040 *
1041 * The object must be locked.
1042 *
1043 * Returns FALSE if some page from the range was not written, as
1044 * reported by the pager, and TRUE otherwise.
1045 */
1046boolean_t
1047vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end,
1048 int flags)
1049{
1050 vm_page_t np, p;
1051 vm_pindex_t pi, tend, tstart;
1052 int curgeneration, n, pagerflags;
1053 boolean_t eio, res, allclean;
1054
1055 VM_OBJECT_ASSERT_WLOCKED(object);
1056
1057 if (!vm_object_mightbedirty(object) || object->resident_page_count == 0)
1058 return (TRUE);
1059
1060 pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
1061 VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
1062 pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
1063
1064 tstart = OFF_TO_IDX(start);
1065 tend = (end == 0) ? object->size : OFF_TO_IDX(end + PAGE_MASK);
1066 allclean = tstart == 0 && tend >= object->size;
1067 res = TRUE;
1068
1069rescan:
1070 curgeneration = object->generation;
1071
1072 for (p = vm_page_find_least(object, tstart); p != NULL; p = np) {
1073 pi = p->pindex;
1074 if (pi >= tend)
1075 break;
1076 np = TAILQ_NEXT(p, listq);
1077 if (vm_page_none_valid(p))
1078 continue;
1079 if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) {
1080 if (object->generation != curgeneration &&
1081 (flags & OBJPC_SYNC) != 0)
1082 goto rescan;
1083 np = vm_page_find_least(object, pi);
1084 continue;
1085 }
1086 if (!vm_object_page_remove_write(p, flags, &allclean)) {
1087 vm_page_xunbusy(p);
1088 continue;
1089 }
1090 if (object->type == OBJT_VNODE) {
1091 n = vm_object_page_collect_flush(object, p, pagerflags,
1092 flags, &allclean, &eio);
1093 if (eio) {
1094 res = FALSE;
1095 allclean = FALSE;
1096 }
1097 if (object->generation != curgeneration &&
1098 (flags & OBJPC_SYNC) != 0)
1099 goto rescan;
1100
1101 /*
1102 * If the VOP_PUTPAGES() did a truncated write, so
1103 * that even the first page of the run is not fully
1104 * written, vm_pageout_flush() returns 0 as the run
1105 * length. Since the condition that caused truncated
1106 * write may be permanent, e.g. exhausted free space,
1107 * accepting n == 0 would cause an infinite loop.
1108 *
1109 * Forwarding the iterator leaves the unwritten page
1110 * behind, but there is not much we can do there if
1111 * filesystem refuses to write it.
1112 */
1113 if (n == 0) {
1114 n = 1;
1115 allclean = FALSE;
1116 }
1117 } else {
1118 n = 1;
1119 vm_page_xunbusy(p);
1120 }
1121 np = vm_page_find_least(object, pi + n);
1122 }
1123#if 0
1124 VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
1125#endif
1126
1127 /*
1128 * Leave updating cleangeneration for tmpfs objects to tmpfs
1129 * scan. It needs to update mtime, which happens for other
1130 * filesystems during page writeouts.
1131 */
1132 if (allclean && object->type == OBJT_VNODE)
1133 object->cleangeneration = curgeneration;
1134 return (res);
1135}
1136
1137static int
1138vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags,
1139 int flags, boolean_t *allclean, boolean_t *eio)
1140{
1141 vm_page_t ma[vm_pageout_page_count], p_first, tp;
1142 int count, i, mreq, runlen;
1143
1144 vm_page_lock_assert(p, MA_NOTOWNED);
1145 vm_page_assert_xbusied(p);
1146 VM_OBJECT_ASSERT_WLOCKED(object);
1147
1148 count = 1;
1149 mreq = 0;
1150
1151 for (tp = p; count < vm_pageout_page_count; count++) {
1152 tp = vm_page_next(tp);
1153 if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1154 break;
1155 if (!vm_object_page_remove_write(tp, flags, allclean)) {
1156 vm_page_xunbusy(tp);
1157 break;
1158 }
1159 }
1160
1161 for (p_first = p; count < vm_pageout_page_count; count++) {
1162 tp = vm_page_prev(p_first);
1163 if (tp == NULL || vm_page_tryxbusy(tp) == 0)
1164 break;
1165 if (!vm_object_page_remove_write(tp, flags, allclean)) {
1166 vm_page_xunbusy(tp);
1167 break;
1168 }
1169 p_first = tp;
1170 mreq++;
1171 }
1172
1173 for (tp = p_first, i = 0; i < count; tp = TAILQ_NEXT(tp, listq), i++)
1174 ma[i] = tp;
1175
1176 vm_pageout_flush(ma, count, pagerflags, mreq, &runlen, eio);
1177 return (runlen);
1178}
1179
1180/*
1181 * Note that there is absolutely no sense in writing out
1182 * anonymous objects, so we track down the vnode object
1183 * to write out.
1184 * We invalidate (remove) all pages from the address space
1185 * for semantic correctness.
1186 *
1187 * If the backing object is a device object with unmanaged pages, then any
1188 * mappings to the specified range of pages must be removed before this
1189 * function is called.
1190 *
1191 * Note: certain anonymous maps, such as MAP_NOSYNC maps,
1192 * may start out with a NULL object.
1193 */
1194boolean_t
1195vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size,
1196 boolean_t syncio, boolean_t invalidate)
1197{
1198 vm_object_t backing_object;
1199 struct vnode *vp;
1200 struct mount *mp;
1201 int error, flags, fsync_after;
1202 boolean_t res;
1203
1204 if (object == NULL)
1205 return (TRUE);
1206 res = TRUE;
1207 error = 0;
1208 VM_OBJECT_WLOCK(object);
1209 while ((backing_object = object->backing_object) != NULL) {
1210 VM_OBJECT_WLOCK(backing_object);
1211 offset += object->backing_object_offset;
1212 VM_OBJECT_WUNLOCK(object);
1213 object = backing_object;
1214 if (object->size < OFF_TO_IDX(offset + size))
1215 size = IDX_TO_OFF(object->size) - offset;
1216 }
1217 /*
1218 * Flush pages if writing is allowed, invalidate them
1219 * if invalidation requested. Pages undergoing I/O
1220 * will be ignored by vm_object_page_remove().
1221 *
1222 * We cannot lock the vnode and then wait for paging
1223 * to complete without deadlocking against vm_fault.
1224 * Instead we simply call vm_object_page_remove() and
1225 * allow it to block internally on a page-by-page
1226 * basis when it encounters pages undergoing async
1227 * I/O.
1228 */
1229 if (object->type == OBJT_VNODE &&
1230 vm_object_mightbedirty(object) != 0 &&
1231 ((vp = object->handle)->v_vflag & VV_NOSYNC) == 0) {
1232 VM_OBJECT_WUNLOCK(object);
1233 (void) vn_start_write(vp, &mp, V_WAIT);
1234 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1235 if (syncio && !invalidate && offset == 0 &&
1236 atop(size) == object->size) {
1237 /*
1238 * If syncing the whole mapping of the file,
1239 * it is faster to schedule all the writes in
1240 * async mode, also allowing the clustering,
1241 * and then wait for i/o to complete.
1242 */
1243 flags = 0;
1244 fsync_after = TRUE;
1245 } else {
1246 flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
1247 flags |= invalidate ? (OBJPC_SYNC | OBJPC_INVAL) : 0;
1248 fsync_after = FALSE;
1249 }
1250 VM_OBJECT_WLOCK(object);
1251 res = vm_object_page_clean(object, offset, offset + size,
1252 flags);
1253 VM_OBJECT_WUNLOCK(object);
1254 if (fsync_after)
1255 error = VOP_FSYNC(vp, MNT_WAIT, curthread);
1256 VOP_UNLOCK(vp);
1257 vn_finished_write(mp);
1258 if (error != 0)
1259 res = FALSE;
1260 VM_OBJECT_WLOCK(object);
1261 }
1262 if ((object->type == OBJT_VNODE ||
1263 object->type == OBJT_DEVICE) && invalidate) {
1264 if (object->type == OBJT_DEVICE)
1265 /*
1266 * The option OBJPR_NOTMAPPED must be passed here
1267 * because vm_object_page_remove() cannot remove
1268 * unmanaged mappings.
1269 */
1270 flags = OBJPR_NOTMAPPED;
1271 else if (old_msync)
1272 flags = 0;
1273 else
1274 flags = OBJPR_CLEANONLY;
1275 vm_object_page_remove(object, OFF_TO_IDX(offset),
1276 OFF_TO_IDX(offset + size + PAGE_MASK), flags);
1277 }
1278 VM_OBJECT_WUNLOCK(object);
1279 return (res);
1280}
1281
1282/*
1283 * Determine whether the given advice can be applied to the object. Advice is
1284 * not applied to unmanaged pages since they never belong to page queues, and
1285 * since MADV_FREE is destructive, it can apply only to anonymous pages that
1286 * have been mapped at most once.
1287 */
1288static bool
1289vm_object_advice_applies(vm_object_t object, int advice)
1290{
1291
1292 if ((object->flags & OBJ_UNMANAGED) != 0)
1293 return (false);
1294 if (advice != MADV_FREE)
1295 return (true);
1296 return ((object->flags & (OBJ_ONEMAPPING | OBJ_ANON)) ==
1297 (OBJ_ONEMAPPING | OBJ_ANON));
1298}
1299
1300static void
1301vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex,
1302 vm_size_t size)
1303{
1304
1305 if (advice == MADV_FREE)
1306 vm_pager_freespace(object, pindex, size);
1307}
1308
1309/*
1310 * vm_object_madvise:
1311 *
1312 * Implements the madvise function at the object/page level.
1313 *
1314 * MADV_WILLNEED (any object)
1315 *
1316 * Activate the specified pages if they are resident.
1317 *
1318 * MADV_DONTNEED (any object)
1319 *
1320 * Deactivate the specified pages if they are resident.
1321 *
1322 * MADV_FREE (OBJT_DEFAULT/OBJT_SWAP objects,
1323 * OBJ_ONEMAPPING only)
1324 *
1325 * Deactivate and clean the specified pages if they are
1326 * resident. This permits the process to reuse the pages
1327 * without faulting or the kernel to reclaim the pages
1328 * without I/O.
1329 */
1330void
1331vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end,
1332 int advice)
1333{
1334 vm_pindex_t tpindex;
1335 vm_object_t backing_object, tobject;
1336 vm_page_t m, tm;
1337
1338 if (object == NULL)
1339 return;
1340
1341relookup:
1342 VM_OBJECT_WLOCK(object);
1343 if (!vm_object_advice_applies(object, advice)) {
1344 VM_OBJECT_WUNLOCK(object);
1345 return;
1346 }
1347 for (m = vm_page_find_least(object, pindex); pindex < end; pindex++) {
1348 tobject = object;
1349
1350 /*
1351 * If the next page isn't resident in the top-level object, we
1352 * need to search the shadow chain. When applying MADV_FREE, we
1353 * take care to release any swap space used to store
1354 * non-resident pages.
1355 */
1356 if (m == NULL || pindex < m->pindex) {
1357 /*
1358 * Optimize a common case: if the top-level object has
1359 * no backing object, we can skip over the non-resident
1360 * range in constant time.
1361 */
1362 if (object->backing_object == NULL) {
1363 tpindex = (m != NULL && m->pindex < end) ?
1364 m->pindex : end;
1365 vm_object_madvise_freespace(object, advice,
1366 pindex, tpindex - pindex);
1367 if ((pindex = tpindex) == end)
1368 break;
1369 goto next_page;
1370 }
1371
1372 tpindex = pindex;
1373 do {
1374 vm_object_madvise_freespace(tobject, advice,
1375 tpindex, 1);
1376 /*
1377 * Prepare to search the next object in the
1378 * chain.
1379 */
1380 backing_object = tobject->backing_object;
1381 if (backing_object == NULL)
1382 goto next_pindex;
1383 VM_OBJECT_WLOCK(backing_object);
1384 tpindex +=
1385 OFF_TO_IDX(tobject->backing_object_offset);
1386 if (tobject != object)
1387 VM_OBJECT_WUNLOCK(tobject);
1388 tobject = backing_object;
1389 if (!vm_object_advice_applies(tobject, advice))
1390 goto next_pindex;
1391 } while ((tm = vm_page_lookup(tobject, tpindex)) ==
1392 NULL);
1393 } else {
1394next_page:
1395 tm = m;
1396 m = TAILQ_NEXT(m, listq);
1397 }
1398
1399 /*
1400 * If the page is not in a normal state, skip it. The page
1401 * can not be invalidated while the object lock is held.
1402 */
1403 if (!vm_page_all_valid(tm) || vm_page_wired(tm))
1404 goto next_pindex;
1405 KASSERT((tm->flags & PG_FICTITIOUS) == 0,
1406 ("vm_object_madvise: page %p is fictitious", tm));
1407 KASSERT((tm->oflags & VPO_UNMANAGED) == 0,
1408 ("vm_object_madvise: page %p is not managed", tm));
1409 if (vm_page_tryxbusy(tm) == 0) {
1410 if (object != tobject)
1411 VM_OBJECT_WUNLOCK(object);
1412 if (advice == MADV_WILLNEED) {
1413 /*
1414 * Reference the page before unlocking and
1415 * sleeping so that the page daemon is less
1416 * likely to reclaim it.
1417 */
1418 vm_page_aflag_set(tm, PGA_REFERENCED);
1419 }
1420 if (!vm_page_busy_sleep(tm, "madvpo", 0))
1421 VM_OBJECT_WUNLOCK(tobject);
1422 goto relookup;
1423 }
1424 vm_page_advise(tm, advice);
1425 vm_page_xunbusy(tm);
1426 vm_object_madvise_freespace(tobject, advice, tm->pindex, 1);
1427next_pindex:
1428 if (tobject != object)
1429 VM_OBJECT_WUNLOCK(tobject);
1430 }
1431 VM_OBJECT_WUNLOCK(object);
1432}
1433
1434/*
1435 * vm_object_shadow:
1436 *
1437 * Create a new object which is backed by the
1438 * specified existing object range. The source
1439 * object reference is deallocated.
1440 *
1441 * The new object and offset into that object
1442 * are returned in the source parameters.
1443 */
1444void
1445vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length,
1446 struct ucred *cred, bool shared)
1447{
1448 vm_object_t source;
1449 vm_object_t result;
1450
1451 source = *object;
1452
1453 /*
1454 * Don't create the new object if the old object isn't shared.
1455 *
1456 * If we hold the only reference we can guarantee that it won't
1457 * increase while we have the map locked. Otherwise the race is
1458 * harmless and we will end up with an extra shadow object that
1459 * will be collapsed later.
1460 */
1461 if (source != NULL && source->ref_count == 1 &&
1462 (source->flags & OBJ_ANON) != 0)
1463 return;
1464
1465 /*
1466 * Allocate a new object with the given length.
1467 */
1468 result = vm_object_allocate_anon(atop(length), source, cred, length);
1469
1470 /*
1471 * Store the offset into the source object, and fix up the offset into
1472 * the new object.
1473 */
1474 result->backing_object_offset = *offset;
1475
1476 if (shared || source != NULL) {
1477 VM_OBJECT_WLOCK(result);
1478
1479 /*
1480 * The new object shadows the source object, adding a
1481 * reference to it. Our caller changes his reference
1482 * to point to the new object, removing a reference to
1483 * the source object. Net result: no change of
1484 * reference count, unless the caller needs to add one
1485 * more reference due to forking a shared map entry.
1486 */
1487 if (shared) {
1488 vm_object_reference_locked(result);
1489 vm_object_clear_flag(result, OBJ_ONEMAPPING);
1490 }
1491
1492 /*
1493 * Try to optimize the result object's page color when
1494 * shadowing in order to maintain page coloring
1495 * consistency in the combined shadowed object.
1496 */
1497 if (source != NULL) {
1498 vm_object_backing_insert(result, source);
1499 result->domain = source->domain;
1500#if VM_NRESERVLEVEL > 0
1501 result->flags |= source->flags & OBJ_COLORED;
1502 result->pg_color = (source->pg_color +
1503 OFF_TO_IDX(*offset)) & ((1 << (VM_NFREEORDER -
1504 1)) - 1);
1505#endif
1506 }
1507 VM_OBJECT_WUNLOCK(result);
1508 }
1509
1510 /*
1511 * Return the new things
1512 */
1513 *offset = 0;
1514 *object = result;
1515}
1516
1517/*
1518 * vm_object_split:
1519 *
1520 * Split the pages in a map entry into a new object. This affords
1521 * easier removal of unused pages, and keeps object inheritance from
1522 * being a negative impact on memory usage.
1523 */
1524void
1525vm_object_split(vm_map_entry_t entry)
1526{
1527 vm_page_t m, m_busy, m_next;
1528 vm_object_t orig_object, new_object, backing_object;
1529 vm_pindex_t idx, offidxstart;
1530 vm_size_t size;
1531
1532 orig_object = entry->object.vm_object;
1533 KASSERT((orig_object->flags & OBJ_ONEMAPPING) != 0,
1534 ("vm_object_split: Splitting object with multiple mappings."));
1535 if ((orig_object->flags & OBJ_ANON) == 0)
1536 return;
1537 if (orig_object->ref_count <= 1)
1538 return;
1539 VM_OBJECT_WUNLOCK(orig_object);
1540
1541 offidxstart = OFF_TO_IDX(entry->offset);
1542 size = atop(entry->end - entry->start);
1543
1544 /*
1545 * If swap_pager_copy() is later called, it will convert new_object
1546 * into a swap object.
1547 */
1548 new_object = vm_object_allocate_anon(size, orig_object,
1549 orig_object->cred, ptoa(size));
1550
1551 /*
1552 * We must wait for the orig_object to complete any in-progress
1553 * collapse so that the swap blocks are stable below. The
1554 * additional reference on backing_object by new object will
1555 * prevent further collapse operations until split completes.
1556 */
1557 VM_OBJECT_WLOCK(orig_object);
1558 vm_object_collapse_wait(orig_object);
1559
1560 /*
1561 * At this point, the new object is still private, so the order in
1562 * which the original and new objects are locked does not matter.
1563 */
1564 VM_OBJECT_WLOCK(new_object);
1565 new_object->domain = orig_object->domain;
1566 backing_object = orig_object->backing_object;
1567 if (backing_object != NULL) {
1568 vm_object_backing_insert_ref(new_object, backing_object);
1569 new_object->backing_object_offset =
1570 orig_object->backing_object_offset + entry->offset;
1571 }
1572 if (orig_object->cred != NULL) {
1573 crhold(orig_object->cred);
1574 KASSERT(orig_object->charge >= ptoa(size),
1575 ("orig_object->charge < 0"));
1576 orig_object->charge -= ptoa(size);
1577 }
1578
1579 /*
1580 * Mark the split operation so that swap_pager_getpages() knows
1581 * that the object is in transition.
1582 */
1583 vm_object_set_flag(orig_object, OBJ_SPLIT);
1584 m_busy = NULL;
1585#ifdef INVARIANTS
1586 idx = 0;
1587#endif
1588retry:
1589 m = vm_page_find_least(orig_object, offidxstart);
1590 KASSERT(m == NULL || idx <= m->pindex - offidxstart,
1591 ("%s: object %p was repopulated", __func__, orig_object));
1592 for (; m != NULL && (idx = m->pindex - offidxstart) < size;
1593 m = m_next) {
1594 m_next = TAILQ_NEXT(m, listq);
1595
1596 /*
1597 * We must wait for pending I/O to complete before we can
1598 * rename the page.
1599 *
1600 * We do not have to VM_PROT_NONE the page as mappings should
1601 * not be changed by this operation.
1602 */
1603 if (vm_page_tryxbusy(m) == 0) {
1604 VM_OBJECT_WUNLOCK(new_object);
1605 if (vm_page_busy_sleep(m, "spltwt", 0))
1606 VM_OBJECT_WLOCK(orig_object);
1607 VM_OBJECT_WLOCK(new_object);
1608 goto retry;
1609 }
1610
1611 /*
1612 * The page was left invalid. Likely placed there by
1613 * an incomplete fault. Just remove and ignore.
1614 */
1615 if (vm_page_none_valid(m)) {
1616 if (vm_page_remove(m))
1617 vm_page_free(m);
1618 continue;
1619 }
1620
1621 /* vm_page_rename() will dirty the page. */
1622 if (vm_page_rename(m, new_object, idx)) {
1623 vm_page_xunbusy(m);
1624 VM_OBJECT_WUNLOCK(new_object);
1625 VM_OBJECT_WUNLOCK(orig_object);
1626 vm_radix_wait();
1627 VM_OBJECT_WLOCK(orig_object);
1628 VM_OBJECT_WLOCK(new_object);
1629 goto retry;
1630 }
1631
1632#if VM_NRESERVLEVEL > 0
1633 /*
1634 * If some of the reservation's allocated pages remain with
1635 * the original object, then transferring the reservation to
1636 * the new object is neither particularly beneficial nor
1637 * particularly harmful as compared to leaving the reservation
1638 * with the original object. If, however, all of the
1639 * reservation's allocated pages are transferred to the new
1640 * object, then transferring the reservation is typically
1641 * beneficial. Determining which of these two cases applies
1642 * would be more costly than unconditionally renaming the
1643 * reservation.
1644 */
1645 vm_reserv_rename(m, new_object, orig_object, offidxstart);
1646#endif
1647
1648 /*
1649 * orig_object's type may change while sleeping, so keep track
1650 * of the beginning of the busied range.
1651 */
1652 if (orig_object->type != OBJT_SWAP)
1653 vm_page_xunbusy(m);
1654 else if (m_busy == NULL)
1655 m_busy = m;
1656 }
1657 if ((orig_object->flags & OBJ_SWAP) != 0) {
1658 /*
1659 * swap_pager_copy() can sleep, in which case the orig_object's
1660 * and new_object's locks are released and reacquired.
1661 */
1662 swap_pager_copy(orig_object, new_object, offidxstart, 0);
1663 if (m_busy != NULL)
1664 TAILQ_FOREACH_FROM(m_busy, &new_object->memq, listq)
1665 vm_page_xunbusy(m_busy);
1666 }
1667 vm_object_clear_flag(orig_object, OBJ_SPLIT);
1668 VM_OBJECT_WUNLOCK(orig_object);
1669 VM_OBJECT_WUNLOCK(new_object);
1670 entry->object.vm_object = new_object;
1671 entry->offset = 0LL;
1672 vm_object_deallocate(orig_object);
1673 VM_OBJECT_WLOCK(new_object);
1674}
1675
1676static vm_page_t
1677vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
1678{
1679 vm_object_t backing_object;
1680
1681 VM_OBJECT_ASSERT_WLOCKED(object);
1682 backing_object = object->backing_object;
1683 VM_OBJECT_ASSERT_WLOCKED(backing_object);
1684
1685 KASSERT(p == NULL || p->object == object || p->object == backing_object,
1686 ("invalid ownership %p %p %p", p, object, backing_object));
1687 /* The page is only NULL when rename fails. */
1688 if (p == NULL) {
1689 VM_OBJECT_WUNLOCK(object);
1690 VM_OBJECT_WUNLOCK(backing_object);
1691 vm_radix_wait();
1692 VM_OBJECT_WLOCK(object);
1693 } else if (p->object == object) {
1694 VM_OBJECT_WUNLOCK(backing_object);
1695 if (vm_page_busy_sleep(p, "vmocol", 0))
1696 VM_OBJECT_WLOCK(object);
1697 } else {
1698 VM_OBJECT_WUNLOCK(object);
1699 if (!vm_page_busy_sleep(p, "vmocol", 0))
1700 VM_OBJECT_WUNLOCK(backing_object);
1701 VM_OBJECT_WLOCK(object);
1702 }
1703 VM_OBJECT_WLOCK(backing_object);
1704 return (TAILQ_FIRST(&backing_object->memq));
1705}
1706
1707static bool
1708vm_object_scan_all_shadowed(vm_object_t object)
1709{
1710 vm_object_t backing_object;
1711 vm_page_t p, pp;
1712 vm_pindex_t backing_offset_index, new_pindex, pi, ps;
1713
1714 VM_OBJECT_ASSERT_WLOCKED(object);
1715 VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1716
1717 backing_object = object->backing_object;
1718
1719 if ((backing_object->flags & OBJ_ANON) == 0)
1720 return (false);
1721
1722 pi = backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1723 p = vm_page_find_least(backing_object, pi);
1724 ps = swap_pager_find_least(backing_object, pi);
1725
1726 /*
1727 * Only check pages inside the parent object's range and
1728 * inside the parent object's mapping of the backing object.
1729 */
1730 for (;; pi++) {
1731 if (p != NULL && p->pindex < pi)
1732 p = TAILQ_NEXT(p, listq);
1733 if (ps < pi)
1734 ps = swap_pager_find_least(backing_object, pi);
1735 if (p == NULL && ps >= backing_object->size)
1736 break;
1737 else if (p == NULL)
1738 pi = ps;
1739 else
1740 pi = MIN(p->pindex, ps);
1741
1742 new_pindex = pi - backing_offset_index;
1743 if (new_pindex >= object->size)
1744 break;
1745
1746 if (p != NULL) {
1747 /*
1748 * If the backing object page is busy a
1749 * grandparent or older page may still be
1750 * undergoing CoW. It is not safe to collapse
1751 * the backing object until it is quiesced.
1752 */
1753 if (vm_page_tryxbusy(p) == 0)
1754 return (false);
1755
1756 /*
1757 * We raced with the fault handler that left
1758 * newly allocated invalid page on the object
1759 * queue and retried.
1760 */
1761 if (!vm_page_all_valid(p))
1762 goto unbusy_ret;
1763 }
1764
1765 /*
1766 * See if the parent has the page or if the parent's object
1767 * pager has the page. If the parent has the page but the page
1768 * is not valid, the parent's object pager must have the page.
1769 *
1770 * If this fails, the parent does not completely shadow the
1771 * object and we might as well give up now.
1772 */
1773 pp = vm_page_lookup(object, new_pindex);
1774
1775 /*
1776 * The valid check here is stable due to object lock
1777 * being required to clear valid and initiate paging.
1778 * Busy of p disallows fault handler to validate pp.
1779 */
1780 if ((pp == NULL || vm_page_none_valid(pp)) &&
1781 !vm_pager_has_page(object, new_pindex, NULL, NULL))
1782 goto unbusy_ret;
1783 if (p != NULL)
1784 vm_page_xunbusy(p);
1785 }
1786 return (true);
1787
1788unbusy_ret:
1789 if (p != NULL)
1790 vm_page_xunbusy(p);
1791 return (false);
1792}
1793
1794static void
1795vm_object_collapse_scan(vm_object_t object)
1796{
1797 vm_object_t backing_object;
1798 vm_page_t next, p, pp;
1799 vm_pindex_t backing_offset_index, new_pindex;
1800
1801 VM_OBJECT_ASSERT_WLOCKED(object);
1802 VM_OBJECT_ASSERT_WLOCKED(object->backing_object);
1803
1804 backing_object = object->backing_object;
1805 backing_offset_index = OFF_TO_IDX(object->backing_object_offset);
1806
1807 /*
1808 * Our scan
1809 */
1810 for (p = TAILQ_FIRST(&backing_object->memq); p != NULL; p = next) {
1811 next = TAILQ_NEXT(p, listq);
1812 new_pindex = p->pindex - backing_offset_index;
1813
1814 /*
1815 * Check for busy page
1816 */
1817 if (vm_page_tryxbusy(p) == 0) {
1818 next = vm_object_collapse_scan_wait(object, p);
1819 continue;
1820 }
1821
1822 KASSERT(object->backing_object == backing_object,
1823 ("vm_object_collapse_scan: backing object mismatch %p != %p",
1824 object->backing_object, backing_object));
1825 KASSERT(p->object == backing_object,
1826 ("vm_object_collapse_scan: object mismatch %p != %p",
1827 p->object, backing_object));
1828
1829 if (p->pindex < backing_offset_index ||
1830 new_pindex >= object->size) {
1831 vm_pager_freespace(backing_object, p->pindex, 1);
1832
1833 KASSERT(!pmap_page_is_mapped(p),
1834 ("freeing mapped page %p", p));
1835 if (vm_page_remove(p))
1836 vm_page_free(p);
1837 continue;
1838 }
1839
1840 if (!vm_page_all_valid(p)) {
1841 KASSERT(!pmap_page_is_mapped(p),
1842 ("freeing mapped page %p", p));
1843 if (vm_page_remove(p))
1844 vm_page_free(p);
1845 continue;
1846 }
1847
1848 pp = vm_page_lookup(object, new_pindex);
1849 if (pp != NULL && vm_page_tryxbusy(pp) == 0) {
1850 vm_page_xunbusy(p);
1851 /*
1852 * The page in the parent is busy and possibly not
1853 * (yet) valid. Until its state is finalized by the
1854 * busy bit owner, we can't tell whether it shadows the
1855 * original page.
1856 */
1857 next = vm_object_collapse_scan_wait(object, pp);
1858 continue;
1859 }
1860
1861 if (pp != NULL && vm_page_none_valid(pp)) {
1862 /*
1863 * The page was invalid in the parent. Likely placed
1864 * there by an incomplete fault. Just remove and
1865 * ignore. p can replace it.
1866 */
1867 if (vm_page_remove(pp))
1868 vm_page_free(pp);
1869 pp = NULL;
1870 }
1871
1872 if (pp != NULL || vm_pager_has_page(object, new_pindex, NULL,
1873 NULL)) {
1874 /*
1875 * The page already exists in the parent OR swap exists
1876 * for this location in the parent. Leave the parent's
1877 * page alone. Destroy the original page from the
1878 * backing object.
1879 */
1880 vm_pager_freespace(backing_object, p->pindex, 1);
1881 KASSERT(!pmap_page_is_mapped(p),
1882 ("freeing mapped page %p", p));
1883 if (vm_page_remove(p))
1884 vm_page_free(p);
1885 if (pp != NULL)
1886 vm_page_xunbusy(pp);
1887 continue;
1888 }
1889
1890 /*
1891 * Page does not exist in parent, rename the page from the
1892 * backing object to the main object.
1893 *
1894 * If the page was mapped to a process, it can remain mapped
1895 * through the rename. vm_page_rename() will dirty the page.
1896 */
1897 if (vm_page_rename(p, object, new_pindex)) {
1898 vm_page_xunbusy(p);
1899 next = vm_object_collapse_scan_wait(object, NULL);
1900 continue;
1901 }
1902
1903 /* Use the old pindex to free the right page. */
1904 vm_pager_freespace(backing_object, new_pindex +
1905 backing_offset_index, 1);
1906
1907#if VM_NRESERVLEVEL > 0
1908 /*
1909 * Rename the reservation.
1910 */
1911 vm_reserv_rename(p, object, backing_object,
1912 backing_offset_index);
1913#endif
1914 vm_page_xunbusy(p);
1915 }
1916 return;
1917}
1918
1919/*
1920 * vm_object_collapse:
1921 *
1922 * Collapse an object with the object backing it.
1923 * Pages in the backing object are moved into the
1924 * parent, and the backing object is deallocated.
1925 */
1926void
1927vm_object_collapse(vm_object_t object)
1928{
1929 vm_object_t backing_object, new_backing_object;
1930
1931 VM_OBJECT_ASSERT_WLOCKED(object);
1932
1933 while (TRUE) {
1934 KASSERT((object->flags & (OBJ_DEAD | OBJ_ANON)) == OBJ_ANON,
1935 ("collapsing invalid object"));
1936
1937 /*
1938 * Wait for the backing_object to finish any pending
1939 * collapse so that the caller sees the shortest possible
1940 * shadow chain.
1941 */
1942 backing_object = vm_object_backing_collapse_wait(object);
1943 if (backing_object == NULL)
1944 return;
1945
1946 KASSERT(object->ref_count > 0 &&
1947 object->ref_count > atomic_load_int(&object->shadow_count),
1948 ("collapse with invalid ref %d or shadow %d count.",
1949 object->ref_count, atomic_load_int(&object->shadow_count)));
1950 KASSERT((backing_object->flags &
1951 (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1952 ("vm_object_collapse: Backing object already collapsing."));
1953 KASSERT((object->flags & (OBJ_COLLAPSING | OBJ_DEAD)) == 0,
1954 ("vm_object_collapse: object is already collapsing."));
1955
1956 /*
1957 * We know that we can either collapse the backing object if
1958 * the parent is the only reference to it, or (perhaps) have
1959 * the parent bypass the object if the parent happens to shadow
1960 * all the resident pages in the entire backing object.
1961 */
1962 if (backing_object->ref_count == 1) {
1963 KASSERT(atomic_load_int(&backing_object->shadow_count)
1964 == 1,
1965 ("vm_object_collapse: shadow_count: %d",
1966 atomic_load_int(&backing_object->shadow_count)));
1967 vm_object_pip_add(object, 1);
1968 vm_object_set_flag(object, OBJ_COLLAPSING);
1969 vm_object_pip_add(backing_object, 1);
1970 vm_object_set_flag(backing_object, OBJ_DEAD);
1971
1972 /*
1973 * If there is exactly one reference to the backing
1974 * object, we can collapse it into the parent.
1975 */
1976 vm_object_collapse_scan(object);
1977
1978#if VM_NRESERVLEVEL > 0
1979 /*
1980 * Break any reservations from backing_object.
1981 */
1982 if (__predict_false(!LIST_EMPTY(&backing_object->rvq)))
1983 vm_reserv_break_all(backing_object);
1984#endif
1985
1986 /*
1987 * Move the pager from backing_object to object.
1988 */
1989 if ((backing_object->flags & OBJ_SWAP) != 0) {
1990 /*
1991 * swap_pager_copy() can sleep, in which case
1992 * the backing_object's and object's locks are
1993 * released and reacquired.
1994 * Since swap_pager_copy() is being asked to
1995 * destroy backing_object, it will change the
1996 * type to OBJT_DEFAULT.
1997 */
1998 swap_pager_copy(
1999 backing_object,
2000 object,
2001 OFF_TO_IDX(object->backing_object_offset), TRUE);
2002 }
2003
2004 /*
2005 * Object now shadows whatever backing_object did.
2006 */
2007 vm_object_clear_flag(object, OBJ_COLLAPSING);
2008 vm_object_backing_transfer(object, backing_object);
2009 object->backing_object_offset +=
2010 backing_object->backing_object_offset;
2011 VM_OBJECT_WUNLOCK(object);
2012 vm_object_pip_wakeup(object);
2013
2014 /*
2015 * Discard backing_object.
2016 *
2017 * Since the backing object has no pages, no pager left,
2018 * and no object references within it, all that is
2019 * necessary is to dispose of it.
2020 */
2021 KASSERT(backing_object->ref_count == 1, (
2022"backing_object %p was somehow re-referenced during collapse!",
2023 backing_object));
2024 vm_object_pip_wakeup(backing_object);
2025 (void)refcount_release(&backing_object->ref_count);
2026 vm_object_terminate(backing_object);
2027 counter_u64_add(object_collapses, 1);
2028 VM_OBJECT_WLOCK(object);
2029 } else {
2030 /*
2031 * If we do not entirely shadow the backing object,
2032 * there is nothing we can do so we give up.
2033 *
2034 * The object lock and backing_object lock must not
2035 * be dropped during this sequence.
2036 */
2037 if (!vm_object_scan_all_shadowed(object)) {
2038 VM_OBJECT_WUNLOCK(backing_object);
2039 break;
2040 }
2041
2042 /*
2043 * Make the parent shadow the next object in the
2044 * chain. Deallocating backing_object will not remove
2045 * it, since its reference count is at least 2.
2046 */
2047 vm_object_backing_remove_locked(object);
2048 new_backing_object = backing_object->backing_object;
2049 if (new_backing_object != NULL) {
2050 vm_object_backing_insert_ref(object,
2051 new_backing_object);
2052 object->backing_object_offset +=
2053 backing_object->backing_object_offset;
2054 }
2055
2056 /*
2057 * Drop the reference count on backing_object. Since
2058 * its ref_count was at least 2, it will not vanish.
2059 */
2060 (void)refcount_release(&backing_object->ref_count);
2061 KASSERT(backing_object->ref_count >= 1, (
2062"backing_object %p was somehow dereferenced during collapse!",
2063 backing_object));
2064 VM_OBJECT_WUNLOCK(backing_object);
2065 counter_u64_add(object_bypasses, 1);
2066 }
2067
2068 /*
2069 * Try again with this object's new backing object.
2070 */
2071 }
2072}
2073
2074/*
2075 * vm_object_page_remove:
2076 *
2077 * For the given object, either frees or invalidates each of the
2078 * specified pages. In general, a page is freed. However, if a page is
2079 * wired for any reason other than the existence of a managed, wired
2080 * mapping, then it may be invalidated but not removed from the object.
2081 * Pages are specified by the given range ["start", "end") and the option
2082 * OBJPR_CLEANONLY. As a special case, if "end" is zero, then the range
2083 * extends from "start" to the end of the object. If the option
2084 * OBJPR_CLEANONLY is specified, then only the non-dirty pages within the
2085 * specified range are affected. If the option OBJPR_NOTMAPPED is
2086 * specified, then the pages within the specified range must have no
2087 * mappings. Otherwise, if this option is not specified, any mappings to
2088 * the specified pages are removed before the pages are freed or
2089 * invalidated.
2090 *
2091 * In general, this operation should only be performed on objects that
2092 * contain managed pages. There are, however, two exceptions. First, it
2093 * is performed on the kernel and kmem objects by vm_map_entry_delete().
2094 * Second, it is used by msync(..., MS_INVALIDATE) to invalidate device-
2095 * backed pages. In both of these cases, the option OBJPR_CLEANONLY must
2096 * not be specified and the option OBJPR_NOTMAPPED must be specified.
2097 *
2098 * The object must be locked.
2099 */
2100void
2101vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
2102 int options)
2103{
2104 vm_page_t p, next;
2105
2106 VM_OBJECT_ASSERT_WLOCKED(object);
2107 KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
2108 (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
2109 ("vm_object_page_remove: illegal options for object %p", object));
2110 if (object->resident_page_count == 0)
2111 return;
2112 vm_object_pip_add(object, 1);
2113again:
2114 p = vm_page_find_least(object, start);
2115
2116 /*
2117 * Here, the variable "p" is either (1) the page with the least pindex
2118 * greater than or equal to the parameter "start" or (2) NULL.
2119 */
2120 for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2121 next = TAILQ_NEXT(p, listq);
2122
2123 /*
2124 * Skip invalid pages if asked to do so. Try to avoid acquiring
2125 * the busy lock, as some consumers rely on this to avoid
2126 * deadlocks.
2127 *
2128 * A thread may concurrently transition the page from invalid to
2129 * valid using only the busy lock, so the result of this check
2130 * is immediately stale. It is up to consumers to handle this,
2131 * for instance by ensuring that all invalid->valid transitions
2132 * happen with a mutex held, as may be possible for a
2133 * filesystem.
2134 */
2135 if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p))
2136 continue;
2137
2138 /*
2139 * If the page is wired for any reason besides the existence
2140 * of managed, wired mappings, then it cannot be freed. For
2141 * example, fictitious pages, which represent device memory,
2142 * are inherently wired and cannot be freed. They can,
2143 * however, be invalidated if the option OBJPR_CLEANONLY is
2144 * not specified.
2145 */
2146 if (vm_page_tryxbusy(p) == 0) {
2147 if (vm_page_busy_sleep(p, "vmopar", 0))
2148 VM_OBJECT_WLOCK(object);
2149 goto again;
2150 }
2151 if ((options & OBJPR_VALIDONLY) != 0 && vm_page_none_valid(p)) {
2152 vm_page_xunbusy(p);
2153 continue;
2154 }
2155 if (vm_page_wired(p)) {
2156wired:
2157 if ((options & OBJPR_NOTMAPPED) == 0 &&
2158 object->ref_count != 0)
2159 pmap_remove_all(p);
2160 if ((options & OBJPR_CLEANONLY) == 0) {
2161 vm_page_invalid(p);
2162 vm_page_undirty(p);
2163 }
2164 vm_page_xunbusy(p);
2165 continue;
2166 }
2167 KASSERT((p->flags & PG_FICTITIOUS) == 0,
2168 ("vm_object_page_remove: page %p is fictitious", p));
2169 if ((options & OBJPR_CLEANONLY) != 0 &&
2170 !vm_page_none_valid(p)) {
2171 if ((options & OBJPR_NOTMAPPED) == 0 &&
2172 object->ref_count != 0 &&
2173 !vm_page_try_remove_write(p))
2174 goto wired;
2175 if (p->dirty != 0) {
2176 vm_page_xunbusy(p);
2177 continue;
2178 }
2179 }
2180 if ((options & OBJPR_NOTMAPPED) == 0 &&
2181 object->ref_count != 0 && !vm_page_try_remove_all(p))
2182 goto wired;
2183 vm_page_free(p);
2184 }
2185 vm_object_pip_wakeup(object);
2186
2187 vm_pager_freespace(object, start, (end == 0 ? object->size : end) -
2188 start);
2189}
2190
2191/*
2192 * vm_object_page_noreuse:
2193 *
2194 * For the given object, attempt to move the specified pages to
2195 * the head of the inactive queue. This bypasses regular LRU
2196 * operation and allows the pages to be reused quickly under memory
2197 * pressure. If a page is wired for any reason, then it will not
2198 * be queued. Pages are specified by the range ["start", "end").
2199 * As a special case, if "end" is zero, then the range extends from
2200 * "start" to the end of the object.
2201 *
2202 * This operation should only be performed on objects that
2203 * contain non-fictitious, managed pages.
2204 *
2205 * The object must be locked.
2206 */
2207void
2208vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2209{
2210 vm_page_t p, next;
2211
2212 VM_OBJECT_ASSERT_LOCKED(object);
2213 KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0,
2214 ("vm_object_page_noreuse: illegal object %p", object));
2215 if (object->resident_page_count == 0)
2216 return;
2217 p = vm_page_find_least(object, start);
2218
2219 /*
2220 * Here, the variable "p" is either (1) the page with the least pindex
2221 * greater than or equal to the parameter "start" or (2) NULL.
2222 */
2223 for (; p != NULL && (p->pindex < end || end == 0); p = next) {
2224 next = TAILQ_NEXT(p, listq);
2225 vm_page_deactivate_noreuse(p);
2226 }
2227}
2228
2229/*
2230 * Populate the specified range of the object with valid pages. Returns
2231 * TRUE if the range is successfully populated and FALSE otherwise.
2232 *
2233 * Note: This function should be optimized to pass a larger array of
2234 * pages to vm_pager_get_pages() before it is applied to a non-
2235 * OBJT_DEVICE object.
2236 *
2237 * The object must be locked.
2238 */
2239boolean_t
2240vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
2241{
2242 vm_page_t m;
2243 vm_pindex_t pindex;
2244 int rv;
2245
2246 VM_OBJECT_ASSERT_WLOCKED(object);
2247 for (pindex = start; pindex < end; pindex++) {
2248 rv = vm_page_grab_valid(&m, object, pindex, VM_ALLOC_NORMAL);
2249 if (rv != VM_PAGER_OK)
2250 break;
2251
2252 /*
2253 * Keep "m" busy because a subsequent iteration may unlock
2254 * the object.
2255 */
2256 }
2257 if (pindex > start) {
2258 m = vm_page_lookup(object, start);
2259 while (m != NULL && m->pindex < pindex) {
2260 vm_page_xunbusy(m);
2261 m = TAILQ_NEXT(m, listq);
2262 }
2263 }
2264 return (pindex == end);
2265}
2266
2267/*
2268 * Routine: vm_object_coalesce
2269 * Function: Coalesces two objects backing up adjoining
2270 * regions of memory into a single object.
2271 *
2272 * returns TRUE if objects were combined.
2273 *
2274 * NOTE: Only works at the moment if the second object is NULL -
2275 * if it's not, which object do we lock first?
2276 *
2277 * Parameters:
2278 * prev_object First object to coalesce
2279 * prev_offset Offset into prev_object
2280 * prev_size Size of reference to prev_object
2281 * next_size Size of reference to the second object
2282 * reserved Indicator that extension region has
2283 * swap accounted for
2284 *
2285 * Conditions:
2286 * The object must *not* be locked.
2287 */
2288boolean_t
2289vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset,
2290 vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
2291{
2292 vm_pindex_t next_pindex;
2293
2294 if (prev_object == NULL)
2295 return (TRUE);
2296 if ((prev_object->flags & OBJ_ANON) == 0)
2297 return (FALSE);
2298
2299 VM_OBJECT_WLOCK(prev_object);
2300 /*
2301 * Try to collapse the object first.
2302 */
2303 vm_object_collapse(prev_object);
2304
2305 /*
2306 * Can't coalesce if: . more than one reference . paged out . shadows
2307 * another object . has a copy elsewhere (any of which mean that the
2308 * pages not mapped to prev_entry may be in use anyway)
2309 */
2310 if (prev_object->backing_object != NULL) {
2311 VM_OBJECT_WUNLOCK(prev_object);
2312 return (FALSE);
2313 }
2314
2315 prev_size >>= PAGE_SHIFT;
2316 next_size >>= PAGE_SHIFT;
2317 next_pindex = OFF_TO_IDX(prev_offset) + prev_size;
2318
2319 if (prev_object->ref_count > 1 &&
2320 prev_object->size != next_pindex &&
2321 (prev_object->flags & OBJ_ONEMAPPING) == 0) {
2322 VM_OBJECT_WUNLOCK(prev_object);
2323 return (FALSE);
2324 }
2325
2326 /*
2327 * Account for the charge.
2328 */
2329 if (prev_object->cred != NULL) {
2330 /*
2331 * If prev_object was charged, then this mapping,
2332 * although not charged now, may become writable
2333 * later. Non-NULL cred in the object would prevent
2334 * swap reservation during enabling of the write
2335 * access, so reserve swap now. Failed reservation
2336 * cause allocation of the separate object for the map
2337 * entry, and swap reservation for this entry is
2338 * managed in appropriate time.
2339 */
2340 if (!reserved && !swap_reserve_by_cred(ptoa(next_size),
2341 prev_object->cred)) {
2342 VM_OBJECT_WUNLOCK(prev_object);
2343 return (FALSE);
2344 }
2345 prev_object->charge += ptoa(next_size);
2346 }
2347
2348 /*
2349 * Remove any pages that may still be in the object from a previous
2350 * deallocation.
2351 */
2352 if (next_pindex < prev_object->size) {
2353 vm_object_page_remove(prev_object, next_pindex, next_pindex +
2354 next_size, 0);
2355#if 0
2356 if (prev_object->cred != NULL) {
2357 KASSERT(prev_object->charge >=
2358 ptoa(prev_object->size - next_pindex),
2359 ("object %p overcharged 1 %jx %jx", prev_object,
2360 (uintmax_t)next_pindex, (uintmax_t)next_size));
2361 prev_object->charge -= ptoa(prev_object->size -
2362 next_pindex);
2363 }
2364#endif
2365 }
2366
2367 /*
2368 * Extend the object if necessary.
2369 */
2370 if (next_pindex + next_size > prev_object->size)
2371 prev_object->size = next_pindex + next_size;
2372
2373 VM_OBJECT_WUNLOCK(prev_object);
2374 return (TRUE);
2375}
2376
2377void
2378vm_object_set_writeable_dirty_(vm_object_t object)
2379{
2380 atomic_add_int(&object->generation, 1);
2381}
2382
2383bool
2384vm_object_mightbedirty_(vm_object_t object)
2385{
2386 return (object->generation != object->cleangeneration);
2387}
2388
2389/*
2390 * vm_object_unwire:
2391 *
2392 * For each page offset within the specified range of the given object,
2393 * find the highest-level page in the shadow chain and unwire it. A page
2394 * must exist at every page offset, and the highest-level page must be
2395 * wired.
2396 */
2397void
2398vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length,
2399 uint8_t queue)
2400{
2401 vm_object_t tobject, t1object;
2402 vm_page_t m, tm;
2403 vm_pindex_t end_pindex, pindex, tpindex;
2404 int depth, locked_depth;
2405
2406 KASSERT((offset & PAGE_MASK) == 0,
2407 ("vm_object_unwire: offset is not page aligned"));
2408 KASSERT((length & PAGE_MASK) == 0,
2409 ("vm_object_unwire: length is not a multiple of PAGE_SIZE"));
2410 /* The wired count of a fictitious page never changes. */
2411 if ((object->flags & OBJ_FICTITIOUS) != 0)
2412 return;
2413 pindex = OFF_TO_IDX(offset);
2414 end_pindex = pindex + atop(length);
2415again:
2416 locked_depth = 1;
2417 VM_OBJECT_RLOCK(object);
2418 m = vm_page_find_least(object, pindex);
2419 while (pindex < end_pindex) {
2420 if (m == NULL || pindex < m->pindex) {
2421 /*
2422 * The first object in the shadow chain doesn't
2423 * contain a page at the current index. Therefore,
2424 * the page must exist in a backing object.
2425 */
2426 tobject = object;
2427 tpindex = pindex;
2428 depth = 0;
2429 do {
2430 tpindex +=
2431 OFF_TO_IDX(tobject->backing_object_offset);
2432 tobject = tobject->backing_object;
2433 KASSERT(tobject != NULL,
2434 ("vm_object_unwire: missing page"));
2435 if ((tobject->flags & OBJ_FICTITIOUS) != 0)
2436 goto next_page;
2437 depth++;
2438 if (depth == locked_depth) {
2439 locked_depth++;
2440 VM_OBJECT_RLOCK(tobject);
2441 }
2442 } while ((tm = vm_page_lookup(tobject, tpindex)) ==
2443 NULL);
2444 } else {
2445 tm = m;
2446 m = TAILQ_NEXT(m, listq);
2447 }
2448 if (vm_page_trysbusy(tm) == 0) {
2449 for (tobject = object; locked_depth >= 1;
2450 locked_depth--) {
2451 t1object = tobject->backing_object;
2452 if (tm->object != tobject)
2453 VM_OBJECT_RUNLOCK(tobject);
2454 tobject = t1object;
2455 }
2456 tobject = tm->object;
2457 if (!vm_page_busy_sleep(tm, "unwbo",
2458 VM_ALLOC_IGN_SBUSY))
2459 VM_OBJECT_RUNLOCK(tobject);
2460 goto again;
2461 }
2462 vm_page_unwire(tm, queue);
2463 vm_page_sunbusy(tm);
2464next_page:
2465 pindex++;
2466 }
2467 /* Release the accumulated object locks. */
2468 for (tobject = object; locked_depth >= 1; locked_depth--) {
2469 t1object = tobject->backing_object;
2470 VM_OBJECT_RUNLOCK(tobject);
2471 tobject = t1object;
2472 }
2473}
2474
2475/*
2476 * Return the vnode for the given object, or NULL if none exists.
2477 * For tmpfs objects, the function may return NULL if there is
2478 * no vnode allocated at the time of the call.
2479 */
2480struct vnode *
2481vm_object_vnode(vm_object_t object)
2482{
2483 struct vnode *vp;
2484
2485 VM_OBJECT_ASSERT_LOCKED(object);
2486 vm_pager_getvp(object, &vp, NULL);
2487 return (vp);
2488}
2489
2490/*
2491 * Busy the vm object. This prevents new pages belonging to the object from
2492 * becoming busy. Existing pages persist as busy. Callers are responsible
2493 * for checking page state before proceeding.
2494 */
2495void
2496vm_object_busy(vm_object_t obj)
2497{
2498
2499 VM_OBJECT_ASSERT_LOCKED(obj);
2500
2501 blockcount_acquire(&obj->busy, 1);
2502 /* The fence is required to order loads of page busy. */
2503 atomic_thread_fence_acq_rel();
2504}
2505
2506void
2507vm_object_unbusy(vm_object_t obj)
2508{
2509
2510 blockcount_release(&obj->busy, 1);
2511}
2512
2513void
2514vm_object_busy_wait(vm_object_t obj, const char *wmesg)
2515{
2516
2517 VM_OBJECT_ASSERT_UNLOCKED(obj);
2518
2519 (void)blockcount_sleep(&obj->busy, NULL, wmesg, PVM);
2520}
2521
2522/*
2523 * This function aims to determine if the object is mapped,
2524 * specifically, if it is referenced by a vm_map_entry. Because
2525 * objects occasionally acquire transient references that do not
2526 * represent a mapping, the method used here is inexact. However, it
2527 * has very low overhead and is good enough for the advisory
2528 * vm.vmtotal sysctl.
2529 */
2530bool
2531vm_object_is_active(vm_object_t obj)
2532{
2533
2534 return (obj->ref_count > atomic_load_int(&obj->shadow_count));
2535}
2536
2537static int
2538vm_object_list_handler(struct sysctl_req *req, bool swap_only)
2539{
2540 struct kinfo_vmobject *kvo;
2541 char *fullpath, *freepath;
2542 struct vnode *vp;
2543 struct vattr va;
2544 vm_object_t obj;
2545 vm_page_t m;
2546 u_long sp;
2547 int count, error;
2548
2549 if (req->oldptr == NULL) {
2550 /*
2551 * If an old buffer has not been provided, generate an
2552 * estimate of the space needed for a subsequent call.
2553 */
2554 mtx_lock(&vm_object_list_mtx);
2555 count = 0;
2556 TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2557 if (obj->type == OBJT_DEAD)
2558 continue;
2559 count++;
2560 }
2561 mtx_unlock(&vm_object_list_mtx);
2562 return (SYSCTL_OUT(req, NULL, sizeof(struct kinfo_vmobject) *
2563 count * 11 / 10));
2564 }
2565
2566 kvo = malloc(sizeof(*kvo), M_TEMP, M_WAITOK);
2567 error = 0;
2568
2569 /*
2570 * VM objects are type stable and are never removed from the
2571 * list once added. This allows us to safely read obj->object_list
2572 * after reacquiring the VM object lock.
2573 */
2574 mtx_lock(&vm_object_list_mtx);
2575 TAILQ_FOREACH(obj, &vm_object_list, object_list) {
2576 if (obj->type == OBJT_DEAD ||
2577 (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0))
2578 continue;
2579 VM_OBJECT_RLOCK(obj);
2580 if (obj->type == OBJT_DEAD ||
2581 (swap_only && (obj->flags & (OBJ_ANON | OBJ_SWAP)) == 0)) {
2582 VM_OBJECT_RUNLOCK(obj);
2583 continue;
2584 }
2585 mtx_unlock(&vm_object_list_mtx);
2586 kvo->kvo_size = ptoa(obj->size);
2587 kvo->kvo_resident = obj->resident_page_count;
2588 kvo->kvo_ref_count = obj->ref_count;
2589 kvo->kvo_shadow_count = atomic_load_int(&obj->shadow_count);
2590 kvo->kvo_memattr = obj->memattr;
2591 kvo->kvo_active = 0;
2592 kvo->kvo_inactive = 0;
2593 if (!swap_only) {
2594 TAILQ_FOREACH(m, &obj->memq, listq) {
2595 /*
2596 * A page may belong to the object but be
2597 * dequeued and set to PQ_NONE while the
2598 * object lock is not held. This makes the
2599 * reads of m->queue below racy, and we do not
2600 * count pages set to PQ_NONE. However, this
2601 * sysctl is only meant to give an
2602 * approximation of the system anyway.
2603 */
2604 if (m->a.queue == PQ_ACTIVE)
2605 kvo->kvo_active++;
2606 else if (m->a.queue == PQ_INACTIVE)
2607 kvo->kvo_inactive++;
2608 }
2609 }
2610
2611 kvo->kvo_vn_fileid = 0;
2612 kvo->kvo_vn_fsid = 0;
2613 kvo->kvo_vn_fsid_freebsd11 = 0;
2614 freepath = NULL;
2615 fullpath = "";
2616 vp = NULL;
2617 kvo->kvo_type = vm_object_kvme_type(obj, swap_only ? NULL : &vp);
2618 if (vp != NULL) {
2619 vref(vp);
2620 } else if ((obj->flags & OBJ_ANON) != 0) {
2621 MPASS(kvo->kvo_type == KVME_TYPE_DEFAULT ||
2622 kvo->kvo_type == KVME_TYPE_SWAP);
2623 kvo->kvo_me = (uintptr_t)obj;
2624 /* tmpfs objs are reported as vnodes */
2625 kvo->kvo_backing_obj = (uintptr_t)obj->backing_object;
2626 sp = swap_pager_swapped_pages(obj);
2627 kvo->kvo_swapped = sp > UINT32_MAX ? UINT32_MAX : sp;
2628 }
2629 VM_OBJECT_RUNLOCK(obj);
2630 if (vp != NULL) {
2631 vn_fullpath(vp, &fullpath, &freepath);
2632 vn_lock(vp, LK_SHARED | LK_RETRY);
2633 if (VOP_GETATTR(vp, &va, curthread->td_ucred) == 0) {
2634 kvo->kvo_vn_fileid = va.va_fileid;
2635 kvo->kvo_vn_fsid = va.va_fsid;
2636 kvo->kvo_vn_fsid_freebsd11 = va.va_fsid;
2637 /* truncate */
2638 }
2639 vput(vp);
2640 }
2641
2642 strlcpy(kvo->kvo_path, fullpath, sizeof(kvo->kvo_path));
2643 if (freepath != NULL)
2644 free(freepath, M_TEMP);
2645
2646 /* Pack record size down */
2647 kvo->kvo_structsize = offsetof(struct kinfo_vmobject, kvo_path)
2648 + strlen(kvo->kvo_path) + 1;
2649 kvo->kvo_structsize = roundup(kvo->kvo_structsize,
2650 sizeof(uint64_t));
2651 error = SYSCTL_OUT(req, kvo, kvo->kvo_structsize);
2652 maybe_yield();
2653 mtx_lock(&vm_object_list_mtx);
2654 if (error)
2655 break;
2656 }
2657 mtx_unlock(&vm_object_list_mtx);
2658 free(kvo, M_TEMP);
2659 return (error);
2660}
2661
2662static int
2663sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
2664{
2665 return (vm_object_list_handler(req, false));
2666}
2667
2668SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP |
2669 CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject",
2670 "List of VM objects");
2671
2672static int
2673sysctl_vm_object_list_swap(SYSCTL_HANDLER_ARGS)
2674{
2675 return (vm_object_list_handler(req, true));
2676}
2677
2678/*
2679 * This sysctl returns list of the anonymous or swap objects. Intent
2680 * is to provide stripped optimized list useful to analyze swap use.
2681 * Since technically non-swap (default) objects participate in the
2682 * shadow chains, and are converted to swap type as needed by swap
2683 * pager, we must report them.
2684 */
2685SYSCTL_PROC(_vm, OID_AUTO, swap_objects,
2686 CTLTYPE_STRUCT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, NULL, 0,
2687 sysctl_vm_object_list_swap, "S,kinfo_vmobject",
2688 "List of swap VM objects");
2689
2690#include "opt_ddb.h"
2691#ifdef DDB
2692#include <sys/kernel.h>
2693
2694#include <sys/cons.h>
2695
2696#include <ddb/ddb.h>
2697
2698static int
2699_vm_object_in_map(vm_map_t map, vm_object_t object, vm_map_entry_t entry)
2700{
2701 vm_map_t tmpm;
2702 vm_map_entry_t tmpe;
2703 vm_object_t obj;
2704
2705 if (map == 0)
2706 return 0;
2707
2708 if (entry == 0) {
2709 VM_MAP_ENTRY_FOREACH(tmpe, map) {
2710 if (_vm_object_in_map(map, object, tmpe)) {
2711 return 1;
2712 }
2713 }
2714 } else if (entry->eflags & MAP_ENTRY_IS_SUB_MAP) {
2715 tmpm = entry->object.sub_map;
2716 VM_MAP_ENTRY_FOREACH(tmpe, tmpm) {
2717 if (_vm_object_in_map(tmpm, object, tmpe)) {
2718 return 1;
2719 }
2720 }
2721 } else if ((obj = entry->object.vm_object) != NULL) {
2722 for (; obj; obj = obj->backing_object)
2723 if (obj == object) {
2724 return 1;
2725 }
2726 }
2727 return 0;
2728}
2729
2730static int
2731vm_object_in_map(vm_object_t object)
2732{
2733 struct proc *p;
2734
2735 /* sx_slock(&allproc_lock); */
2736 FOREACH_PROC_IN_SYSTEM(p) {
2737 if (!p->p_vmspace /* || (p->p_flag & (P_SYSTEM|P_WEXIT)) */)
2738 continue;
2739 if (_vm_object_in_map(&p->p_vmspace->vm_map, object, 0)) {
2740 /* sx_sunlock(&allproc_lock); */
2741 return 1;
2742 }
2743 }
2744 /* sx_sunlock(&allproc_lock); */
2745 if (_vm_object_in_map(kernel_map, object, 0))
2746 return 1;
2747 return 0;
2748}
2749
2750DB_SHOW_COMMAND(vmochk, vm_object_check)
2751{
2752 vm_object_t object;
2753
2754 /*
2755 * make sure that internal objs are in a map somewhere
2756 * and none have zero ref counts.
2757 */
2758 TAILQ_FOREACH(object, &vm_object_list, object_list) {
2759 if ((object->flags & OBJ_ANON) != 0) {
2760 if (object->ref_count == 0) {
2761 db_printf("vmochk: internal obj has zero ref count: %ld\n",
2762 (long)object->size);
2763 }
2764 if (!vm_object_in_map(object)) {
2765 db_printf(
2766 "vmochk: internal obj is not in a map: "
2767 "ref: %d, size: %lu: 0x%lx, backing_object: %p\n",
2768 object->ref_count, (u_long)object->size,
2769 (u_long)object->size,
2770 (void *)object->backing_object);
2771 }
2772 }
2773 if (db_pager_quit)
2774 return;
2775 }
2776}
2777
2778/*
2779 * vm_object_print: [ debug ]
2780 */
2781DB_SHOW_COMMAND(object, vm_object_print_static)
2782{
2783 /* XXX convert args. */
2784 vm_object_t object = (vm_object_t)addr;
2785 boolean_t full = have_addr;
2786
2787 vm_page_t p;
2788
2789 /* XXX count is an (unused) arg. Avoid shadowing it. */
2790#define count was_count
2791
2792 int count;
2793
2794 if (object == NULL)
2795 return;
2796
2797 db_iprintf(
2798 "Object %p: type=%d, size=0x%jx, res=%d, ref=%d, flags=0x%x ruid %d charge %jx\n",
2799 object, (int)object->type, (uintmax_t)object->size,
2800 object->resident_page_count, object->ref_count, object->flags,
2801 object->cred ? object->cred->cr_ruid : -1, (uintmax_t)object->charge);
2802 db_iprintf(" sref=%d, backing_object(%d)=(%p)+0x%jx\n",
2803 atomic_load_int(&object->shadow_count),
2804 object->backing_object ? object->backing_object->ref_count : 0,
2805 object->backing_object, (uintmax_t)object->backing_object_offset);
2806
2807 if (!full)
2808 return;
2809
2810 db_indent += 2;
2811 count = 0;
2812 TAILQ_FOREACH(p, &object->memq, listq) {
2813 if (count == 0)
2814 db_iprintf("memory:=");
2815 else if (count == 6) {
2816 db_printf("\n");
2817 db_iprintf(" ...");
2818 count = 0;
2819 } else
2820 db_printf(",");
2821 count++;
2822
2823 db_printf("(off=0x%jx,page=0x%jx)",
2824 (uintmax_t)p->pindex, (uintmax_t)VM_PAGE_TO_PHYS(p));
2825
2826 if (db_pager_quit)
2827 break;
2828 }
2829 if (count != 0)
2830 db_printf("\n");
2831 db_indent -= 2;
2832}
2833
2834/* XXX. */
2835#undef count
2836
2837/* XXX need this non-static entry for calling from vm_map_print. */
2838void
2839vm_object_print(
2840 /* db_expr_t */ long addr,
2841 boolean_t have_addr,
2842 /* db_expr_t */ long count,
2843 char *modif)
2844{
2845 vm_object_print_static(addr, have_addr, count, modif);
2846}
2847
2848DB_SHOW_COMMAND(vmopag, vm_object_print_pages)
2849{
2850 vm_object_t object;
2851 vm_pindex_t fidx;
2852 vm_paddr_t pa;
2853 vm_page_t m, prev_m;
2854 int rcount;
2855
2856 TAILQ_FOREACH(object, &vm_object_list, object_list) {
2857 db_printf("new object: %p\n", (void *)object);
2858 if (db_pager_quit)
2859 return;
2860
2861 rcount = 0;
2862 fidx = 0;
2863 pa = -1;
2864 TAILQ_FOREACH(m, &object->memq, listq) {
2865 if (m->pindex > 128)
2866 break;
2867 if ((prev_m = TAILQ_PREV(m, pglist, listq)) != NULL &&
2868 prev_m->pindex + 1 != m->pindex) {
2869 if (rcount) {
2870 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2871 (long)fidx, rcount, (long)pa);
2872 if (db_pager_quit)
2873 return;
2874 rcount = 0;
2875 }
2876 }
2877 if (rcount &&
2878 (VM_PAGE_TO_PHYS(m) == pa + rcount * PAGE_SIZE)) {
2879 ++rcount;
2880 continue;
2881 }
2882 if (rcount) {
2883 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2884 (long)fidx, rcount, (long)pa);
2885 if (db_pager_quit)
2886 return;
2887 }
2888 fidx = m->pindex;
2889 pa = VM_PAGE_TO_PHYS(m);
2890 rcount = 1;
2891 }
2892 if (rcount) {
2893 db_printf(" index(%ld)run(%d)pa(0x%lx)\n",
2894 (long)fidx, rcount, (long)pa);
2895 if (db_pager_quit)
2896 return;
2897 }
2898 }
2899}
2900#endif /* DDB */
default_phys_pg_ops
const struct phys_pager_ops default_phys_pg_ops
Definition: phys_pager.c:61
pmap_remove_all
void pmap_remove_all(vm_page_t m)
pmap_remove_write
void pmap_remove_write(vm_page_t m)
uma_zone
Definition: uma_int.h:450
vm_map_entry
Definition: vm_map.h:101
vm_map_entry::eflags
vm_eflags_t eflags
Definition: vm_map.h:110
vm_map_entry::object
union vm_map_object object
Definition: vm_map.h:108
vm_map_entry::offset
vm_ooffset_t offset
Definition: vm_map.h:109
vm_map_entry::start
vm_offset_t start
Definition: vm_map.h:104
vm_map_entry::end
vm_offset_t end
Definition: vm_map.h:105
vm_map
Definition: vm_map.h:197
vm_object
Definition: vm_object.h:100
vm_object::backing_object_offset
vm_ooffset_t backing_object_offset
Definition: vm_object.h:121
vm_object::cred
struct ucred * cred
Definition: vm_object.h:188
vm_object::type
objtype_t type
Definition: vm_object.h:114
vm_object::lock
struct rwlock lock
Definition: vm_object.h:101
vm_object::swp
struct vm_object::@0::@4 swp
vm_object::size
vm_pindex_t size
Definition: vm_object.h:107
vm_object::busy
blockcount_t busy
Definition: vm_object.h:118
vm_object::swp_blks
struct pctrie swp_blks
Definition: vm_object.h:173
vm_object::domain
struct domainset_ref domain
Definition: vm_object.h:108
vm_object::rtree
struct vm_radix rtree
Definition: vm_object.h:106
vm_object::ref_count
volatile u_int ref_count
Definition: vm_object.h:111
vm_object::shadow_count
int shadow_count
Definition: vm_object.h:112
vm_object::handle
void * handle
Definition: vm_object.h:124
vm_object::cleangeneration
int cleangeneration
Definition: vm_object.h:110
vm_object::generation
volatile int generation
Definition: vm_object.h:109
vm_object::charge
vm_ooffset_t charge
Definition: vm_object.h:189
vm_object::memattr
vm_memattr_t memattr
Definition: vm_object.h:113
vm_object::flags
u_short flags
Definition: vm_object.h:115
vm_object::resident_page_count
int resident_page_count
Definition: vm_object.h:119
vm_object::paging_in_progress
blockcount_t paging_in_progress
Definition: vm_object.h:117
vm_object::un_pager
union vm_object::@0 un_pager
vm_object::backing_object
struct vm_object * backing_object
Definition: vm_object.h:120
vm_object::memq
struct pglist memq
Definition: vm_object.h:105
vm_object::pg_color
u_short pg_color
Definition: vm_object.h:116
swap_reserve_by_cred
bool swap_reserve_by_cred(vm_ooffset_t incr, struct ucred *cred)
Definition: swap_pager.c:262
swap_pager_find_least
vm_pindex_t swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)
Definition: swap_pager.c:2293
swap_release_by_cred
void swap_release_by_cred(vm_ooffset_t decr, struct ucred *cred)
Definition: swap_pager.c:357
swap_pager_copy
void swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t offset, int destroysource)
Definition: swap_pager.c:1060
swap_pager_swapped_pages
u_long swap_pager_swapped_pages(vm_object_t object)
Definition: swap_pager.c:1790
swap_pager.h
uma_zfree
static __inline void uma_zfree(uma_zone_t zone, void *item)
Definition: uma.h:373
UMA_ALIGN_PTR
#define UMA_ALIGN_PTR
Definition: uma.h:268
uma_zalloc
static __inline void * uma_zalloc(uma_zone_t zone, int flags)
Definition: uma.h:332
uma_zcreate
uma_zone_t uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, uma_init uminit, uma_fini fini, int align, uint32_t flags)
Definition: uma_core.c:3251
UMA_ZONE_NOFREE
#define UMA_ZONE_NOFREE
Definition: uma.h:241
vm_map_object::vm_object
struct vm_object * vm_object
Definition: vm_map.h:91
vm_map_object::sub_map
struct vm_map * sub_map
Definition: vm_map.h:92
objtype_t
u_char objtype_t
Definition: vm.h:102
vm_object_t
struct vm_object * vm_object_t
Definition: vm.h:114
OBJT_DEFAULT
@ OBJT_DEFAULT
Definition: vm.h:92
OBJT_SG
@ OBJT_SG
Definition: vm.h:98
OBJT_MGTDEVICE
@ OBJT_MGTDEVICE
Definition: vm.h:99
OBJT_VNODE
@ OBJT_VNODE
Definition: vm.h:94
OBJT_PHYS
@ OBJT_PHYS
Definition: vm.h:96
OBJT_DEAD
@ OBJT_DEAD
Definition: vm.h:97
OBJT_SWAP
@ OBJT_SWAP
Definition: vm.h:93
OBJT_FIRST_DYN
@ OBJT_FIRST_DYN
Definition: vm.h:100
OBJT_DEVICE
@ OBJT_DEVICE
Definition: vm.h:95
vm_extern.h
vm_kern.h
kernel_map
#define kernel_map
Definition: vm_kern.h:70
vm_map.h
VM_MAP_ENTRY_FOREACH
#define VM_MAP_ENTRY_FOREACH(it, map)
Definition: vm_map.h:505
MAP_ENTRY_IS_SUB_MAP
#define MAP_ENTRY_IS_SUB_MAP
Definition: vm_map.h:121
vm_object_collapse_scan_wait
static vm_page_t vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p)
Definition: vm_object.c:1677
obj_zone
static uma_zone_t obj_zone
Definition: vm_object.c:172
vm_object_deallocate_anon
static vm_object_t vm_object_deallocate_anon(vm_object_t backing_object)
Definition: vm_object.c:571
vm_object_deallocate_vnode
static void vm_object_deallocate_vnode(vm_object_t object)
Definition: vm_object.c:542
vm_object_destroy
void vm_object_destroy(vm_object_t object)
Definition: vm_object.c:705
vm_object_split
void vm_object_split(vm_map_entry_t entry)
Definition: vm_object.c:1525
vm_object_madvise_freespace
static void vm_object_madvise_freespace(vm_object_t object, int advice, vm_pindex_t pindex, vm_size_t size)
Definition: vm_object.c:1301
vm_object_init
void vm_object_init(void)
Definition: vm_object.c:278
vm_object_reference
void vm_object_reference(vm_object_t object)
Definition: vm_object.c:504
vm_object_pip_sleep
static void vm_object_pip_sleep(vm_object_t object, const char *waitid)
Definition: vm_object.c:372
vm_object_terminate_pages
static void vm_object_terminate_pages(vm_object_t object)
Definition: vm_object.c:906
vm_object_pip_wakeupn
void vm_object_pip_wakeupn(vm_object_t object, short i)
Definition: vm_object.c:359
vm_object_set_writeable_dirty_
void vm_object_set_writeable_dirty_(vm_object_t object)
Definition: vm_object.c:2378
vm_object_sync
boolean_t vm_object_sync(vm_object_t object, vm_ooffset_t offset, vm_size_t size, boolean_t syncio, boolean_t invalidate)
Definition: vm_object.c:1195
vm_object_allocate_anon
vm_object_t vm_object_allocate_anon(vm_pindex_t size, vm_object_t backing_object, struct ucred *cred, vm_size_t charge)
Definition: vm_object.c:461
sysctl_vm_object_list_swap
static int sysctl_vm_object_list_swap(SYSCTL_HANDLER_ARGS)
Definition: vm_object.c:2673
vm_object_unwire
void vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length, uint8_t queue)
Definition: vm_object.c:2398
vm_object_list_mtx
struct mtx vm_object_list_mtx
Definition: vm_object.c:150
vm_object_backing_collapse_wait
static vm_object_t vm_object_backing_collapse_wait(vm_object_t object)
Definition: vm_object.c:879
vm_object_backing_transfer
static void vm_object_backing_transfer(vm_object_t object, vm_object_t backing_object)
Definition: vm_object.c:831
vm_object_reference_vnode
static void vm_object_reference_vnode(vm_object_t object)
Definition: vm_object.c:481
vm_object_set_memattr
int vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr)
Definition: vm_object.c:329
vm_object_pip_wait
void vm_object_pip_wait(vm_object_t object, const char *waitid)
Definition: vm_object.c:380
vm_object_backing_remove
static void vm_object_backing_remove(vm_object_t object)
Definition: vm_object.c:753
sysctl_vm_object_list
static int sysctl_vm_object_list(SYSCTL_HANDLER_ARGS)
Definition: vm_object.c:2663
COUNTER_U64_DEFINE_EARLY
static COUNTER_U64_DEFINE_EARLY(object_collapses)
vm_object_sub_shadow
static void vm_object_sub_shadow(vm_object_t object)
Definition: vm_object.c:725
vm_object_scan_all_shadowed
static bool vm_object_scan_all_shadowed(vm_object_t object)
Definition: vm_object.c:1708
vm_object_list_handler
static int vm_object_list_handler(struct sysctl_req *req, bool swap_only)
Definition: vm_object.c:2538
vm_object_collapse
void vm_object_collapse(vm_object_t object)
Definition: vm_object.c:1927
vm_object_page_noreuse
void vm_object_page_noreuse(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
Definition: vm_object.c:2208
vm_object_page_collect_flush
static int vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, int flags, boolean_t *allclean, boolean_t *eio)
Definition: vm_object.c:1138
vm_object_pip_wait_unlocked
void vm_object_pip_wait_unlocked(vm_object_t object, const char *waitid)
Definition: vm_object.c:390
vm_object_page_remove
void vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int options)
Definition: vm_object.c:2101
SYSCTL_COUNTER_U64
SYSCTL_COUNTER_U64(_vm_stats_object, OID_AUTO, collapses, CTLFLAG_RD, &object_collapses, "VM object collapses")
vm_object_busy
void vm_object_busy(vm_object_t obj)
Definition: vm_object.c:2496
vm_object_pip_add
void vm_object_pip_add(vm_object_t object, short i)
Definition: vm_object.c:344
vm_object_backing_remove_locked
static void vm_object_backing_remove_locked(vm_object_t object)
Definition: vm_object.c:733
vm_object_pip_wakeup
void vm_object_pip_wakeup(vm_object_t object)
Definition: vm_object.c:352
vm_object_reference_locked
void vm_object_reference_locked(vm_object_t object)
Definition: vm_object.c:526
vm_object_allocate_dyn
vm_object_t vm_object_allocate_dyn(objtype_t dyntype, vm_pindex_t size, u_short flags)
Definition: vm_object.c:442
__FBSDID
__FBSDID("$FreeBSD$")
vm_object_clear_flag
void vm_object_clear_flag(vm_object_t object, u_short bits)
Definition: vm_object.c:312
SYSCTL_NODE
static SYSCTL_NODE(_vm_stats, OID_AUTO, object, CTLFLAG_RD|CTLFLAG_MPSAFE, 0, "VM object stats")
vm_object_is_active
bool vm_object_is_active(vm_object_t obj)
Definition: vm_object.c:2531
vm_object_terminate
void vm_object_terminate(vm_object_t object)
Definition: vm_object.c:953
vm_object_allocate
vm_object_t vm_object_allocate(objtype_t type, vm_pindex_t size)
Definition: vm_object.c:404
vm_object_zinit
static int vm_object_zinit(void *mem, int size, int flags)
Definition: vm_object.c:211
vm_object_vnode
struct vnode * vm_object_vnode(vm_object_t object)
Definition: vm_object.c:2481
old_msync
static int old_msync
Definition: vm_object.c:112
vm_object_backing_insert_ref
static void vm_object_backing_insert_ref(vm_object_t object, vm_object_t backing_object)
Definition: vm_object.c:807
vm_object_page_remove_write
static boolean_t vm_object_page_remove_write(vm_page_t p, int flags, boolean_t *allclean)
Definition: vm_object.c:1005
vm_object_madvise
void vm_object_madvise(vm_object_t object, vm_pindex_t pindex, vm_pindex_t end, int advice)
Definition: vm_object.c:1331
vm_object_coalesce
boolean_t vm_object_coalesce(vm_object_t prev_object, vm_ooffset_t prev_offset, vm_size_t prev_size, vm_size_t next_size, boolean_t reserved)
Definition: vm_object.c:2289
vm_object_backing_insert_locked
static void vm_object_backing_insert_locked(vm_object_t object, vm_object_t backing_object)
Definition: vm_object.c:771
vm_object_backing_insert
static void vm_object_backing_insert(vm_object_t object, vm_object_t backing_object)
Definition: vm_object.c:787
vm_object_page_clean
boolean_t vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end, int flags)
Definition: vm_object.c:1047
SYSCTL_INT
SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0, "Use old (insecure) msync behavior")
vm_object_busy_wait
void vm_object_busy_wait(vm_object_t obj, const char *wmesg)
Definition: vm_object.c:2514
vm_object_populate
boolean_t vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
Definition: vm_object.c:2240
vm_object_collapse_wait
static void vm_object_collapse_wait(vm_object_t object)
Definition: vm_object.c:863
_vm_object_allocate
static void _vm_object_allocate(objtype_t type, vm_pindex_t size, u_short flags, vm_object_t object, void *handle)
Definition: vm_object.c:235
vm_object_mightbedirty_
bool vm_object_mightbedirty_(vm_object_t object)
Definition: vm_object.c:2384
SYSCTL_PROC
SYSCTL_PROC(_vm, OID_AUTO, objects, CTLTYPE_STRUCT|CTLFLAG_RW|CTLFLAG_SKIP|CTLFLAG_MPSAFE, NULL, 0, sysctl_vm_object_list, "S,kinfo_vmobject", "List of VM objects")
kernel_object_store
struct vm_object kernel_object_store
Definition: vm_object.c:152
vm_object_advice_applies
static bool vm_object_advice_applies(vm_object_t object, int advice)
Definition: vm_object.c:1289
vm_object_list
struct object_q vm_object_list
Definition: vm_object.c:149
vm_object_collapse_scan
static void vm_object_collapse_scan(vm_object_t object)
Definition: vm_object.c:1795
vm_object_deallocate
void vm_object_deallocate(vm_object_t object)
Definition: vm_object.c:625
vm_object_shadow
void vm_object_shadow(vm_object_t *object, vm_ooffset_t *offset, vm_size_t length, struct ucred *cred, bool shared)
Definition: vm_object.c:1445
vm_object_unbusy
void vm_object_unbusy(vm_object_t obj)
Definition: vm_object.c:2507
vm_object.h
VM_OBJECT_ASSERT_UNLOCKED
#define VM_OBJECT_ASSERT_UNLOCKED(object)
Definition: vm_object.h:254
vm_object_busied
static bool vm_object_busied(vm_object_t object)
Definition: vm_object.h:347
OBJ_ANON
#define OBJ_ANON
Definition: vm_object.h:200
VM_OBJECT_RLOCK
#define VM_OBJECT_RLOCK(object)
Definition: vm_object.h:258
OBJ_COLORED
#define OBJ_COLORED
Definition: vm_object.h:208
OBJ_UNMANAGED
#define OBJ_UNMANAGED
Definition: vm_object.h:197
OBJPC_INVAL
#define OBJPC_INVAL
Definition: vm_object.h:227
OBJPC_SYNC
#define OBJPC_SYNC
Definition: vm_object.h:226
OBJ_ONEMAPPING
#define OBJ_ONEMAPPING
Definition: vm_object.h:209
umtx_shm_object_terminated
void umtx_shm_object_terminated(vm_object_t object)
OBJ_SHADOWLIST
#define OBJ_SHADOWLIST
Definition: vm_object.h:204
OBJ_PG_DTOR
#define OBJ_PG_DTOR
Definition: vm_object.h:203
vm_object_kvme_type
int vm_object_kvme_type(vm_object_t object, struct vnode **vpp)
Definition: vm_pager.c:611
OBJ_FICTITIOUS
#define OBJ_FICTITIOUS
Definition: vm_object.h:196
VM_OBJECT_RUNLOCK
#define VM_OBJECT_RUNLOCK(object)
Definition: vm_object.h:260
OBJPR_NOTMAPPED
#define OBJPR_NOTMAPPED
Definition: vm_object.h:234
umtx_shm_vnobj_persistent
int umtx_shm_vnobj_persistent
OBJPR_CLEANONLY
#define OBJPR_CLEANONLY
Definition: vm_object.h:233
IDX_TO_OFF
#define IDX_TO_OFF(idx)
Definition: vm_object.h:220
OBJPR_VALIDONLY
#define OBJPR_VALIDONLY
Definition: vm_object.h:235
kernel_object
#define kernel_object
Definition: vm_object.h:245
vm_object_mightbedirty
bool vm_object_mightbedirty(vm_object_t object)
Definition: vm_pager.c:594
OBJ_SPLIT
#define OBJ_SPLIT
Definition: vm_object.h:206
VM_OBJECT_TRYWLOCK
#define VM_OBJECT_TRYWLOCK(object)
Definition: vm_object.h:266
VM_OBJECT_ASSERT_LOCKED
#define VM_OBJECT_ASSERT_LOCKED(object)
Definition: vm_object.h:248
VM_OBJECT_WLOCK
#define VM_OBJECT_WLOCK(object)
Definition: vm_object.h:270
vm_object_set_flag
static __inline void vm_object_set_flag(vm_object_t object, u_short bits)
Definition: vm_object.h:294
OBJPC_NOSYNC
#define OBJPC_NOSYNC
Definition: vm_object.h:228
OBJ_SWAP
#define OBJ_SWAP
Definition: vm_object.h:205
OBJ_DEAD
#define OBJ_DEAD
Definition: vm_object.h:199
umtx_shm_object_init
void umtx_shm_object_init(vm_object_t object)
OFF_TO_IDX
#define OFF_TO_IDX(off)
Definition: vm_object.h:221
VM_OBJECT_WUNLOCK
#define VM_OBJECT_WUNLOCK(object)
Definition: vm_object.h:274
VM_OBJECT_ASSERT_WLOCKED
#define VM_OBJECT_ASSERT_WLOCKED(object)
Definition: vm_object.h:252
vm_object_print
void vm_object_print(long addr, boolean_t have_addr, long count, char *modif)
OBJ_COLLAPSING
#define OBJ_COLLAPSING
Definition: vm_object.h:207
vm_page_advise
void vm_page_advise(vm_page_t m, int advice)
Definition: vm_page.c:4370
vm_page_lookup
vm_page_t vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
Definition: vm_page.c:1627
vm_page_busy_acquire
bool vm_page_busy_acquire(vm_page_t m, int allocflags)
Definition: vm_page.c:869
vm_page_prev
vm_page_t vm_page_prev(vm_page_t m)
Definition: vm_page.c:1745
vm_page_tryxbusy
int vm_page_tryxbusy(vm_page_t m)
Definition: vm_page.c:1151
vm_page_remove
bool vm_page_remove(vm_page_t m)
Definition: vm_page.c:1594
vm_page_find_least
vm_page_t vm_page_find_least(vm_object_t object, vm_pindex_t pindex)
Definition: vm_page.c:1708
vm_page_next
vm_page_t vm_page_next(vm_page_t m)
Definition: vm_page.c:1725
vm_page_try_remove_all
bool vm_page_try_remove_all(vm_page_t m)
Definition: vm_page.c:4348
vm_page_grab_valid
int vm_page_grab_valid(vm_page_t *mp, vm_object_t object, vm_pindex_t pindex, int allocflags)
Definition: vm_page.c:4647
vm_page_invalid
void vm_page_invalid(vm_page_t m)
Definition: vm_page.c:5318
vm_page_rename
int vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
Definition: vm_page.c:1845
vm_page_sunbusy
void vm_page_sunbusy(vm_page_t m)
Definition: vm_page.c:972
vm_page_busy_sleep
bool vm_page_busy_sleep(vm_page_t m, const char *wmesg, int allocflags)
Definition: vm_page.c:1014
vm_page_unwire
void vm_page_unwire(vm_page_t m, uint8_t nqueue)
Definition: vm_page.c:4079
vm_page_try_remove_write
bool vm_page_try_remove_write(vm_page_t m)
Definition: vm_page.c:4358
vm_page_trysbusy
int vm_page_trysbusy(vm_page_t m)
Definition: vm_page.c:1113
vm_page_free
void vm_page_free(vm_page_t m)
Definition: vm_page.c:1326
vm_page_deactivate_noreuse
void vm_page_deactivate_noreuse(vm_page_t m)
Definition: vm_page.c:4177
vm_page.h
vm_page_aflag_set
static void vm_page_aflag_set(vm_page_t m, uint16_t bits)
Definition: vm_page.h:858
PQ_ACTIVE
#define PQ_ACTIVE
Definition: vm_page.h:333
vm_page_assert_xbusied
#define vm_page_assert_xbusied(m)
Definition: vm_page.h:745
vm_page_all_valid
static bool vm_page_all_valid(vm_page_t m)
Definition: vm_page.h:990
VPO_UNMANAGED
#define VPO_UNMANAGED
Definition: vm_page.h:296
vm_page_assert_busied
#define vm_page_assert_busied(m)
Definition: vm_page.h:723
PGA_REFERENCED
#define PGA_REFERENCED
Definition: vm_page.h:438
PQ_INACTIVE
#define PQ_INACTIVE
Definition: vm_page.h:332
vm_page_none_valid
static bool vm_page_none_valid(vm_page_t m)
Definition: vm_page.h:997
vm_page_undirty
static __inline void vm_page_undirty(vm_page_t m)
Definition: vm_page.h:902
VM_ALLOC_IGN_SBUSY
#define VM_ALLOC_IGN_SBUSY
Definition: vm_page.h:548
vm_page_drop
static u_int vm_page_drop(vm_page_t m, u_int val)
Definition: vm_page.h:959
vm_page_assert_unbusied
#define vm_page_assert_unbusied(m)
Definition: vm_page.h:733
VM_ALLOC_NORMAL
#define VM_ALLOC_NORMAL
Definition: vm_page.h:535
VPRC_OBJREF
#define VPRC_OBJREF
Definition: vm_page.h:276
VM_ALLOC_WAITFAIL
#define VM_ALLOC_WAITFAIL
Definition: vm_page.h:540
vm_page_xunbusy
#define vm_page_xunbusy(m)
Definition: vm_page.h:764
PGA_NOSYNC
#define PGA_NOSYNC
Definition: vm_page.h:444
vm_page_lock_assert
#define vm_page_lock_assert(m, a)
Definition: vm_page.h:389
vm_page_wired
static bool vm_page_wired(vm_page_t m)
Definition: vm_page.h:983
PG_FICTITIOUS
#define PG_FICTITIOUS
Definition: vm_page.h:460
VM_PAGE_TO_PHYS
#define VM_PAGE_TO_PHYS(entry)
Definition: vm_page.h:506
vm_pageout_page_count
int vm_pageout_page_count
Definition: vm_pageout.c:152
vm_pageout_flush
int vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen, boolean_t *eio)
Definition: vm_pageout.c:471
vm_pageout.h
vm_pagequeue.h
vm_pager_deallocate
void vm_pager_deallocate(vm_object_t object)
Definition: vm_pager.c:271
vm_pager.h
vm_pager_freespace
static __inline void vm_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size)
Definition: vm_pager.h:242
VM_PAGER_PUT_SYNC
#define VM_PAGER_PUT_SYNC
Definition: vm_pager.h:117
VM_PAGER_OK
#define VM_PAGER_OK
Definition: vm_pager.h:110
vm_pager_getvp
static __inline void vm_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)
Definition: vm_pager.h:229
VM_PAGER_CLUSTER_OK
#define VM_PAGER_CLUSTER_OK
Definition: vm_pager.h:120
VM_PAGER_PUT_INVAL
#define VM_PAGER_PUT_INVAL
Definition: vm_pager.h:118
vm_pager_has_page
static __inline boolean_t vm_pager_has_page(vm_object_t object, vm_pindex_t offset, int *before, int *after)
Definition: vm_pager.h:163
vm_param.h
KERN_SUCCESS
#define KERN_SUCCESS
Definition: vm_param.h:107
KERN_FAILURE
#define KERN_FAILURE
Definition: vm_param.h:112
KERN_INVALID_ARGUMENT
#define KERN_INVALID_ARGUMENT
Definition: vm_param.h:111
vm_phys.h
vm_radix_reclaim_allnodes
void vm_radix_reclaim_allnodes(struct vm_radix *rtree)
Definition: vm_radix.c:825
vm_radix_zinit
void vm_radix_zinit(void)
Definition: vm_radix.c:372
vm_radix_wait
void vm_radix_wait(void)
Definition: vm_radix.c:881
vm_radix.h
vm_radix_init
static __inline void vm_radix_init(struct vm_radix *rtree)
Definition: vm_radix.h:53
vm_radix_is_empty
static __inline boolean_t vm_radix_is_empty(struct vm_radix *rtree)
Definition: vm_radix.h:60
vm_reserv.h