47#include "opt_watchdog.h"
54#include <sys/capsicum.h>
55#include <sys/condvar.h>
57#include <sys/counter.h>
58#include <sys/dirent.h>
60#include <sys/eventhandler.h>
61#include <sys/extattr.h>
66#include <sys/kernel.h>
67#include <sys/kthread.h>
70#include <sys/malloc.h>
73#include <sys/pctrie.h>
75#include <sys/reboot.h>
76#include <sys/refcount.h>
77#include <sys/rwlock.h>
79#include <sys/sleepqueue.h>
83#include <sys/sysctl.h>
84#include <sys/syslog.h>
85#include <sys/vmmeter.h>
87#include <sys/watchdog.h>
89#include <machine/stdarg.h>
91#include <security/mac/mac_framework.h>
94#include <vm/vm_object.h>
95#include <vm/vm_extern.h>
98#include <vm/vm_page.h>
99#include <vm/vm_kern.h>
108 int slpflag,
int slptimeo);
114static void vgonel(
struct vnode *);
123 daddr_t startlbn, daddr_t endlbn);
133 "Number of vnodes in existence");
137 "Number of vnodes created by getnewvnode");
144 VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
145 VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON
148 0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK,
149 S_IFSOCK, S_IFIFO, S_IFMT, S_IFMT
156static struct vnode *vnode_list_free_marker;
157static struct vnode *vnode_list_reclaim_marker;
182static long wantfreevnodes;
183static long __exclusive_cache_line freevnodes;
185 &freevnodes, 0,
"Number of \"free\" vnodes");
186static long freevnodes_old;
188static counter_u64_t recycles_count;
190 "Number of vnodes recycled to meet vnode cache targets");
192static counter_u64_t recycles_free_count;
194 "Number of free vnodes recycled to meet vnode cache targets");
196static counter_u64_t deferred_inact;
198 "Number of times inactive processing was deferred");
201static struct mtx mntid_mtx;
209static struct mtx __exclusive_cache_line vnode_list_mtx;
212struct nfs_public nfs_pub;
214static uma_zone_t buf_trie_zone;
215static smr_t buf_trie_smr;
218static uma_zone_t vnode_zone;
221__read_frequently smr_t vfs_smr;
248static int syncer_delayno;
249static long syncer_mask;
251static struct synclist *syncer_workitem_pending;
262static struct mtx sync_mtx;
263static struct cv sync_wakeup;
265#define SYNCER_MAXDELAY 32
267static int syncdelay = 30;
268static int filedelay = 30;
269SYSCTL_INT(_kern, OID_AUTO, filedelay, CTLFLAG_RW, &filedelay, 0,
270 "Time to delay syncing files (in seconds)");
271static int dirdelay = 29;
272SYSCTL_INT(_kern, OID_AUTO, dirdelay, CTLFLAG_RW, &dirdelay, 0,
273 "Time to delay syncing directories (in seconds)");
274static int metadelay = 28;
275SYSCTL_INT(_kern, OID_AUTO, metadelay, CTLFLAG_RW, &metadelay, 0,
276 "Time to delay syncing metadata (in seconds)");
278static int stat_rush_requests;
279SYSCTL_INT(_debug, OID_AUTO, rush_requests, CTLFLAG_RW, &stat_rush_requests, 0,
280 "Number of times I/O speeded up (rush requests)");
282#define VDBATCH_SIZE 8
296#define SYNCER_SHUTDOWN_SPEEDUP 4
323 if (error != 0 || req->newptr == NULL)
328 mtx_lock(&vnode_list_mtx);
332 mtx_unlock(&vnode_list_mtx);
345 "LU",
"Target for maximum number of vnodes");
353 val = wantfreevnodes;
355 if (error != 0 || req->newptr == NULL)
358 if (val == wantfreevnodes)
360 mtx_lock(&vnode_list_mtx);
361 wantfreevnodes = val;
363 mtx_unlock(&vnode_list_mtx);
369 "LU",
"Target for minimum number of \"free\" vnodes");
372 &wantfreevnodes, 0,
"Old name for vfs.wantfreevnodes (legacy)");
375 &
vnlru_nowhere, 0,
"Number of times the vnlru process ran without success");
383 unsigned long ndflags;
386 if (req->newptr == NULL)
388 if (req->newlen >= PATH_MAX)
391 buf =
malloc(PATH_MAX, M_TEMP, M_WAITOK);
392 error = SYSCTL_IN(req,
buf, req->newlen);
396 buf[req->newlen] =
'\0';
398 ndflags = LOCKLEAF | NOFOLLOW | AUDITVNODE1 | SAVENAME;
399 NDINIT(&nd, LOOKUP, ndflags, UIO_SYSSPACE,
buf);
400 if ((error =
namei(&nd)) != 0)
404 if (VN_IS_DOOMED(vp)) {
415 counter_u64_add(recycles_count, 1);
427 struct thread *td = curthread;
433 if (req->newptr == NULL)
444 error = vn_lock(vp, LK_EXCLUSIVE);
448 counter_u64_add(recycles_count, 1);
457 CTLTYPE_STRING | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0,
460 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_WR, NULL, 0,
462 "Try to reclaim a vnode by its file descriptor");
468 sizeof(
struct vnode) < 1UL << (
vnsz2log + 1),
469 "vnsz2log needs to be updated");
478 return (uma_zalloc_smr(buf_trie_zone, M_NOWAIT));
484 uma_zfree_smr(buf_trie_zone, node);
497#define MAXVNODES_MAX (512UL * 1024 * 1024 / 64)
507 vp =
malloc(
sizeof(
struct vnode), M_VNODE_MARKER, M_WAITOK | M_ZERO);
508 vp->v_type = VMARKER;
518 MPASS(vp->v_type == VMARKER);
519 free(vp, M_VNODE_MARKER);
524vnode_ctor(
void *mem,
int size,
void *arg __unused,
int flags __unused)
526 kasan_mark(mem, size, roundup2(size, UMA_ALIGN_PTR + 1), 0);
531vnode_dtor(
void *mem,
int size,
void *arg __unused)
533 size_t end1, end2, off1, off2;
536 offsetof(
struct vnode, v_dbatchcpu),
537 "KASAN marks require updating");
539 off1 = offsetof(
struct vnode, v_vnodelist);
540 off2 = offsetof(
struct vnode, v_dbatchcpu);
541 end1 = off1 +
sizeof(((
struct vnode *)NULL)->v_vnodelist);
542 end2 = off2 +
sizeof(((
struct vnode *)NULL)->v_dbatchcpu);
553 off1 = rounddown2(off1, KASAN_SHADOW_SCALE);
557 off1 = roundup2(end1, KASAN_SHADOW_SCALE);
558 off2 = rounddown2(off2, KASAN_SHADOW_SCALE);
560 kasan_mark((
void *)((
char *)mem + off1), off2 - off1,
561 off2 - off1, KASAN_UMA_FREED);
564 off2 = roundup2(end2, KASAN_SHADOW_SCALE);
565 kasan_mark((
void *)((
char *)mem + off2), size - off2, size - off2,
583 vp->v_vnlock = &vp->v_lock;
584 mtx_init(&vp->v_interlock,
"vnode interlock", NULL, MTX_DEF);
588 lockinit(vp->v_vnlock, PVFS,
"vnode", VLKTIMEOUT,
589 LK_NOSHARE | LK_IS_VNODE);
603 vp->v_dbatchcpu = NOCPU;
608 vp->v_holdcnt = VHOLD_NO_SMR;
610 mtx_lock(&vnode_list_mtx);
611 TAILQ_INSERT_BEFORE(vnode_list_free_marker, vp, v_vnodelist);
612 mtx_unlock(&vnode_list_mtx);
627 mtx_lock(&vnode_list_mtx);
628 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist);
629 mtx_unlock(&vnode_list_mtx);
632 mtx_destroy(&vp->v_interlock);
634 rw_destroy(BO_LOCKPTR(bo));
653#define NFS_NCLNODE_SZ (528 + 64)
656#define NFS_NCLNODE_SZ (360 + 32)
666 int cpu, physvnodes, virtvnodes;
679 physvnodes =
maxproc + pgtok(vm_cnt.v_page_count) / 64 +
680 3 * min(98304 * 16, pgtok(vm_cnt.v_page_count)) / 64;
681 virtvnodes =
vm_kmem_size / (10 * (
sizeof(
struct vm_object) +
686 printf(
"Reducing kern.maxvnodes %lu -> %lu\n",
691 mtx_init(&mntid_mtx,
"mntid", NULL, MTX_DEF);
692 TAILQ_INIT(&vnode_list);
693 mtx_init(&vnode_list_mtx,
"vnode_list", NULL, MTX_DEF);
697 mtx_lock(&vnode_list_mtx);
699 mtx_unlock(&vnode_list_mtx);
701 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_free_marker, v_vnodelist);
703 TAILQ_INSERT_HEAD(&vnode_list, vnode_list_reclaim_marker, v_vnodelist);
712 vnode_zone = uma_zcreate(
"VNODE",
sizeof(
struct vnode), ctor, dtor,
714 uma_zone_set_smr(vnode_zone, vfs_smr);
723 UMA_ZONE_NOFREE | UMA_ZONE_SMR);
724 buf_trie_smr = uma_zone_get_smr(buf_trie_zone);
725 uma_prealloc(buf_trie_zone,
nbuf);
735 syncer_workitem_pending =
hashinit(syncer_maxdelay, M_VNODE,
737 syncer_maxdelay = syncer_mask + 1;
738 mtx_init(&sync_mtx,
"Syncer mtx", NULL, MTX_DEF);
739 cv_init(&sync_wakeup,
"syncer");
742 vd = DPCPU_ID_PTR((cpu), vd);
743 bzero(vd,
sizeof(*vd));
744 mtx_init(&vd->lock,
"vdbatch", NULL, MTX_DEF);
788 struct mount_pcpu *mpcpu;
790 MPASS((
flags & ~MBF_MASK) == 0);
791 CTR3(KTR_VFS,
"%s: mp %p with flags %d", __func__, mp,
flags);
793 if (vfs_op_thread_enter(mp, mpcpu)) {
794 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
795 MPASS((mp->mnt_kern_flag & MNTK_UNMOUNT) == 0);
796 MPASS((mp->mnt_kern_flag & MNTK_REFEXPIRE) == 0);
797 vfs_mp_count_add_pcpu(mpcpu, ref, 1);
798 vfs_mp_count_add_pcpu(mpcpu, lockref, 1);
799 vfs_op_thread_exit(mp, mpcpu);
800 if (
flags & MBF_MNTLSTLOCK)
806 vfs_assert_mount_counters(mp);
820 while (mp->mnt_kern_flag & MNTK_UNMOUNT) {
821 KASSERT(TAILQ_EMPTY(&mp->mnt_uppers),
822 (
"%s: non-empty upper mount list with pending unmount",
824 if (
flags & MBF_NOWAIT || mp->mnt_kern_flag & MNTK_REFEXPIRE) {
827 CTR1(KTR_VFS,
"%s: failed busying before sleeping",
831 if (
flags & MBF_MNTLSTLOCK)
833 mp->mnt_kern_flag |= MNTK_MWAIT;
834 msleep(mp, MNT_MTX(mp), PVFS | PDROP,
"vfs_busy", 0);
835 if (
flags & MBF_MNTLSTLOCK)
839 if (
flags & MBF_MNTLSTLOCK)
852 struct mount_pcpu *mpcpu;
855 CTR2(KTR_VFS,
"%s: mp %p", __func__, mp);
857 if (vfs_op_thread_enter(mp, mpcpu)) {
858 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
859 vfs_mp_count_sub_pcpu(mpcpu, lockref, 1);
860 vfs_mp_count_sub_pcpu(mpcpu, ref, 1);
861 vfs_op_thread_exit(mp, mpcpu);
866 vfs_assert_mount_counters(mp);
868 c = --mp->mnt_lockref;
869 if (mp->mnt_vfs_ops == 0) {
870 MPASS((mp->mnt_kern_flag & MNTK_DRAINING) == 0);
875 vfs_dump_mount_counters(mp);
876 if (c == 0 && (mp->mnt_kern_flag & MNTK_DRAINING) != 0) {
877 MPASS(mp->mnt_kern_flag & MNTK_UNMOUNT);
878 CTR1(KTR_VFS,
"%s: waking up waiters", __func__);
879 mp->mnt_kern_flag &= ~MNTK_DRAINING;
893 CTR2(KTR_VFS,
"%s: fsid %p", __func__, fsid);
895 TAILQ_FOREACH(mp, &
mountlist, mnt_list) {
896 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) {
903 CTR2(KTR_VFS,
"%s: lookup failed for %p id", __func__, fsid);
904 return ((
struct mount *) 0);
920#define FSID_CACHE_SIZE 256
921 typedef struct mount *
volatile vmp_t;
927 CTR2(KTR_VFS,
"%s: fsid %p", __func__, fsid);
928 hash = fsid->val[0] ^ fsid->val[1];
931 if (mp == NULL || fsidcmp(&mp->mnt_stat.f_fsid, fsid) != 0)
937 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0)
944 TAILQ_FOREACH(mp, &
mountlist, mnt_list) {
945 if (fsidcmp(&mp->mnt_stat.f_fsid, fsid) == 0) {
946 error =
vfs_busy(mp, MBF_MNTLSTLOCK);
956 CTR2(KTR_VFS,
"%s: lookup failed for %p id", __func__, fsid);
958 return ((
struct mount *) 0);
969 if (jailed(td->td_ucred)) {
974 if (!
prison_allow(td->td_ucred, mp->mnt_vfc->vfc_prison_flag))
992 if (!(mp->mnt_vfc->vfc_flags & VFCF_DELEGADMIN) &&
993 mp->mnt_cred->cr_uid != td->td_ucred->cr_uid) {
994 if ((error =
priv_check(td, PRIV_VFS_MOUNT_OWNER)) != 0)
1015 static uint16_t mntid_base;
1020 CTR2(KTR_VFS,
"%s: mp %p", __func__, mp);
1021 mtx_lock(&mntid_mtx);
1022 mtype = mp->mnt_vfc->vfc_typenum;
1023 tfsid.val[1] = mtype;
1024 mtype = (mtype & 0xFF) << 24;
1026 tfsid.val[0] = makedev(255,
1027 mtype | ((mntid_base & 0xFF00) << 8) | (mntid_base & 0xFF));
1033 mp->mnt_stat.f_fsid.val[0] = tfsid.val[0];
1034 mp->mnt_stat.f_fsid.val[1] = tfsid.val[1];
1035 mtx_unlock(&mntid_mtx);
1051 "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to us, "
1052 "3+: sec + ns (max. precision))");
1072 TIMEVAL_TO_TIMESPEC(&tv, tsp);
1088 vap->va_type = VNON;
1089 vap->va_size = VNOVAL;
1090 vap->va_bytes = VNOVAL;
1091 vap->va_mode = VNOVAL;
1092 vap->va_nlink = VNOVAL;
1093 vap->va_uid = VNOVAL;
1094 vap->va_gid = VNOVAL;
1095 vap->va_fsid = VNOVAL;
1096 vap->va_fileid = VNOVAL;
1097 vap->va_blocksize = VNOVAL;
1098 vap->va_rdev = VNOVAL;
1099 vap->va_atime.tv_sec = VNOVAL;
1100 vap->va_atime.tv_nsec = VNOVAL;
1101 vap->va_mtime.tv_sec = VNOVAL;
1102 vap->va_mtime.tv_nsec = VNOVAL;
1103 vap->va_ctime.tv_sec = VNOVAL;
1104 vap->va_ctime.tv_nsec = VNOVAL;
1105 vap->va_birthtime.tv_sec = VNOVAL;
1106 vap->va_birthtime.tv_nsec = VNOVAL;
1107 vap->va_flags = VNOVAL;
1108 vap->va_gen = VNOVAL;
1109 vap->va_vaflags = 0;
1148 struct vnode *vp, *mvp;
1150 struct vm_object *object;
1154 mtx_assert(&vnode_list_mtx, MA_OWNED);
1159 mvp = vnode_list_reclaim_marker;
1162 while (done < target) {
1163 vp = TAILQ_NEXT(vp, v_vnodelist);
1164 if (__predict_false(vp == NULL))
1167 if (__predict_false(vp->v_type == VMARKER))
1176 if (vp->v_usecount > 0 || vp->v_holdcnt == 0 ||
1177 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)))
1180 if (vp->v_type == VBAD || vp->v_type == VNON)
1183 object = atomic_load_ptr(&vp->v_object);
1184 if (
object == NULL || object->resident_page_count > trigger) {
1195 if (!VI_TRYLOCK(vp))
1197 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) {
1201 if (vp->v_mount == NULL) {
1207 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1208 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
1209 mtx_unlock(&vnode_list_mtx);
1213 goto next_iter_unlocked;
1215 if (VOP_LOCK(vp, LK_EXCLUSIVE|LK_NOWAIT) != 0) {
1218 goto next_iter_unlocked;
1222 if (vp->v_usecount > 0 ||
1223 (!reclaim_nc_src && !LIST_EMPTY(&vp->v_cache_src)) ||
1224 (vp->v_object != NULL && vp->v_object->handle == vp &&
1225 vp->v_object->resident_page_count > trigger)) {
1229 goto next_iter_unlocked;
1231 counter_u64_add(recycles_count, 1);
1240 mtx_lock(&vnode_list_mtx);
1243 MPASS(vp->v_type != VMARKER);
1246 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1247 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
1248 mtx_unlock(&vnode_list_mtx);
1250 mtx_lock(&vnode_list_mtx);
1253 if (done == 0 && !retried) {
1254 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1255 TAILQ_INSERT_HEAD(&vnode_list, mvp, v_vnodelist);
1265 "limit on vnode free requests per call to the vnlru_free routine");
1277 mtx_assert(&vnode_list_mtx, MA_OWNED);
1286 vp = TAILQ_NEXT(vp, v_vnodelist);
1287 if (__predict_false(vp == NULL)) {
1288 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1289 TAILQ_INSERT_TAIL(&vnode_list, mvp, v_vnodelist);
1292 if (__predict_false(vp->v_type == VMARKER))
1294 if (vp->v_holdcnt > 0)
1302 if (mnt_op != NULL && (mp = vp->v_mount) != NULL &&
1303 mp->mnt_op != mnt_op) {
1306 if (__predict_false(vp->v_type == VBAD || vp->v_type == VNON)) {
1311 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1312 TAILQ_INSERT_AFTER(&vnode_list, vp, mvp, v_vnodelist);
1313 mtx_unlock(&vnode_list_mtx);
1337 mtx_lock(&vnode_list_mtx);
1340 return (ocount -
count);
1347 mtx_assert(&vnode_list_mtx, MA_OWNED);
1355 MPASS(mnt_op != NULL);
1357 VNPASS(mvp->v_type == VMARKER, mvp);
1358 mtx_lock(&vnode_list_mtx);
1360 mtx_unlock(&vnode_list_mtx);
1369 mtx_lock(&vnode_list_mtx);
1370 TAILQ_INSERT_BEFORE(vnode_list_free_marker, mvp, v_vnodelist);
1371 mtx_unlock(&vnode_list_mtx);
1378 mtx_lock(&vnode_list_mtx);
1379 TAILQ_REMOVE(&vnode_list, mvp, v_vnodelist);
1380 mtx_unlock(&vnode_list_mtx);
1388 mtx_assert(&vnode_list_mtx, MA_OWNED);
1410#define VNLRU_FREEVNODES_SLOP 128
1441 mtx_assert(&vnode_list_mtx, MA_OWNED);
1442 if (freevnodes > freevnodes_old)
1443 slop = freevnodes - freevnodes_old;
1445 slop = freevnodes_old - freevnodes;
1447 return (freevnodes >= 0 ? freevnodes : 0);
1448 freevnodes_old = freevnodes;
1450 vd = DPCPU_ID_PTR((cpu), vd);
1451 freevnodes_old += vd->freevnodes;
1453 return (freevnodes_old >= 0 ? freevnodes_old : 0);
1459 u_long rfreevnodes, space;
1465 if (space < limit) {
1467 if (rfreevnodes > wantfreevnodes)
1468 space += rfreevnodes - wantfreevnodes;
1470 return (space < limit);
1476 long rfreevnodes, space;
1482 if (space < limit) {
1483 rfreevnodes = atomic_load_long(&freevnodes);
1484 if (rfreevnodes > wantfreevnodes)
1485 space += rfreevnodes - wantfreevnodes;
1487 return (space < limit);
1494 mtx_assert(&vnode_list_mtx, MA_OWNED);
1504 u_long rnumvnodes, rfreevnodes, target;
1505 unsigned long onumvnodes;
1506 int done, force, trigger, usevnodes;
1507 bool reclaim_nc_src, want_reread;
1510 SHUTDOWN_PRI_FIRST);
1513 want_reread =
false;
1516 mtx_lock(&vnode_list_mtx);
1517 rnumvnodes = atomic_load_long(&
numvnodes);
1521 want_reread =
false;
1531 rnumvnodes = atomic_load_long(&
numvnodes);
1540 if (
vstir && force == 0) {
1548 PVFS|PDROP,
"vlruwt",
hz);
1553 onumvnodes = rnumvnodes;
1562 usevnodes = rnumvnodes - rfreevnodes;
1564 usevnodes = rnumvnodes;
1575 trigger = vm_cnt.v_page_count * 2 / usevnodes;
1578 reclaim_nc_src = force >= 3;
1580 target = target / 10 + 1;
1581 done =
vlrureclaim(reclaim_nc_src, trigger, target);
1582 mtx_unlock(&vnode_list_mtx);
1584 uma_reclaim(UMA_RECLAIM_DRAIN);
1586 if (force == 0 || force == 1) {
1628 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
1629 VNASSERT(vp->v_holdcnt, vp,
1630 (
"vtryrecycle: Recycling vp %p without a reference.", vp));
1635 if (VOP_LOCK(vp, LK_EXCLUSIVE | LK_NOWAIT) != 0) {
1637 "%s: impossible to recycle, vp %p lock is already held",
1640 return (EWOULDBLOCK);
1648 "%s: impossible to recycle, cannot start the write for %p",
1660 if (vp->v_usecount) {
1665 "%s: impossible to recycle, %p is already referenced",
1669 if (!VN_IS_DOOMED(vp)) {
1670 counter_u64_add(recycles_free_count, 1);
1696static struct vnode * __noinline
1699 u_long rnumvnodes, rfreevnodes;
1701 mtx_lock(&vnode_list_mtx);
1702 rnumvnodes = atomic_load_long(&
numvnodes);
1724 if (mp == NULL || (mp->mnt_kern_flag & MNTK_SUSPEND) == 0) {
1735 rnumvnodes = atomic_fetchadd_long(&
numvnodes, 1) + 1;
1738 mtx_unlock(&vnode_list_mtx);
1739 return (uma_zalloc_smr(vnode_zone, M_WAITOK));
1742static struct vnode *
1749 rnumvnodes = atomic_fetchadd_long(&
numvnodes, 1) + 1;
1755 return (uma_zalloc_smr(vnode_zone, M_WAITOK));
1763 uma_zfree_smr(vnode_zone, vp);
1770getnewvnode(
const char *tag,
struct mount *mp,
struct vop_vector *vops,
1775 struct lock_object *lo;
1777 CTR3(KTR_VFS,
"%s: mp %p with tag %s", __func__, mp, tag);
1779 KASSERT(vops->registered,
1780 (
"%s: not registered vector op %p\n", __func__, vops));
1783 if (td->td_vp_reserved != NULL) {
1784 vp = td->td_vp_reserved;
1785 td->td_vp_reserved = NULL;
1804 lo = &vp->v_vnlock->lock_object;
1806 if (lo->lo_name != tag) {
1810 WITNESS_DESTROY(lo);
1811 WITNESS_INIT(lo, tag);
1817 vp->v_vnlock->lock_object.lo_flags |= LK_NOSHARE;
1821 KASSERT(vp->v_object == NULL, (
"stale v_object %p", vp));
1822 KASSERT(vp->v_lockf == NULL, (
"stale v_lockf %p", vp));
1823 KASSERT(vp->v_pollinfo == NULL, (
"stale v_pollinfo %p", vp));
1831 if (mp == NULL && vops != &dead_vnodeops)
1832 printf(
"NULL mp in getnewvnode(9), tag %s\n", tag);
1836 if (mp != NULL && (mp->mnt_flag & MNT_MULTILABEL) == 0)
1837 mac_vnode_associate_singlelabel(mp, vp);
1840 vp->v_bufobj.bo_bsize = mp->mnt_stat.f_iosize;
1849 vp->v_hash = (uintptr_t)vp >>
vnsz2log;
1861 MPASS(td->td_vp_reserved == NULL);
1862 td->td_vp_reserved =
vn_alloc(NULL);
1871 if (td->td_vp_reserved != NULL) {
1873 td->td_vp_reserved = NULL;
1877static void __noinline
1891 CTR2(KTR_VFS,
"%s: destroying the vnode %p", __func__, vp);
1898 VNASSERT(vp->v_data == NULL, vp, (
"cleaned vnode isn't"));
1899 VNPASS(vp->v_holdcnt == VHOLD_NO_SMR, vp);
1900 VNASSERT(vp->v_usecount == 0, vp, (
"Non-zero use count"));
1901 VNASSERT(vp->v_writecount == 0, vp, (
"Non-zero write count"));
1902 VNASSERT(bo->bo_numoutput == 0, vp, (
"Clean vnode has pending I/O's"));
1903 VNASSERT(bo->bo_clean.bv_cnt == 0, vp, (
"cleanbufcnt not 0"));
1904 VNASSERT(pctrie_is_empty(&bo->bo_clean.bv_root), vp,
1905 (
"clean blk trie not empty"));
1906 VNASSERT(bo->bo_dirty.bv_cnt == 0, vp, (
"dirtybufcnt not 0"));
1907 VNASSERT(pctrie_is_empty(&bo->bo_dirty.bv_root), vp,
1908 (
"dirty blk trie not empty"));
1909 VNASSERT(TAILQ_EMPTY(&vp->v_cache_dst), vp, (
"vp has namecache dst"));
1910 VNASSERT(LIST_EMPTY(&vp->v_cache_src), vp, (
"vp has namecache src"));
1911 VNASSERT(vp->v_cache_dd == NULL, vp, (
"vp has namecache for .."));
1912 VNASSERT(TAILQ_EMPTY(&vp->v_rl.rl_waiters), vp,
1913 (
"Dangling rangelock waiters"));
1914 VNASSERT((vp->v_iflag & (VI_DOINGINACT | VI_OWEINACT)) == 0, vp,
1915 (
"Leaked inactivation"));
1918 mac_vnode_destroy(vp);
1920 if (vp->v_pollinfo != NULL) {
1921 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1924 vp->v_pollinfo = NULL;
1926 vp->v_mountedhere = NULL;
1929 vp->v_fifoinfo = NULL;
1944 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp);
1953 VNASSERT(mp->mnt_nvnodelistsize > 0, vp,
1954 (
"bad mount point vnode list size"));
1955 TAILQ_REMOVE(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
1956 mp->mnt_nvnodelistsize--;
1965 KASSERT(vp->v_mount == NULL,
1966 (
"insmntque: vnode already on per mount vnode list"));
1967 VNASSERT(mp != NULL, vp, (
"Don't call insmntque(foo, NULL)"));
1968 if ((mp->mnt_kern_flag & MNTK_UNLOCKED_INSMNTQUE) == 0) {
1969 ASSERT_VOP_ELOCKED(vp,
"insmntque: non-locked vp");
1972 (
"%s: can't have MNTK_UNLOCKED_INSMNTQUE and cleanup",
1987 if (((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0 &&
1988 ((mp->mnt_kern_flag & MNTK_UNMOUNTF) != 0 ||
1989 mp->mnt_nvnodelistsize == 0)) &&
1990 (vp->v_vflag & VV_FORCEINSMQ) == 0) {
1995 vp->v_op = &dead_vnodeops;
2003 TAILQ_INSERT_TAIL(&mp->mnt_nvnodelist, vp, v_nmntvnodes);
2004 VNASSERT(mp->mnt_nvnodelistsize >= 0, vp,
2005 (
"neg mount point vnode list size"));
2006 mp->mnt_nvnodelistsize++;
2039 if (
flags & V_SAVE) {
2045 if (bo->bo_dirty.bv_cnt > 0) {
2048 error = BO_SYNC(bo, MNT_WAIT);
2049 }
while (error == ERELOOKUP);
2053 if (bo->bo_numoutput > 0 || bo->bo_dirty.bv_cnt > 0) {
2066 flags, bo, slpflag, slptimeo);
2067 if (error == 0 && !(
flags & V_CLEANONLY))
2069 flags, bo, slpflag, slptimeo);
2070 if (error != 0 && error != EAGAIN) {
2074 }
while (error != 0);
2083 if ((
flags & V_VMIO) == 0 && bo->bo_object != NULL) {
2085 vm_object_pip_wait_unlocked(bo->bo_object,
"bovlbx");
2088 }
while (bo->bo_numoutput > 0);
2094 if (bo->bo_object != NULL &&
2095 (
flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0) {
2096 VM_OBJECT_WLOCK(bo->bo_object);
2097 vm_object_page_remove(bo->bo_object, 0, 0, (
flags & V_SAVE) ?
2098 OBJPR_CLEANONLY : 0);
2099 VM_OBJECT_WUNLOCK(bo->bo_object);
2104 if ((
flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO |
2105 V_ALLOWCLEAN)) == 0 && (bo->bo_dirty.bv_cnt > 0 ||
2106 bo->bo_clean.bv_cnt > 0))
2107 panic(
"vinvalbuf: flush failed");
2108 if ((
flags & (V_ALT | V_NORMAL | V_CLEANONLY | V_VMIO)) == 0 &&
2109 bo->bo_dirty.bv_cnt > 0)
2110 panic(
"vinvalbuf: flush dirty failed");
2124 CTR3(KTR_VFS,
"%s: vp %p with flags %d", __func__, vp,
flags);
2125 ASSERT_VOP_LOCKED(vp,
"vinvalbuf");
2126 if (vp->v_object != NULL && vp->v_object->handle != vp)
2139 struct buf *bp, *nbp;
2144 ASSERT_BO_WLOCKED(bo);
2147 TAILQ_FOREACH_SAFE(bp, &bufv->bv_hd, b_bobufs, nbp) {
2155 if (((
flags & (V_NORMAL | V_ALT)) != (V_NORMAL | V_ALT)) &&
2156 (((
flags & V_NORMAL) && (bp->b_xflags & BX_ALTDATA) != 0) ||
2157 ((
flags & V_ALT) && (bp->b_xflags & BX_ALTDATA) == 0))) {
2161 lblkno = nbp->b_lblkno;
2162 xflags = nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN);
2165 error = BUF_TIMELOCK(bp,
2166 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, BO_LOCKPTR(bo),
2167 "flushbuf", slpflag, slptimeo);
2170 return (error != ENOLCK ? error : EAGAIN);
2172 KASSERT(bp->b_bufobj == bo,
2173 (
"bp %p wrong b_bufobj %p should be %p",
2174 bp, bp->b_bufobj, bo));
2181 if (((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI) &&
2184 bp->b_flags |= B_ASYNC;
2190 bp->b_flags |= (B_INVAL | B_RELBUF);
2191 bp->b_flags &= ~B_ASYNC;
2197 if (nbp == NULL || (nbp->b_xflags & (BX_VNDIRTY | BX_VNCLEAN))
2205bnoreuselist(
struct bufv *bufv,
struct bufobj *bo, daddr_t startn, daddr_t endn)
2211 ASSERT_BO_LOCKED(bo);
2213 for (lblkno = startn;;) {
2215 bp = BUF_PCTRIE_LOOKUP_GE(&bufv->bv_root, lblkno);
2216 if (bp == NULL || bp->b_lblkno >= endn ||
2217 bp->b_lblkno < startn)
2219 error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL |
2220 LK_INTERLOCK, BO_LOCKPTR(bo),
"brlsfl", 0, 0);
2223 if (error == ENOLCK)
2227 KASSERT(bp->b_bufobj == bo,
2228 (
"bp %p wrong b_bufobj %p should be %p",
2229 bp, bp->b_bufobj, bo));
2230 lblkno = bp->b_lblkno + 1;
2231 if ((bp->b_flags & B_MANAGED) == 0)
2233 bp->b_flags |= B_RELBUF;
2240 if ((bp->b_flags & B_VMIO) != 0)
2241 bp->b_flags |= B_NOREUSE;
2256 struct buf *bp, *nbp;
2260 CTR4(KTR_VFS,
"%s: vp %p with block %d:%ju", __func__,
2261 vp, blksize, (uintmax_t)length);
2266 startlbn = howmany(length, blksize);
2268 ASSERT_VOP_LOCKED(vp,
"vtruncbuf");
2279 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2280 if (bp->b_lblkno > 0)
2287 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
2288 BO_LOCKPTR(bo)) == ENOLCK)
2289 goto restart_unlocked;
2291 VNASSERT((bp->b_flags & B_DELWRI), vp,
2292 (
"buf(%p) on dirty queue without DELWRI", bp));
2303 vnode_pager_setsize(vp, length);
2319 ASSERT_VOP_LOCKED(vp,
"v_inval_buf_range");
2321 start = blksize * startlbn;
2322 end = blksize * endlbn;
2326 MPASS(blksize == bo->bo_bsize);
2337 daddr_t startlbn, daddr_t endlbn)
2339 struct buf *bp, *nbp;
2342 ASSERT_VOP_LOCKED(vp,
"v_inval_buf_range_locked");
2343 ASSERT_BO_LOCKED(bo);
2347 TAILQ_FOREACH_SAFE(bp, &bo->bo_clean.bv_hd, b_bobufs, nbp) {
2348 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn)
2351 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
2352 BO_LOCKPTR(bo)) == ENOLCK) {
2358 bp->b_flags |= B_INVAL | B_RELBUF;
2359 bp->b_flags &= ~B_ASYNC;
2365 (((nbp->b_xflags & BX_VNCLEAN) == 0) ||
2367 (nbp->b_flags & B_DELWRI) != 0))
2371 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2372 if (bp->b_lblkno < startlbn || bp->b_lblkno >= endlbn)
2375 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK,
2376 BO_LOCKPTR(bo)) == ENOLCK) {
2381 bp->b_flags |= B_INVAL | B_RELBUF;
2382 bp->b_flags &= ~B_ASYNC;
2388 (((nbp->b_xflags & BX_VNDIRTY) == 0) ||
2389 (nbp->b_vp != vp) ||
2390 (nbp->b_flags & B_DELWRI) == 0))
2403 flags = bp->b_xflags;
2405 KASSERT(bp->b_bufobj != NULL, (
"No b_bufobj %p", bp));
2406 ASSERT_BO_WLOCKED(bp->b_bufobj);
2407 KASSERT((
flags & (BX_VNDIRTY | BX_VNCLEAN)) != 0 &&
2408 (
flags & (BX_VNDIRTY | BX_VNCLEAN)) != (BX_VNDIRTY | BX_VNCLEAN),
2409 (
"%s: buffer %p has invalid queue state", __func__, bp));
2411 if ((
flags & BX_VNDIRTY) != 0)
2412 bv = &bp->b_bufobj->bo_dirty;
2414 bv = &bp->b_bufobj->bo_clean;
2415 BUF_PCTRIE_REMOVE(&bv->bv_root, bp->b_lblkno);
2416 TAILQ_REMOVE(&bv->bv_hd, bp, b_bobufs);
2418 bp->b_xflags &= ~(BX_VNDIRTY | BX_VNCLEAN);
2433 ASSERT_BO_WLOCKED(bo);
2434 KASSERT((bo->bo_flag & BO_NOBUFS) == 0,
2435 (
"buf_vlist_add: bo %p does not allow bufs", bo));
2436 KASSERT((xflags & BX_VNDIRTY) == 0 || (bo->bo_flag & BO_DEAD) == 0,
2437 (
"dead bo %p", bo));
2438 KASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0,
2439 (
"buf_vlist_add: Buf %p has existing xflags %d", bp, bp->b_xflags));
2440 bp->b_xflags |= xflags;
2441 if (xflags & BX_VNDIRTY)
2451 if (bv->bv_cnt == 0 ||
2452 bp->b_lblkno > TAILQ_LAST(&bv->bv_hd, buflists)->b_lblkno)
2453 TAILQ_INSERT_TAIL(&bv->bv_hd, bp, b_bobufs);
2454 else if ((n = BUF_PCTRIE_LOOKUP_LE(&bv->bv_root, bp->b_lblkno)) == NULL)
2455 TAILQ_INSERT_HEAD(&bv->bv_hd, bp, b_bobufs);
2457 TAILQ_INSERT_AFTER(&bv->bv_hd, n, bp, b_bobufs);
2458 error = BUF_PCTRIE_INSERT(&bv->bv_root, bp);
2460 panic(
"buf_vlist_add: Preallocated nodes insufficient.");
2472 ASSERT_BO_LOCKED(bo);
2473 bp = BUF_PCTRIE_LOOKUP(&bo->bo_clean.bv_root, lblkno);
2476 return (BUF_PCTRIE_LOOKUP(&bo->bo_dirty.bv_root, lblkno));
2490 ASSERT_BO_UNLOCKED(bo);
2491 bp = BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_clean.bv_root, lblkno);
2494 return (BUF_PCTRIE_LOOKUP_UNLOCKED(&bo->bo_dirty.bv_root, lblkno));
2506 ASSERT_BO_WLOCKED(bo);
2507 VNASSERT(bp->b_vp == NULL, bp->b_vp, (
"bgetvp: not free"));
2509 CTR3(KTR_BUF,
"bgetvp(%p) vp %p flags %X", bp, vp, bp->b_flags);
2510 VNASSERT((bp->b_xflags & (BX_VNDIRTY|BX_VNCLEAN)) == 0, vp,
2511 (
"bgetvp: bp already attached! %p", bp));
2531 CTR3(KTR_BUF,
"brelvp(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2532 KASSERT(bp->b_vp != NULL, (
"brelvp: NULL"));
2541 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2542 bo->bo_flag &= ~BO_ONWORKLST;
2543 mtx_lock(&sync_mtx);
2544 LIST_REMOVE(bo, bo_synclist);
2546 mtx_unlock(&sync_mtx);
2549 bp->b_bufobj = NULL;
2562 ASSERT_BO_WLOCKED(bo);
2564 mtx_lock(&sync_mtx);
2565 if (bo->bo_flag & BO_ONWORKLST)
2566 LIST_REMOVE(bo, bo_synclist);
2568 bo->bo_flag |= BO_ONWORKLST;
2572 if (delay > syncer_maxdelay - 2)
2573 delay = syncer_maxdelay - 2;
2574 slot = (syncer_delayno + delay) & syncer_mask;
2576 LIST_INSERT_HEAD(&syncer_workitem_pending[slot], bo, bo_synclist);
2577 mtx_unlock(&sync_mtx);
2585 mtx_lock(&sync_mtx);
2587 mtx_unlock(&sync_mtx);
2588 error = SYSCTL_OUT(req, &len,
sizeof(len));
2593 CTLTYPE_INT | CTLFLAG_MPSAFE| CTLFLAG_RD, NULL, 0,
2606sync_vnode(
struct synclist *slp,
struct bufobj **bo,
struct thread *td)
2611 *bo = LIST_FIRST(slp);
2615 if (VOP_ISLOCKED(vp) != 0 || VI_TRYLOCK(vp) == 0)
2624 mtx_unlock(&sync_mtx);
2628 mtx_lock(&sync_mtx);
2629 return (*bo == LIST_FIRST(slp));
2631 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2632 (void) VOP_FSYNC(vp, MNT_LAZY, td);
2636 if (((*bo)->bo_flag & BO_ONWORKLST) != 0) {
2647 mtx_lock(&sync_mtx);
2659 struct synclist *next, *slp;
2662 struct thread *td = curthread;
2664 int net_worklist_len;
2665 int syncer_final_iter;
2669 syncer_final_iter = 0;
2672 td->td_pflags |= TDP_NORUNNINGBUF;
2674 EVENTHANDLER_REGISTER(shutdown_pre_sync,
syncer_shutdown, td->td_proc,
2677 mtx_lock(&sync_mtx);
2680 syncer_final_iter == 0) {
2681 mtx_unlock(&sync_mtx);
2683 mtx_lock(&sync_mtx);
2689 printf(
"\nSyncing disks, vnodes remaining... ");
2692 printf(
"%d ", net_worklist_len);
2703 slp = &syncer_workitem_pending[syncer_delayno];
2704 syncer_delayno += 1;
2705 if (syncer_delayno == syncer_maxdelay)
2707 next = &syncer_workitem_pending[syncer_delayno];
2715 net_worklist_len == 0 &&
2716 last_work_seen == syncer_delayno) {
2730 last_work_seen = syncer_delayno;
2733 while (!LIST_EMPTY(slp)) {
2736 LIST_REMOVE(bo, bo_synclist);
2737 LIST_INSERT_HEAD(next, bo, bo_synclist);
2746 mtx_unlock(&sync_mtx);
2747 wdog_kern_pat(WD_LASTVAL);
2748 mtx_lock(&sync_mtx);
2752 syncer_final_iter--;
2786 cv_timedwait(&sync_wakeup, &sync_mtx,
2789 cv_timedwait(&sync_wakeup, &sync_mtx,
hz);
2803 mtx_lock(&sync_mtx);
2804 if (rushjob < syncdelay / 2) {
2806 stat_rush_requests += 1;
2809 mtx_unlock(&sync_mtx);
2810 cv_broadcast(&sync_wakeup);
2822 if (howto & RB_NOSYNC)
2824 mtx_lock(&sync_mtx);
2827 mtx_unlock(&sync_mtx);
2828 cv_broadcast(&sync_wakeup);
2843 mtx_lock(&sync_mtx);
2846 mtx_unlock(&sync_mtx);
2847 cv_broadcast(&sync_wakeup);
2867 KASSERT((bp->b_flags & B_PAGING) == 0,
2868 (
"%s: cannot reassign paging buffer %p", __func__, bp));
2870 CTR3(KTR_BUF,
"reassignbuf(%p) vp %p flags %X",
2871 bp, bp->b_vp, bp->b_flags);
2880 if (bp->b_flags & B_DELWRI) {
2881 if ((bo->bo_flag & BO_ONWORKLST) == 0) {
2882 switch (vp->v_type) {
2898 if ((bo->bo_flag & BO_ONWORKLST) && bo->bo_dirty.bv_cnt == 0) {
2899 mtx_lock(&sync_mtx);
2900 LIST_REMOVE(bo, bo_synclist);
2902 mtx_unlock(&sync_mtx);
2903 bo->bo_flag &= ~BO_ONWORKLST;
2908 bp = TAILQ_FIRST(&bv->bv_hd);
2909 KASSERT(bp == NULL || bp->b_bufobj == bo,
2910 (
"bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2911 bp = TAILQ_LAST(&bv->bv_hd, buflists);
2912 KASSERT(bp == NULL || bp->b_bufobj == bo,
2913 (
"bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2915 bp = TAILQ_FIRST(&bv->bv_hd);
2916 KASSERT(bp == NULL || bp->b_bufobj == bo,
2917 (
"bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2918 bp = TAILQ_LAST(&bv->bv_hd, buflists);
2919 KASSERT(bp == NULL || bp->b_bufobj == bo,
2920 (
"bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
2929 VNASSERT(vp->v_type == VNON && vp->v_data == NULL && vp->v_iflag == 0,
2930 vp, (
"%s called for an initialized vnode", __FUNCTION__));
2931 ASSERT_VI_UNLOCKED(vp, __FUNCTION__);
2933 refcount_init(&vp->v_holdcnt, 1);
2934 refcount_init(&vp->v_usecount, 1);
2958 VFS_SMR_ASSERT_ENTERED();
2960 if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
2976 if (refcount_acquire_if_not_zero(&vp->v_usecount)) {
2997 __assert_unreachable();
3015 if ((
flags & LK_INTERLOCK) != 0)
3016 ASSERT_VI_LOCKED(vp, __func__);
3018 ASSERT_VI_UNLOCKED(vp, __func__);
3019 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp);
3020 VNPASS(vp->v_holdcnt > 0, vp);
3021 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp);
3023 error = vn_lock(vp,
flags);
3024 if (__predict_false(error != 0)) {
3026 CTR2(KTR_VFS,
"%s: impossible to lock vnode %p", __func__,
3040 VNPASS(vs == VGET_HOLDCNT || vs == VGET_USECOUNT, vp);
3041 VNPASS(vp->v_holdcnt > 0, vp);
3042 VNPASS(vs == VGET_HOLDCNT || vp->v_usecount > 0, vp);
3044 if (vs == VGET_USECOUNT)
3052 old = atomic_fetchadd_int(&vp->v_usecount, 1);
3053 VNASSERT(old >= 0, vp, (
"%s: wrong use count %d", __func__, old));
3056 old = atomic_fetchadd_int(&vp->v_holdcnt, -1);
3057 VNASSERT(old > 1, vp, (
"%s: wrong hold count %d", __func__, old));
3059 refcount_release(&vp->v_holdcnt);
3069 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3078 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3080 int old = atomic_fetchadd_int(&vp->v_usecount, 1);
3081 VNASSERT(old > 0, vp, (
"%s: wrong use count %d", __func__, old));
3083 refcount_acquire(&vp->v_usecount);
3092 VNASSERT(vp->v_holdcnt > 0, vp, (
"%s: vnode not held", __func__));
3094 if ((vp->v_mflag & VMP_LAZYLIST) != 0)
3099 if (VN_IS_DOOMED(vp))
3102 mtx_lock(&mp->mnt_listmtx);
3103 if ((vp->v_mflag & VMP_LAZYLIST) == 0) {
3104 vp->v_mflag |= VMP_LAZYLIST;
3105 TAILQ_INSERT_TAIL(&mp->mnt_lazyvnodelist, vp, v_lazylist);
3106 mp->mnt_lazyvnodelistsize++;
3108 mtx_unlock(&mp->mnt_listmtx);
3116 ASSERT_VI_LOCKED(vp, __func__);
3117 VNPASS(!VN_IS_DOOMED(vp), vp);
3120 mtx_lock(&mp->mnt_listmtx);
3121 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp);
3128 if (vp->v_holdcnt == 0) {
3129 vp->v_mflag &= ~VMP_LAZYLIST;
3130 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist);
3131 mp->mnt_lazyvnodelistsize--;
3133 mtx_unlock(&mp->mnt_listmtx);
3145 ASSERT_VOP_ELOCKED(vp, __func__);
3146 ASSERT_VI_LOCKED(vp, __func__);
3147 VNPASS(!VN_IS_DOOMED(vp), vp);
3149 if (vp->v_mflag & VMP_LAZYLIST) {
3151 mtx_lock(&mp->mnt_listmtx);
3152 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp);
3153 vp->v_mflag &= ~VMP_LAZYLIST;
3154 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, vp, v_lazylist);
3155 mp->mnt_lazyvnodelistsize--;
3156 mtx_unlock(&mp->mnt_listmtx);
3164 ASSERT_VI_LOCKED(vp, __func__);
3165 VNASSERT(vp->v_holdcnt > 0, vp,
3166 (
"%s: vnode without hold count", __func__));
3167 if (VN_IS_DOOMED(vp)) {
3171 if (vp->v_iflag & VI_DEFINACT) {
3172 VNASSERT(vp->v_holdcnt > 1, vp, (
"lost hold count"));
3176 if (vp->v_usecount > 0) {
3177 vp->v_iflag &= ~VI_OWEINACT;
3182 vp->v_iflag |= VI_DEFINACT;
3184 counter_u64_add(deferred_inact, 1);
3192 if ((vp->v_iflag & VI_OWEINACT) == 0) {
3223 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3224 VNPASS(vp->v_holdcnt > 0, vp);
3232 if (vp->v_usecount > 0)
3239 if (VN_IS_DOOMED(vp))
3242 if (__predict_true(VOP_NEED_INACTIVE(vp) == 0))
3245 if (vp->v_iflag & VI_DOINGINACT)
3254 vp->v_iflag |= VI_OWEINACT;
3255 want_unlock =
false;
3259 switch (VOP_ISLOCKED(vp)) {
3265 error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
3280 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
3281 error = VOP_LOCK(vp, LK_UPGRADE | LK_INTERLOCK |
3287 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE) {
3288 error = VOP_LOCK(vp, LK_TRYUPGRADE | LK_INTERLOCK);
3295 VNASSERT((vp->v_vflag & VV_UNREF) == 0, vp,
3296 (
"recursive vunref"));
3297 vp->v_vflag |= VV_UNREF;
3303 if (error != ERELOOKUP || !want_unlock)
3305 VOP_LOCK(vp, LK_EXCLUSIVE);
3308 vp->v_vflag &= ~VV_UNREF;
3337 ASSERT_VI_UNLOCKED(vp, __func__);
3338 if (!refcount_release(&vp->v_usecount))
3351 ASSERT_VOP_LOCKED(vp, __func__);
3352 ASSERT_VI_UNLOCKED(vp, __func__);
3353 if (!refcount_release(&vp->v_usecount)) {
3368 ASSERT_VOP_LOCKED(vp, __func__);
3369 ASSERT_VI_UNLOCKED(vp, __func__);
3370 if (!refcount_release(&vp->v_usecount))
3380 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3381 old = atomic_fetchadd_int(&vp->v_holdcnt, 1);
3382 VNASSERT(old >= 0 && (old & VHOLD_ALL_FLAGS) == 0, vp,
3383 (
"%s: wrong hold count %d", __func__, old));
3392 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3394 int old = atomic_fetchadd_int(&vp->v_holdcnt, 1);
3395 VNASSERT(old > 0 && (old & VHOLD_ALL_FLAGS) == 0, vp,
3396 (
"%s: wrong hold count %d", __func__, old));
3398 atomic_add_int(&vp->v_holdcnt, 1);
3426 VFS_SMR_ASSERT_ENTERED();
3428 count = atomic_load_int(&vp->v_holdcnt);
3430 if (
count & VHOLD_NO_SMR) {
3431 VNASSERT((
count & ~VHOLD_NO_SMR) == 0, vp,
3432 (
"non-zero hold count with flags %d\n",
count));
3435 VNASSERT(
count >= 0, vp, (
"invalid hold count %d\n",
count));
3436 if (atomic_fcmpset_int(&vp->v_holdcnt, &
count,
count + 1)) {
3465 mtx_assert(&vnode_list_mtx, MA_OWNED);
3467 count = atomic_load_int(&vp->v_holdcnt);
3469 if (
count & VHOLD_NO_SMR) {
3470 VNASSERT((
count & ~VHOLD_NO_SMR) == 0, vp,
3471 (
"non-zero hold count with flags %d\n",
count));
3474 VNASSERT(
count >= 0, vp, (
"invalid hold count %d\n",
count));
3478 if (atomic_fcmpset_int(&vp->v_holdcnt, &
count,
count + 1)) {
3485static void __noinline
3491 mtx_assert(&vd->lock, MA_OWNED);
3492 MPASS(curthread->td_pinned > 0);
3495 mtx_lock(&vnode_list_mtx);
3497 freevnodes += vd->freevnodes;
3500 TAILQ_REMOVE(&vnode_list, vp, v_vnodelist);
3501 TAILQ_INSERT_TAIL(&vnode_list, vp, v_vnodelist);
3502 MPASS(vp->v_dbatchcpu != NOCPU);
3503 vp->v_dbatchcpu = NOCPU;
3505 mtx_unlock(&vnode_list_mtx);
3507 bzero(vd->tab,
sizeof(vd->tab));
3517 ASSERT_VI_LOCKED(vp, __func__);
3518 VNASSERT(!VN_IS_DOOMED(vp), vp,
3519 (
"%s: deferring requeue of a doomed vnode", __func__));
3521 if (vp->v_dbatchcpu != NOCPU) {
3528 mtx_lock(&vd->lock);
3530 MPASS(vd->tab[vd->index] == NULL);
3535 vp->v_dbatchcpu = curcpu;
3536 vd->tab[vd->index] = vp;
3541 mtx_unlock(&vd->lock);
3557 VNASSERT(vp->v_type == VBAD || vp->v_type == VNON, vp,
3558 (
"%s: called for a used vnode\n", __func__));
3560 cpu = vp->v_dbatchcpu;
3564 vd = DPCPU_ID_PTR(cpu, vd);
3565 mtx_lock(&vd->lock);
3566 for (i = 0; i < vd->index; i++) {
3567 if (vd->tab[i] != vp)
3569 vp->v_dbatchcpu = NOCPU;
3571 vd->tab[i] = vd->tab[vd->index];
3572 vd->tab[vd->index] = NULL;
3575 mtx_unlock(&vd->lock);
3579 MPASS(vp->v_dbatchcpu == NOCPU);
3591static void __noinline
3595 ASSERT_VI_LOCKED(vp, __func__);
3596 VNPASS(VN_IS_DOOMED(vp), vp);
3603 if (__predict_false(!atomic_cmpset_int(&vp->v_holdcnt, 0, VHOLD_NO_SMR))) {
3622 ASSERT_VI_UNLOCKED(vp, __func__);
3623 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3624 if (refcount_release_if_not_last(&vp->v_holdcnt))
3630static void __always_inline
3634 ASSERT_VI_LOCKED(vp, __func__);
3635 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3636 if (!refcount_release(&vp->v_holdcnt)) {
3640 VNPASS((vp->v_iflag & VI_OWEINACT) == 0, vp);
3641 VNPASS((vp->v_iflag & VI_DEFINACT) == 0, vp);
3642 if (VN_IS_DOOMED(vp)) {
3648 if (vp->v_mflag & VMP_LAZYLIST) {
3699 struct vm_object *obj;
3702 ASSERT_VOP_ELOCKED(vp,
"vinactive");
3703 ASSERT_VI_LOCKED(vp,
"vinactive");
3704 VNASSERT((vp->v_iflag & VI_DOINGINACT) == 0, vp,
3705 (
"vinactive: recursed on VI_DOINGINACT"));
3706 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3707 vp->v_iflag |= VI_DOINGINACT;
3708 vp->v_iflag &= ~VI_OWEINACT;
3720 if ((obj = vp->v_object) != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
3721 vm_object_mightbedirty(obj)) {
3722 VM_OBJECT_WLOCK(obj);
3723 vm_object_page_clean(obj, 0, 0, 0);
3724 VM_OBJECT_WUNLOCK(obj);
3726 error = VOP_INACTIVE(vp);
3728 VNASSERT(vp->v_iflag & VI_DOINGINACT, vp,
3729 (
"vinactive: lost VI_DOINGINACT"));
3730 vp->v_iflag &= ~VI_DOINGINACT;
3738 ASSERT_VOP_ELOCKED(vp,
"vinactive");
3739 ASSERT_VI_LOCKED(vp,
"vinactive");
3740 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3742 if ((vp->v_iflag & VI_OWEINACT) == 0)
3744 if (vp->v_iflag & VI_DOINGINACT)
3746 if (vp->v_usecount > 0) {
3747 vp->v_iflag &= ~VI_OWEINACT;
3774static int busyprt = 0;
3775SYSCTL_INT(_debug, OID_AUTO, busyprt, CTLFLAG_RW, &busyprt, 0,
"Print out busy vnodes");
3781 struct vnode *vp, *mvp, *rootvp = NULL;
3783 int busy = 0, error;
3785 CTR4(KTR_VFS,
"%s: mp %p with rootrefs %d and flags %d", __func__, mp,
3788 KASSERT((
flags & (SKIPSYSTEM | WRITECLOSE)) == 0,
3789 (
"vflush: bad args"));
3794 if ((error = VFS_ROOT(mp, LK_EXCLUSIVE, &rootvp)) != 0) {
3795 CTR2(KTR_VFS,
"%s: vfs_root lookup failed with %d",
3802 MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
3804 error = vn_lock(vp, LK_INTERLOCK | LK_EXCLUSIVE);
3807 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
3813 if ((
flags & SKIPSYSTEM) && (vp->v_vflag & VV_SYSTEM)) {
3823 if (
flags & WRITECLOSE) {
3824 if (vp->v_object != NULL) {
3825 VM_OBJECT_WLOCK(vp->v_object);
3826 vm_object_page_clean(vp->v_object, 0, 0, 0);
3827 VM_OBJECT_WUNLOCK(vp->v_object);
3830 error = VOP_FSYNC(vp, MNT_WAIT, td);
3831 }
while (error == ERELOOKUP);
3835 MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
3838 error = VOP_GETATTR(vp, &vattr, td->td_ucred);
3841 if ((vp->v_type == VNON ||
3842 (error == 0 && vattr.va_nlink > 0)) &&
3843 (vp->v_writecount <= 0 || vp->v_type != VREG)) {
3856 if (vp->v_usecount == 0 || (
flags & FORCECLOSE)) {
3868 if (rootrefs > 0 && (
flags & FORCECLOSE) == 0) {
3874 KASSERT(
busy > 0, (
"vflush: not busy"));
3875 VNASSERT(rootvp->v_usecount >= rootrefs, rootvp,
3876 (
"vflush: usecount %d < rootrefs %d",
3877 rootvp->v_usecount, rootrefs));
3878 if (
busy == 1 && rootvp->v_usecount == rootrefs) {
3879 VOP_LOCK(rootvp, LK_EXCLUSIVE|LK_INTERLOCK);
3887 CTR2(KTR_VFS,
"%s: failing as %d vnodes are busy", __func__,
3891 for (; rootrefs > 0; rootrefs--)
3918 ASSERT_VOP_ELOCKED(vp, __func__);
3919 ASSERT_VI_LOCKED(vp, __func__);
3920 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
3922 if (vp->v_usecount == 0) {
3948 struct mount_upper_node *ump;
3950 mp = atomic_load_ptr(&vp->v_mount);
3953 if (TAILQ_EMPTY(&mp->mnt_notify))
3957 mp->mnt_upper_pending++;
3958 KASSERT(mp->mnt_upper_pending > 0,
3959 (
"%s: mnt_upper_pending %d", __func__, mp->mnt_upper_pending));
3960 TAILQ_FOREACH(ump, &mp->mnt_notify, mnt_upper_link) {
3963 case VFS_NOTIFY_UPPER_RECLAIM:
3964 VFS_RECLAIM_LOWERVP(ump->mp, vp);
3966 case VFS_NOTIFY_UPPER_UNLINK:
3967 VFS_UNLINK_LOWERVP(ump->mp, vp);
3972 mp->mnt_upper_pending--;
3973 if ((mp->mnt_kern_flag & MNTK_UPPER_WAITER) != 0 &&
3974 mp->mnt_upper_pending == 0) {
3975 mp->mnt_kern_flag &= ~MNTK_UPPER_WAITER;
3990 bool active, doinginact, oweinact;
3992 ASSERT_VOP_ELOCKED(vp,
"vgonel");
3993 ASSERT_VI_LOCKED(vp,
"vgonel");
3994 VNASSERT(vp->v_holdcnt, vp,
3995 (
"vgonel: vp %p has no reference.", vp));
3996 CTR2(KTR_VFS,
"%s: vp %p", __func__, vp);
4002 if (VN_IS_DOOMED(vp))
4019 active = vp->v_usecount > 0;
4020 oweinact = (vp->v_iflag & VI_OWEINACT) != 0;
4021 doinginact = (vp->v_iflag & VI_DOINGINACT) != 0;
4026 if (vp->v_iflag & VI_DEFINACT) {
4027 VNASSERT(vp->v_holdcnt > 1, vp, (
"lost hold count"));
4028 vp->v_iflag &= ~VI_DEFINACT;
4031 VNASSERT(vp->v_holdcnt > 0, vp, (
"vnode without hold count"));
4042 VOP_CLOSE(vp, FNONBLOCK, NOCRED, td);
4045 if (oweinact || active) {
4048 oweinact = (vp->v_iflag & VI_OWEINACT) != 0;
4053 if (vp->v_type == VSOCK)
4061 if (!TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd))
4068 BO_LOCK(&vp->v_bufobj);
4069 KASSERT(TAILQ_EMPTY(&vp->v_bufobj.bo_dirty.bv_hd) &&
4070 vp->v_bufobj.bo_dirty.bv_cnt == 0 &&
4071 TAILQ_EMPTY(&vp->v_bufobj.bo_clean.bv_hd) &&
4072 vp->v_bufobj.bo_clean.bv_cnt == 0,
4073 (
"vp %p bufobj not invalidated", vp));
4080 object = vp->v_bufobj.bo_object;
4082 vp->v_bufobj.bo_flag |= BO_DEAD;
4083 BO_UNLOCK(&vp->v_bufobj);
4091 if (
object != NULL && object->type == OBJT_VNODE &&
4092 object->handle == vp)
4093 vnode_destroy_vobject(vp);
4098 if (VOP_RECLAIM(vp))
4099 panic(
"vgone: cannot reclaim");
4102 VNASSERT(vp->v_object == NULL, vp,
4103 (
"vop_reclaim left v_object vp=%p", vp));
4107 (void)VOP_ADVLOCKPURGE(vp);
4118 vp->v_vnlock = &vp->v_lock;
4119 vp->v_op = &dead_vnodeops;
4126static const char *
const typename[] =
4127{
"VNON",
"VREG",
"VDIR",
"VBLK",
"VCHR",
"VLNK",
"VSOCK",
"VFIFO",
"VBAD",
4131 "new hold count flag not added to vn_printf");
4137 char buf[256], buf2[16];
4145 printf(
"%p: ", (
void *)vp);
4146 printf(
"type %s\n",
typename[vp->v_type]);
4147 holdcnt = atomic_load_int(&vp->v_holdcnt);
4148 printf(
" usecount %d, writecount %d, refcount %d seqc users %d",
4149 vp->v_usecount, vp->v_writecount, holdcnt & ~VHOLD_ALL_FLAGS,
4151 switch (vp->v_type) {
4153 printf(
" mountedhere %p\n", vp->v_mountedhere);
4156 printf(
" rdev %p\n", vp->v_rdev);
4159 printf(
" socket %p\n", vp->v_unpcb);
4162 printf(
" fifoinfo %p\n", vp->v_fifoinfo);
4170 if (holdcnt & VHOLD_NO_SMR)
4171 strlcat(
buf,
"|VHOLD_NO_SMR",
sizeof(
buf));
4172 printf(
" hold count flags (%s)\n",
buf + 1);
4176 irflag = vn_irflag_read(vp);
4177 if (irflag & VIRF_DOOMED)
4178 strlcat(
buf,
"|VIRF_DOOMED",
sizeof(
buf));
4179 if (irflag & VIRF_PGREAD)
4180 strlcat(
buf,
"|VIRF_PGREAD",
sizeof(
buf));
4181 if (irflag & VIRF_MOUNTPOINT)
4182 strlcat(
buf,
"|VIRF_MOUNTPOINT",
sizeof(
buf));
4183 if (irflag & VIRF_TEXT_REF)
4184 strlcat(
buf,
"|VIRF_TEXT_REF",
sizeof(
buf));
4185 flags = irflag & ~(VIRF_DOOMED | VIRF_PGREAD | VIRF_MOUNTPOINT | VIRF_TEXT_REF);
4188 strlcat(
buf, buf2,
sizeof(
buf));
4190 if (vp->v_vflag & VV_ROOT)
4191 strlcat(
buf,
"|VV_ROOT",
sizeof(
buf));
4192 if (vp->v_vflag & VV_ISTTY)
4193 strlcat(
buf,
"|VV_ISTTY",
sizeof(
buf));
4194 if (vp->v_vflag & VV_NOSYNC)
4195 strlcat(
buf,
"|VV_NOSYNC",
sizeof(
buf));
4196 if (vp->v_vflag & VV_ETERNALDEV)
4197 strlcat(
buf,
"|VV_ETERNALDEV",
sizeof(
buf));
4198 if (vp->v_vflag & VV_CACHEDLABEL)
4199 strlcat(
buf,
"|VV_CACHEDLABEL",
sizeof(
buf));
4200 if (vp->v_vflag & VV_VMSIZEVNLOCK)
4201 strlcat(
buf,
"|VV_VMSIZEVNLOCK",
sizeof(
buf));
4202 if (vp->v_vflag & VV_COPYONWRITE)
4203 strlcat(
buf,
"|VV_COPYONWRITE",
sizeof(
buf));
4204 if (vp->v_vflag & VV_SYSTEM)
4205 strlcat(
buf,
"|VV_SYSTEM",
sizeof(
buf));
4206 if (vp->v_vflag & VV_PROCDEP)
4207 strlcat(
buf,
"|VV_PROCDEP",
sizeof(
buf));
4208 if (vp->v_vflag & VV_DELETED)
4209 strlcat(
buf,
"|VV_DELETED",
sizeof(
buf));
4210 if (vp->v_vflag & VV_MD)
4211 strlcat(
buf,
"|VV_MD",
sizeof(
buf));
4212 if (vp->v_vflag & VV_FORCEINSMQ)
4213 strlcat(
buf,
"|VV_FORCEINSMQ",
sizeof(
buf));
4214 if (vp->v_vflag & VV_READLINK)
4215 strlcat(
buf,
"|VV_READLINK",
sizeof(
buf));
4216 flags = vp->v_vflag & ~(VV_ROOT | VV_ISTTY | VV_NOSYNC | VV_ETERNALDEV |
4217 VV_CACHEDLABEL | VV_VMSIZEVNLOCK | VV_COPYONWRITE | VV_SYSTEM |
4218 VV_PROCDEP | VV_DELETED | VV_MD | VV_FORCEINSMQ | VV_READLINK);
4221 strlcat(
buf, buf2,
sizeof(
buf));
4223 if (vp->v_iflag & VI_MOUNT)
4224 strlcat(
buf,
"|VI_MOUNT",
sizeof(
buf));
4225 if (vp->v_iflag & VI_DOINGINACT)
4226 strlcat(
buf,
"|VI_DOINGINACT",
sizeof(
buf));
4227 if (vp->v_iflag & VI_OWEINACT)
4228 strlcat(
buf,
"|VI_OWEINACT",
sizeof(
buf));
4229 if (vp->v_iflag & VI_DEFINACT)
4230 strlcat(
buf,
"|VI_DEFINACT",
sizeof(
buf));
4231 if (vp->v_iflag & VI_FOPENING)
4232 strlcat(
buf,
"|VI_FOPENING",
sizeof(
buf));
4233 flags = vp->v_iflag & ~(VI_MOUNT | VI_DOINGINACT |
4234 VI_OWEINACT | VI_DEFINACT | VI_FOPENING);
4237 strlcat(
buf, buf2,
sizeof(
buf));
4239 if (vp->v_mflag & VMP_LAZYLIST)
4240 strlcat(
buf,
"|VMP_LAZYLIST",
sizeof(
buf));
4241 flags = vp->v_mflag & ~(VMP_LAZYLIST);
4244 strlcat(
buf, buf2,
sizeof(
buf));
4247 if (mtx_owned(VI_MTX(vp)))
4250 if (vp->v_object != NULL)
4251 printf(
" v_object %p ref %d pages %d "
4252 "cleanbuf %d dirtybuf %d\n",
4253 vp->v_object, vp->v_object->ref_count,
4254 vp->v_object->resident_page_count,
4255 vp->v_bufobj.bo_clean.bv_cnt,
4256 vp->v_bufobj.bo_dirty.bv_cnt);
4259 if (vp->v_data != NULL)
4268DB_SHOW_COMMAND(lockedvnods, lockedvnodes)
4279 db_printf(
"Locked vnodes\n");
4280 TAILQ_FOREACH(mp, &
mountlist, mnt_list) {
4281 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4282 if (vp->v_type != VMARKER && VOP_ISLOCKED(vp))
4291DB_SHOW_COMMAND(vnode, db_show_vnode)
4297 vp = (
struct vnode *)
addr;
4304DB_SHOW_COMMAND(mount, db_show_mount)
4316 TAILQ_FOREACH(mp, &
mountlist, mnt_list) {
4317 db_printf(
"%p %s on %s (%s)\n", mp,
4318 mp->mnt_stat.f_mntfromname,
4319 mp->mnt_stat.f_mntonname,
4320 mp->mnt_stat.f_fstypename);
4324 db_printf(
"\nMore info: show mount <addr>\n");
4328 mp = (
struct mount *)
addr;
4329 db_printf(
"%p %s on %s (%s)\n", mp, mp->mnt_stat.f_mntfromname,
4330 mp->mnt_stat.f_mntonname, mp->mnt_stat.f_fstypename);
4333 mflags = mp->mnt_flag;
4334#define MNT_FLAG(flag) do { \
4335 if (mflags & (flag)) { \
4336 if (buf[0] != '\0') \
4337 strlcat(buf, ", ", sizeof(buf)); \
4338 strlcat(buf, (#flag) + 4, sizeof(buf)); \
4339 mflags &= ~(flag); \
4342 MNT_FLAG(MNT_RDONLY);
4343 MNT_FLAG(MNT_SYNCHRONOUS);
4344 MNT_FLAG(MNT_NOEXEC);
4345 MNT_FLAG(MNT_NOSUID);
4346 MNT_FLAG(MNT_NFS4ACLS);
4347 MNT_FLAG(MNT_UNION);
4348 MNT_FLAG(MNT_ASYNC);
4349 MNT_FLAG(MNT_SUIDDIR);
4350 MNT_FLAG(MNT_SOFTDEP);
4351 MNT_FLAG(MNT_NOSYMFOLLOW);
4352 MNT_FLAG(MNT_GJOURNAL);
4353 MNT_FLAG(MNT_MULTILABEL);
4355 MNT_FLAG(MNT_NOATIME);
4356 MNT_FLAG(MNT_NOCLUSTERR);
4357 MNT_FLAG(MNT_NOCLUSTERW);
4359 MNT_FLAG(MNT_EXRDONLY);
4360 MNT_FLAG(MNT_EXPORTED);
4361 MNT_FLAG(MNT_DEFEXPORTED);
4362 MNT_FLAG(MNT_EXPORTANON);
4363 MNT_FLAG(MNT_EXKERB);
4364 MNT_FLAG(MNT_EXPUBLIC);
4365 MNT_FLAG(MNT_LOCAL);
4366 MNT_FLAG(MNT_QUOTA);
4367 MNT_FLAG(MNT_ROOTFS);
4369 MNT_FLAG(MNT_IGNORE);
4370 MNT_FLAG(MNT_UPDATE);
4371 MNT_FLAG(MNT_DELEXPORT);
4372 MNT_FLAG(MNT_RELOAD);
4373 MNT_FLAG(MNT_FORCE);
4374 MNT_FLAG(MNT_SNAPSHOT);
4375 MNT_FLAG(MNT_BYFSID);
4379 strlcat(
buf,
", ",
sizeof(
buf));
4381 "0x%016jx", mflags);
4383 db_printf(
" mnt_flag = %s\n",
buf);
4386 flags = mp->mnt_kern_flag;
4387#define MNT_KERN_FLAG(flag) do { \
4388 if (flags & (flag)) { \
4389 if (buf[0] != '\0') \
4390 strlcat(buf, ", ", sizeof(buf)); \
4391 strlcat(buf, (#flag) + 5, sizeof(buf)); \
4395 MNT_KERN_FLAG(MNTK_UNMOUNTF);
4396 MNT_KERN_FLAG(MNTK_ASYNC);
4397 MNT_KERN_FLAG(MNTK_SOFTDEP);
4398 MNT_KERN_FLAG(MNTK_NOMSYNC);
4399 MNT_KERN_FLAG(MNTK_DRAINING);
4400 MNT_KERN_FLAG(MNTK_REFEXPIRE);
4401 MNT_KERN_FLAG(MNTK_EXTENDED_SHARED);
4402 MNT_KERN_FLAG(MNTK_SHARED_WRITES);
4403 MNT_KERN_FLAG(MNTK_NO_IOPF);
4404 MNT_KERN_FLAG(MNTK_RECURSE);
4405 MNT_KERN_FLAG(MNTK_UPPER_WAITER);
4406 MNT_KERN_FLAG(MNTK_UNLOCKED_INSMNTQUE);
4407 MNT_KERN_FLAG(MNTK_USES_BCACHE);
4408 MNT_KERN_FLAG(MNTK_VMSETSIZE_BUG);
4409 MNT_KERN_FLAG(MNTK_FPLOOKUP);
4410 MNT_KERN_FLAG(MNTK_TASKQUEUE_WAITER);
4411 MNT_KERN_FLAG(MNTK_NOASYNC);
4412 MNT_KERN_FLAG(MNTK_UNMOUNT);
4413 MNT_KERN_FLAG(MNTK_MWAIT);
4414 MNT_KERN_FLAG(MNTK_SUSPEND);
4415 MNT_KERN_FLAG(MNTK_SUSPEND2);
4416 MNT_KERN_FLAG(MNTK_SUSPENDED);
4417 MNT_KERN_FLAG(MNTK_NULL_NOCACHE);
4418 MNT_KERN_FLAG(MNTK_LOOKUP_SHARED);
4422 strlcat(
buf,
", ",
sizeof(
buf));
4426 db_printf(
" mnt_kern_flag = %s\n",
buf);
4428 db_printf(
" mnt_opt = ");
4429 opt = TAILQ_FIRST(mp->mnt_opt);
4431 db_printf(
"%s", opt->name);
4432 opt = TAILQ_NEXT(opt, link);
4433 while (opt != NULL) {
4434 db_printf(
", %s", opt->name);
4435 opt = TAILQ_NEXT(opt, link);
4441 db_printf(
" mnt_stat = { version=%u type=%u flags=0x%016jx "
4442 "bsize=%ju iosize=%ju blocks=%ju bfree=%ju bavail=%jd files=%ju "
4443 "ffree=%jd syncwrites=%ju asyncwrites=%ju syncreads=%ju "
4444 "asyncreads=%ju namemax=%u owner=%u fsid=[%d, %d] }\n",
4445 (u_int)sp->f_version, (u_int)sp->f_type, (uintmax_t)sp->f_flags,
4446 (uintmax_t)sp->f_bsize, (uintmax_t)sp->f_iosize,
4447 (uintmax_t)sp->f_blocks, (uintmax_t)sp->f_bfree,
4448 (intmax_t)sp->f_bavail, (uintmax_t)sp->f_files,
4449 (intmax_t)sp->f_ffree, (uintmax_t)sp->f_syncwrites,
4450 (uintmax_t)sp->f_asyncwrites, (uintmax_t)sp->f_syncreads,
4451 (uintmax_t)sp->f_asyncreads, (u_int)sp->f_namemax,
4452 (u_int)sp->f_owner, (
int)sp->f_fsid.val[0], (
int)sp->f_fsid.val[1]);
4454 db_printf(
" mnt_cred = { uid=%u ruid=%u",
4455 (u_int)mp->mnt_cred->cr_uid, (u_int)mp->mnt_cred->cr_ruid);
4456 if (jailed(mp->mnt_cred))
4457 db_printf(
", jail=%d", mp->mnt_cred->cr_prison->pr_id);
4459 db_printf(
" mnt_ref = %d (with %d in the struct)\n",
4461 db_printf(
" mnt_gen = %d\n", mp->mnt_gen);
4462 db_printf(
" mnt_nvnodelistsize = %d\n", mp->mnt_nvnodelistsize);
4463 db_printf(
" mnt_lazyvnodelistsize = %d\n",
4464 mp->mnt_lazyvnodelistsize);
4465 db_printf(
" mnt_writeopcount = %d (with %d in the struct)\n",
4467 db_printf(
" mnt_iosize_max = %d\n", mp->mnt_iosize_max);
4468 db_printf(
" mnt_hashseed = %u\n", mp->mnt_hashseed);
4469 db_printf(
" mnt_lockref = %d (with %d in the struct)\n",
4471 db_printf(
" mnt_secondary_writes = %d\n", mp->mnt_secondary_writes);
4472 db_printf(
" mnt_secondary_accwrites = %d\n",
4473 mp->mnt_secondary_accwrites);
4474 db_printf(
" mnt_gjprovider = %s\n",
4475 mp->mnt_gjprovider != NULL ? mp->mnt_gjprovider :
"NULL");
4476 db_printf(
" mnt_vfs_ops = %d\n", mp->mnt_vfs_ops);
4478 db_printf(
"\n\nList of active vnodes\n");
4479 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4480 if (vp->v_type != VMARKER && vp->v_holdcnt > 0) {
4486 db_printf(
"\n\nList of inactive vnodes\n");
4487 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4488 if (vp->v_type != VMARKER && vp->v_holdcnt == 0) {
4503 struct xvfsconf xvfsp;
4505 bzero(&xvfsp,
sizeof(xvfsp));
4506 strcpy(xvfsp.vfc_name, vfsp->vfc_name);
4507 xvfsp.vfc_typenum = vfsp->vfc_typenum;
4508 xvfsp.vfc_refcount = vfsp->vfc_refcount;
4509 xvfsp.vfc_flags = vfsp->vfc_flags;
4514 xvfsp.vfc_vfsops = NULL;
4515 xvfsp.vfc_next = NULL;
4516 return (SYSCTL_OUT(req, &xvfsp,
sizeof(xvfsp)));
4519#ifdef COMPAT_FREEBSD32
4521 uint32_t vfc_vfsops;
4522 char vfc_name[MFSNAMELEN];
4523 int32_t vfc_typenum;
4524 int32_t vfc_refcount;
4530vfsconf2x32(
struct sysctl_req *req,
struct vfsconf *vfsp)
4532 struct xvfsconf32 xvfsp;
4534 bzero(&xvfsp,
sizeof(xvfsp));
4535 strcpy(xvfsp.vfc_name, vfsp->vfc_name);
4536 xvfsp.vfc_typenum = vfsp->vfc_typenum;
4537 xvfsp.vfc_refcount = vfsp->vfc_refcount;
4538 xvfsp.vfc_flags = vfsp->vfc_flags;
4539 return (SYSCTL_OUT(req, &xvfsp,
sizeof(xvfsp)));
4554 TAILQ_FOREACH(vfsp, &
vfsconf, vfc_list) {
4555#ifdef COMPAT_FREEBSD32
4556 if (req->flags & SCTL_MASK32)
4557 error = vfsconf2x32(req, vfsp);
4570 "S,xvfsconf",
"List of all configured filesystems");
4578 int *
name = (
int *)arg1 - 1;
4579 u_int namelen = arg2 + 1;
4582 log(LOG_WARNING,
"userland calling deprecated sysctl, "
4583 "please rebuild world\n");
4585#if 1 || defined(COMPAT_PRELITE2)
4592 case VFS_MAXTYPENUM:
4595 return (SYSCTL_OUT(req, &
maxvfsconf,
sizeof(
int)));
4600 TAILQ_FOREACH(vfsp, &
vfsconf, vfc_list) {
4601 if (vfsp->vfc_typenum ==
name[2])
4606 return (EOPNOTSUPP);
4607#ifdef COMPAT_FREEBSD32
4608 if (req->flags & SCTL_MASK32)
4609 return (vfsconf2x32(req, vfsp));
4614 return (EOPNOTSUPP);
4617static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD | CTLFLAG_SKIP |
4619 "Generic filesystem");
4621#if 1 || defined(COMPAT_PRELITE2)
4628 struct ovfsconf ovfs;
4631 TAILQ_FOREACH(vfsp, &
vfsconf, vfc_list) {
4632 bzero(&ovfs,
sizeof(ovfs));
4633 ovfs.vfc_vfsops = vfsp->vfc_vfsops;
4634 strcpy(ovfs.vfc_name, vfsp->vfc_name);
4635 ovfs.vfc_index = vfsp->vfc_typenum;
4636 ovfs.vfc_refcount = vfsp->vfc_refcount;
4637 ovfs.vfc_flags = vfsp->vfc_flags;
4638 error = SYSCTL_OUT(req, &ovfs,
sizeof ovfs);
4651#define KINFO_VNODESLOP 10
4658sysctl_vnode(SYSCTL_HANDLER_ARGS)
4672 return (SYSCTL_OUT(req, 0, len));
4677 xvn =
malloc(len, M_TEMP, M_ZERO | M_WAITOK);
4680 TAILQ_FOREACH(mp, &
mountlist, mnt_list) {
4681 if (
vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK))
4684 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
4688 xvn[n].xv_size =
sizeof *xvn;
4689 xvn[n].xv_vnode = vp;
4691#define XV_COPY(field) xvn[n].xv_##field = vp->v_##field
4693 XV_COPY(writecount);
4699 xvn[n].xv_flag = vp->v_vflag;
4701 switch (vp->v_type) {
4708 if (vp->v_rdev == NULL) {
4712 xvn[n].xv_dev = dev2udev(vp->v_rdev);
4715 xvn[n].xv_socket = vp->v_socket;
4718 xvn[n].xv_fifo = vp->v_fifoinfo;
4738 error = SYSCTL_OUT(req, xvn, n *
sizeof *xvn);
4743SYSCTL_PROC(_kern, KERN_VNODE, vnode, CTLTYPE_OPAQUE | CTLFLAG_RD |
4744 CTLFLAG_MPSAFE, 0, 0, sysctl_vnode,
"S,xvnode",
4753 error =
dounmount(mp, MNT_FORCE, curthread);
4755 printf(
"unmount of %s failed (", mp->mnt_stat.f_mntonname);
4770 struct mount *mp, *tmp;
4772 CTR1(KTR_VFS,
"%s: unmounting all filesystems", __func__);
4777 TAILQ_FOREACH_REVERSE_SAFE(mp, &
mountlist, mntlist, mnt_list, tmp) {
4798 ASSERT_VI_LOCKED(vp, __func__);
4799 VNASSERT((vp->v_iflag & VI_DEFINACT) == 0, vp, (
"VI_DEFINACT still set"));
4800 if ((vp->v_iflag & VI_OWEINACT) == 0) {
4804 if (vn_lock(vp, lkflags) == 0) {
4818 return (vp->v_iflag & VI_DEFINACT);
4821static void __noinline
4824 struct vnode *vp, *mvp;
4827 lkflags = LK_EXCLUSIVE | LK_INTERLOCK;
4828 if (
flags != MNT_WAIT)
4829 lkflags |= LK_NOWAIT;
4832 if ((vp->v_iflag & VI_DEFINACT) == 0) {
4836 vp->v_iflag &= ~VI_DEFINACT;
4844 struct vm_object *obj;
4850 if (vp->v_vflag & VV_NOSYNC)
4853 return (obj != NULL && vm_object_mightbedirty(obj));
4860 if (vp->v_vflag & VV_NOSYNC)
4862 if (vp->v_iflag & VI_DEFINACT)
4867static void __noinline
4870 struct vnode *vp, *mvp;
4871 struct vm_object *obj;
4872 int lkflags, objflags;
4875 lkflags = LK_EXCLUSIVE | LK_INTERLOCK;
4876 if (
flags != MNT_WAIT) {
4877 lkflags |= LK_NOWAIT;
4878 objflags = OBJPC_NOSYNC;
4880 objflags = OBJPC_SYNC;
4885 if (vp->v_iflag & VI_DEFINACT) {
4886 vp->v_iflag &= ~VI_DEFINACT;
4896 if (
vget(vp, lkflags) == 0) {
4898 if (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0) {
4899 VM_OBJECT_WLOCK(obj);
4900 vm_object_page_clean(obj, 0, 0, objflags);
4901 VM_OBJECT_WUNLOCK(obj);
4917 CTR2(KTR_VFS,
"%s: mp %p", __func__, mp);
4919 if ((mp->mnt_kern_flag & MNTK_NOMSYNC) != 0)
4930 mtx_destroy(&vi->vpi_lock);
4931 free(vi, M_VNODEPOLL);
4938 knlist_clear(&vi->vpi_selinfo.si_note, 1);
4949 struct vpollinfo *vi;
4951 if (vp->v_pollinfo != NULL)
4953 vi =
malloc(
sizeof(*vi), M_VNODEPOLL, M_WAITOK | M_ZERO);
4954 mtx_init(&vi->vpi_lock,
"vnode pollinfo", NULL, MTX_DEF);
4958 if (vp->v_pollinfo != NULL) {
4963 vp->v_pollinfo = vi;
4980 mtx_lock(&vp->v_pollinfo->vpi_lock);
4981 if (vp->v_pollinfo->vpi_revents & events) {
4989 events &= vp->v_pollinfo->vpi_revents;
4990 vp->v_pollinfo->vpi_revents &= ~events;
4992 mtx_unlock(&vp->v_pollinfo->vpi_lock);
4995 vp->v_pollinfo->vpi_events |= events;
4996 selrecord(td, &vp->v_pollinfo->vpi_selinfo);
4997 mtx_unlock(&vp->v_pollinfo->vpi_lock);
5004#define sync_close ((int (*)(struct vop_close_args *))nullop)
5005static int sync_fsync(
struct vop_fsync_args *);
5010 .vop_bypass = VOP_EOPNOTSUPP,
5030 static long start, incr, next;
5036 panic(
"vfs_allocate_syncvnode: getnewvnode() failed");
5038 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
5039 vp->v_vflag |= VV_FORCEINSMQ;
5042 panic(
"vfs_allocate_syncvnode: insmntque() failed");
5043 vp->v_vflag &= ~VV_FORCEINSMQ;
5052 if (next == 0 || next > syncer_maxdelay) {
5056 start = syncer_maxdelay / 2;
5057 incr = syncer_maxdelay;
5065 mtx_lock(&sync_mtx);
5067 if (mp->mnt_syncer == NULL) {
5068 mp->mnt_syncer = vp;
5071 mtx_unlock(&sync_mtx);
5074 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
5085 mtx_lock(&sync_mtx);
5086 vp = mp->mnt_syncer;
5088 mp->mnt_syncer = NULL;
5089 mtx_unlock(&sync_mtx);
5100 struct vnode *syncvp = ap->a_vp;
5101 struct mount *mp = syncvp->v_mount;
5108 if (ap->a_waitfor != MNT_LAZY)
5114 bo = &syncvp->v_bufobj;
5126 save = curthread_pflags_set(TDP_SYNCIO);
5132 error = VFS_SYNC(mp, MNT_LAZY);
5133 curthread_pflags_restore(save);
5134 vn_lock(syncvp, LK_EXCLUSIVE | LK_RETRY);
5158 struct vnode *vp = ap->a_vp;
5163 mtx_lock(&sync_mtx);
5164 if (vp->v_mount->mnt_syncer == vp)
5165 vp->v_mount->mnt_syncer = NULL;
5166 if (bo->bo_flag & BO_ONWORKLST) {
5167 LIST_REMOVE(bo, bo_synclist);
5170 bo->bo_flag &= ~BO_ONWORKLST;
5172 mtx_unlock(&sync_mtx);
5181 struct vm_object *obj;
5184 return (obj != NULL && (vp->v_vflag & VV_NOSYNC) == 0 &&
5185 vm_object_mightbedirty(obj));
5196 if (vp->v_type != VCHR) {
5202 if (vp->v_rdev == NULL)
5204 else if (vp->v_rdev->si_devsw == NULL)
5206 else if (!(vp->v_rdev->si_devsw->d_flags & D_DISK))
5211 return (error == 0);
5231 VFS_SMR_ASSERT_ENTERED();
5234 if (cred->cr_uid == file_uid) {
5235 if (file_mode & S_IXUSR)
5242 if (file_mode & S_IXGRP)
5248 if (file_mode & S_IXOTH)
5284vaccess(
enum vtype
type, mode_t file_mode, uid_t file_uid, gid_t file_gid,
5285 accmode_t
accmode,
struct ucred *cred)
5287 accmode_t dac_granted;
5288 accmode_t priv_granted;
5290 KASSERT((
accmode & ~(VEXEC | VWRITE | VREAD | VADMIN | VAPPEND)) == 0,
5291 (
"invalid bit in accmode"));
5293 (
"VAPPEND without VWRITE"));
5303 if (cred->cr_uid == file_uid) {
5304 dac_granted |= VADMIN;
5305 if (file_mode & S_IXUSR)
5306 dac_granted |= VEXEC;
5307 if (file_mode & S_IRUSR)
5308 dac_granted |= VREAD;
5309 if (file_mode & S_IWUSR)
5310 dac_granted |= (VWRITE | VAPPEND);
5320 if (file_mode & S_IXGRP)
5321 dac_granted |= VEXEC;
5322 if (file_mode & S_IRGRP)
5323 dac_granted |= VREAD;
5324 if (file_mode & S_IWGRP)
5325 dac_granted |= (VWRITE | VAPPEND);
5334 if (file_mode & S_IXOTH)
5335 dac_granted |= VEXEC;
5336 if (file_mode & S_IROTH)
5337 dac_granted |= VREAD;
5338 if (file_mode & S_IWOTH)
5339 dac_granted |= (VWRITE | VAPPEND);
5357 if ((
accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
5359 priv_granted |= VEXEC;
5366 if ((
accmode & VEXEC) && ((dac_granted & VEXEC) == 0) &&
5367 (file_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 &&
5369 priv_granted |= VEXEC;
5372 if ((
accmode & VREAD) && ((dac_granted & VREAD) == 0) &&
5374 priv_granted |= VREAD;
5376 if ((
accmode & VWRITE) && ((dac_granted & VWRITE) == 0) &&
5378 priv_granted |= (VWRITE | VAPPEND);
5380 if ((
accmode & VADMIN) && ((dac_granted & VADMIN) == 0) &&
5382 priv_granted |= VADMIN;
5388 return ((
accmode & VADMIN) ? EPERM : EACCES);
5397 struct thread *td, accmode_t
accmode)
5411 case EXTATTR_NAMESPACE_SYSTEM:
5414 case EXTATTR_NAMESPACE_USER:
5415 return (VOP_ACCESS(vp,
accmode, cred, td));
5421#ifdef DEBUG_VFS_LOCKS
5422int vfs_badlock_ddb = 1;
5423SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_ddb, CTLFLAG_RW, &vfs_badlock_ddb, 0,
5424 "Drop into debugger on lock violation");
5426int vfs_badlock_mutex = 1;
5427SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_mutex, CTLFLAG_RW, &vfs_badlock_mutex,
5428 0,
"Check for interlock across VOPs");
5430int vfs_badlock_print = 1;
5431SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_print, CTLFLAG_RW, &vfs_badlock_print,
5432 0,
"Print lock violations");
5434int vfs_badlock_vnode = 1;
5435SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_vnode, CTLFLAG_RW, &vfs_badlock_vnode,
5436 0,
"Print vnode details on lock violations");
5439int vfs_badlock_backtrace = 1;
5440SYSCTL_INT(_debug, OID_AUTO, vfs_badlock_backtrace, CTLFLAG_RW,
5441 &vfs_badlock_backtrace, 0,
"Print backtrace at lock violations");
5445vfs_badlock(
const char *msg,
const char *str,
struct vnode *vp)
5449 if (vfs_badlock_backtrace)
5452 if (vfs_badlock_vnode)
5454 if (vfs_badlock_print)
5455 printf(
"%s: %p %s\n", str, (
void *)vp, msg);
5456 if (vfs_badlock_ddb)
5457 kdb_enter(KDB_WHY_VFSLOCK,
"lock violation");
5461assert_vi_locked(
struct vnode *vp,
const char *str)
5464 if (vfs_badlock_mutex && !mtx_owned(VI_MTX(vp)))
5465 vfs_badlock(
"interlock is not locked but should be", str, vp);
5469assert_vi_unlocked(
struct vnode *vp,
const char *str)
5472 if (vfs_badlock_mutex && mtx_owned(VI_MTX(vp)))
5473 vfs_badlock(
"interlock is locked but should not be", str, vp);
5477assert_vop_locked(
struct vnode *vp,
const char *str)
5481 if (KERNEL_PANICKED() || vp == NULL)
5484 locked = VOP_ISLOCKED(vp);
5485 if (locked == 0 || locked == LK_EXCLOTHER)
5486 vfs_badlock(
"is not locked but should be", str, vp);
5490assert_vop_unlocked(
struct vnode *vp,
const char *str)
5492 if (KERNEL_PANICKED() || vp == NULL)
5495 if (VOP_ISLOCKED(vp) == LK_EXCLUSIVE)
5496 vfs_badlock(
"is locked but should not be", str, vp);
5500assert_vop_elocked(
struct vnode *vp,
const char *str)
5502 if (KERNEL_PANICKED() || vp == NULL)
5505 if (VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
5506 vfs_badlock(
"is not exclusive locked but should be", str, vp);
5514 if (ap->a_tvp != NULL)
5516 if (ap->a_tdvp == ap->a_tvp)
5527 struct vop_rename_args *a = ap;
5529#ifdef DEBUG_VFS_LOCKS
5531 ASSERT_VI_UNLOCKED(a->a_tvp,
"VOP_RENAME");
5532 ASSERT_VI_UNLOCKED(a->a_tdvp,
"VOP_RENAME");
5533 ASSERT_VI_UNLOCKED(a->a_fvp,
"VOP_RENAME");
5534 ASSERT_VI_UNLOCKED(a->a_fdvp,
"VOP_RENAME");
5537 if (a->a_tdvp->v_vnlock != a->a_fdvp->v_vnlock &&
5538 (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fdvp->v_vnlock))
5539 ASSERT_VOP_UNLOCKED(a->a_fdvp,
"vop_rename: fdvp locked");
5540 if (a->a_tvp == NULL || a->a_tvp->v_vnlock != a->a_fvp->v_vnlock)
5541 ASSERT_VOP_UNLOCKED(a->a_fvp,
"vop_rename: fvp locked");
5545 ASSERT_VOP_LOCKED(a->a_tvp,
"vop_rename: tvp not locked");
5546 ASSERT_VOP_LOCKED(a->a_tdvp,
"vop_rename: tdvp not locked");
5556 if (a->a_tdvp != a->a_fdvp)
5558 if (a->a_tvp != a->a_fvp)
5565#ifdef DEBUG_VFS_LOCKS
5567vop_fplookup_vexec_debugpre(
void *ap __unused)
5570 VFS_SMR_ASSERT_ENTERED();
5574vop_fplookup_vexec_debugpost(
void *ap __unused,
int rc __unused)
5577 VFS_SMR_ASSERT_ENTERED();
5581vop_fplookup_symlink_debugpre(
void *ap __unused)
5584 VFS_SMR_ASSERT_ENTERED();
5588vop_fplookup_symlink_debugpost(
void *ap __unused,
int rc __unused)
5591 VFS_SMR_ASSERT_ENTERED();
5595vop_fsync_debugprepost(
struct vnode *vp,
const char *
name)
5597 if (vp->v_type == VCHR)
5599 else if (MNT_EXTENDED_SHARED(vp->v_mount))
5600 ASSERT_VOP_LOCKED(vp,
name);
5602 ASSERT_VOP_ELOCKED(vp,
name);
5606vop_fsync_debugpre(
void *a)
5608 struct vop_fsync_args *ap;
5611 vop_fsync_debugprepost(ap->a_vp,
"fsync");
5615vop_fsync_debugpost(
void *a,
int rc __unused)
5617 struct vop_fsync_args *ap;
5620 vop_fsync_debugprepost(ap->a_vp,
"fsync");
5624vop_fdatasync_debugpre(
void *a)
5626 struct vop_fdatasync_args *ap;
5629 vop_fsync_debugprepost(ap->a_vp,
"fsync");
5633vop_fdatasync_debugpost(
void *a,
int rc __unused)
5635 struct vop_fdatasync_args *ap;
5638 vop_fsync_debugprepost(ap->a_vp,
"fsync");
5642vop_strategy_debugpre(
void *ap)
5644 struct vop_strategy_args *a;
5653 if ((bp->b_flags & B_CLUSTER) != 0)
5656 if (!KERNEL_PANICKED() && !BUF_ISLOCKED(bp)) {
5657 if (vfs_badlock_print)
5659 "VOP_STRATEGY: bp is not locked but should be\n");
5660 if (vfs_badlock_ddb)
5661 kdb_enter(KDB_WHY_VFSLOCK,
"lock violation");
5666vop_lock_debugpre(
void *ap)
5668 struct vop_lock1_args *a = ap;
5670 if ((a->a_flags & LK_INTERLOCK) == 0)
5671 ASSERT_VI_UNLOCKED(a->a_vp,
"VOP_LOCK");
5673 ASSERT_VI_LOCKED(a->a_vp,
"VOP_LOCK");
5677vop_lock_debugpost(
void *ap,
int rc)
5679 struct vop_lock1_args *a = ap;
5681 ASSERT_VI_UNLOCKED(a->a_vp,
"VOP_LOCK");
5682 if (rc == 0 && (a->a_flags & LK_EXCLOTHER) == 0)
5683 ASSERT_VOP_LOCKED(a->a_vp,
"VOP_LOCK");
5687vop_unlock_debugpre(
void *ap)
5689 struct vop_unlock_args *a = ap;
5691 ASSERT_VOP_LOCKED(a->a_vp,
"VOP_UNLOCK");
5695vop_need_inactive_debugpre(
void *ap)
5697 struct vop_need_inactive_args *a = ap;
5699 ASSERT_VI_LOCKED(a->a_vp,
"VOP_NEED_INACTIVE");
5703vop_need_inactive_debugpost(
void *ap,
int rc)
5705 struct vop_need_inactive_args *a = ap;
5707 ASSERT_VI_LOCKED(a->a_vp,
"VOP_NEED_INACTIVE");
5714 struct vop_create_args *a;
5725 struct vop_create_args *a;
5732 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
5738 struct vop_whiteout_args *a;
5749 struct vop_whiteout_args *a;
5760 struct vop_deleteextattr_args *a;
5771 struct vop_deleteextattr_args *a;
5778 VFS_KNOTE_LOCKED(a->a_vp, NOTE_ATTRIB);
5784 struct vop_link_args *a;
5785 struct vnode *vp, *tdvp;
5797 struct vop_link_args *a;
5798 struct vnode *vp, *tdvp;
5806 VFS_KNOTE_LOCKED(vp, NOTE_LINK);
5807 VFS_KNOTE_LOCKED(tdvp, NOTE_WRITE);
5814 struct vop_mkdir_args *a;
5825 struct vop_mkdir_args *a;
5832 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK);
5835#ifdef DEBUG_VFS_LOCKS
5837vop_mkdir_debugpost(
void *ap,
int rc)
5839 struct vop_mkdir_args *a;
5843 cache_validate(a->a_dvp, *a->a_vpp, a->a_cnp);
5850 struct vop_mknod_args *a;
5861 struct vop_mknod_args *a;
5868 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
5874 struct vop_reclaim_args *a;
5879 ASSERT_VOP_IN_SEQC(vp);
5881 VFS_KNOTE_LOCKED(vp, NOTE_REVOKE);
5887 struct vop_remove_args *a;
5888 struct vnode *dvp, *vp;
5900 struct vop_remove_args *a;
5901 struct vnode *dvp, *vp;
5909 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
5910 VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
5917 struct vop_rename_args *a = ap;
5922 if (a->a_fdvp == a->a_tdvp) {
5923 if (a->a_tvp != NULL && a->a_tvp->v_type == VDIR)
5925 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);
5926 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
5928 hint |= NOTE_EXTEND;
5929 if (a->a_fvp->v_type == VDIR)
5931 VFS_KNOTE_UNLOCKED(a->a_fdvp, hint);
5933 if (a->a_fvp->v_type == VDIR && a->a_tvp != NULL &&
5934 a->a_tvp->v_type == VDIR)
5936 VFS_KNOTE_UNLOCKED(a->a_tdvp, hint);
5939 VFS_KNOTE_UNLOCKED(a->a_fvp, NOTE_RENAME);
5941 VFS_KNOTE_UNLOCKED(a->a_tvp, NOTE_DELETE);
5943 if (a->a_tdvp != a->a_fdvp)
5945 if (a->a_tvp != a->a_fvp)
5955 struct vop_rmdir_args *a;
5956 struct vnode *dvp, *vp;
5968 struct vop_rmdir_args *a;
5969 struct vnode *dvp, *vp;
5977 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE | NOTE_LINK);
5978 VFS_KNOTE_LOCKED(vp, NOTE_DELETE);
5985 struct vop_setattr_args *a;
5996 struct vop_setattr_args *a;
6003 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
6009 struct vop_setacl_args *a;
6020 struct vop_setacl_args *a;
6031 struct vop_setextattr_args *a;
6042 struct vop_setextattr_args *a;
6049 VFS_KNOTE_LOCKED(vp, NOTE_ATTRIB);
6055 struct vop_symlink_args *a;
6066 struct vop_symlink_args *a;
6073 VFS_KNOTE_LOCKED(dvp, NOTE_WRITE);
6079 struct vop_open_args *a = ap;
6082 VFS_KNOTE_LOCKED(a->a_vp, NOTE_OPEN);
6088 struct vop_close_args *a = ap;
6090 if (!rc && (a->a_cred != NOCRED ||
6091 !VN_IS_DOOMED(a->a_vp))) {
6092 VFS_KNOTE_LOCKED(a->a_vp, (a->a_fflag & FWRITE) != 0 ?
6093 NOTE_CLOSE_WRITE : NOTE_CLOSE);
6100 struct vop_read_args *a = ap;
6103 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
6109 struct vop_read_pgcache_args *a = ap;
6112 VFS_KNOTE_UNLOCKED(a->a_vp, NOTE_READ);
6118 struct vop_readdir_args *a = ap;
6121 VFS_KNOTE_LOCKED(a->a_vp, NOTE_READ);
6156 kn->kn_flags |= EV_CLEAR;
6172 kn->kn_fflags |= kn->kn_sfflags & hint;
6174 return (kn->kn_fflags != 0);
6184 error = SYSCTL_IN(req, &vc,
sizeof(vc));
6187 if (vc.vc_vers != VFS_CTL_VERS1)
6193 if (strcmp(vc.vc_fstypename,
"*") != 0 &&
6194 strcmp(vc.vc_fstypename, mp->mnt_vfc->vfc_name) != 0) {
6198 VCTLTOREQ(&vc, req);
6199 error = VFS_SYSCTL(mp, vc.vc_op, req);
6204SYSCTL_PROC(_vfs, OID_AUTO, ctl, CTLTYPE_OPAQUE | CTLFLAG_MPSAFE | CTLFLAG_WR,
6218 return (((u_quad_t)
bt.sec << 32LL) | (
bt.frac >> 32LL));
6244 struct vnode *vp = arg;
6246 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
6252 struct vnode *vp = arg;
6260#ifdef DEBUG_VFS_LOCKS
6261 struct vnode *vp = arg;
6263 if (what == LA_LOCKED)
6264 ASSERT_VOP_LOCKED(vp,
"vfs_knl_assert_locked");
6266 ASSERT_VOP_UNLOCKED(vp,
"vfs_knl_assert_unlocked");
6273 struct vnode *vp = ap->a_vp;
6274 struct knote *kn = ap->a_kn;
6277 KASSERT(vp->v_type != VFIFO || (kn->kn_filter != EVFILT_READ &&
6278 kn->kn_filter != EVFILT_WRITE),
6279 (
"READ/WRITE filter on a FIFO leaked through"));
6280 switch (kn->kn_filter) {
6294 kn->kn_hook = (caddr_t)vp;
6297 if (vp->v_pollinfo == NULL)
6299 knl = &vp->v_pollinfo->vpi_selinfo.si_note;
6312 struct vnode *vp = (
struct vnode *)kn->kn_hook;
6314 KASSERT(vp->v_pollinfo != NULL, (
"Missing v_pollinfo"));
6323 struct vnode *vp = (
struct vnode *)kn->kn_hook;
6331 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
6333 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
6338 if (VOP_GETATTR(vp, &va, curthread->td_ucred))
6342 kn->kn_data = va.va_size - kn->kn_fp->f_offset;
6343 res = (kn->kn_sfflags & NOTE_FILE_POLL) != 0 || kn->kn_data != 0;
6352 struct vnode *vp = (
struct vnode *)kn->kn_hook;
6360 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD))
6361 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
6371 struct vnode *vp = (
struct vnode *)kn->kn_hook;
6375 if (kn->kn_sfflags & hint)
6376 kn->kn_fflags |= hint;
6377 if (hint == NOTE_REVOKE || (hint == 0 && vp->v_type == VBAD)) {
6378 kn->kn_flags |= EV_EOF;
6382 res = (kn->kn_fflags != 0);
6398 struct dirent *dirent, *dp, *endp;
6404 ASSERT_VOP_LOCKED(vp,
"vfs_emptydir");
6405 VNASSERT(vp->v_type == VDIR, vp, (
"vp is not a directory"));
6407 dirent =
malloc(
sizeof(
struct dirent), M_TEMP, M_WAITOK);
6408 iov.iov_base = dirent;
6409 iov.iov_len =
sizeof(
struct dirent);
6414 uio.uio_resid =
sizeof(
struct dirent);
6415 uio.uio_segflg = UIO_SYSSPACE;
6416 uio.uio_rw = UIO_READ;
6417 uio.uio_td = curthread;
6419 while (eof == 0 && error == 0) {
6420 error = VOP_READDIR(vp, &uio, curthread->td_ucred, &eof,
6424 endp = (
void *)((uint8_t *)dirent +
6425 sizeof(
struct dirent) - uio.uio_resid);
6426 for (dp = dirent; dp < endp;
6427 dp = (
void *)((uint8_t *)dp + GENERIC_DIRSIZ(dp))) {
6428 if (dp->d_type == DT_WHT)
6430 if (dp->d_namlen == 0)
6432 if (dp->d_type != DT_DIR &&
6433 dp->d_type != DT_UNKNOWN) {
6437 if (dp->d_namlen > 2) {
6441 if (dp->d_namlen == 1 &&
6442 dp->d_name[0] !=
'.') {
6446 if (dp->d_namlen == 2 &&
6447 dp->d_name[1] !=
'.') {
6451 uio.uio_resid =
sizeof(
struct dirent);
6454 free(dirent, M_TEMP);
6463 if (dp->d_reclen > ap->a_uio->uio_resid)
6464 return (ENAMETOOLONG);
6465 error =
uiomove(dp, dp->d_reclen, ap->a_uio);
6467 if (ap->a_ncookies != NULL) {
6468 if (ap->a_cookies != NULL)
6469 free(ap->a_cookies, M_TEMP);
6470 ap->a_cookies = NULL;
6471 *ap->a_ncookies = 0;
6475 if (ap->a_ncookies == NULL)
6478 KASSERT(ap->a_cookies,
6479 (
"NULL ap->a_cookies value with non-NULL ap->a_ncookies!"));
6481 *ap->a_cookies =
realloc(*ap->a_cookies,
6482 (*ap->a_ncookies + 1) *
sizeof(uint64_t), M_TEMP, M_WAITOK | M_ZERO);
6483 (*ap->a_cookies)[*ap->a_ncookies] = off;
6484 *ap->a_ncookies += 1;
6507 if (*
accmode & VEXPLICIT_DENY) {
6518 if (*
accmode & (VDELETE_CHILD | VDELETE))
6521 if (*
accmode & VADMIN_PERMS) {
6530 *
accmode &= ~(VSTAT_PERMS | VSYNCHRONIZE);
6539static int __noinline
6546 if (mp->mnt_rootvnode != NULL) {
6548 vp = mp->mnt_rootvnode;
6550 if (!VN_IS_DOOMED(vp)) {
6553 error = vn_lock(vp,
flags);
6564 mp->mnt_rootvnode = NULL;
6572 error = VFS_CACHEDROOT(mp,
flags, vpp);
6575 if (mp->mnt_vfs_ops == 0) {
6577 if (mp->mnt_vfs_ops != 0) {
6581 if (mp->mnt_rootvnode == NULL) {
6583 mp->mnt_rootvnode = *vpp;
6585 if (mp->mnt_rootvnode != *vpp) {
6586 if (!VN_IS_DOOMED(mp->mnt_rootvnode)) {
6587 panic(
"%s: mismatch between vnode returned "
6588 " by VFS_CACHEDROOT and the one cached "
6590 __func__, *vpp, mp->mnt_rootvnode);
6602 struct mount_pcpu *mpcpu;
6606 if (!vfs_op_thread_enter(mp, mpcpu))
6608 vp = atomic_load_ptr(&mp->mnt_rootvnode);
6609 if (vp == NULL || VN_IS_DOOMED(vp)) {
6610 vfs_op_thread_exit(mp, mpcpu);
6614 vfs_op_thread_exit(mp, mpcpu);
6615 error = vn_lock(vp,
flags);
6632 MPASS(mp->mnt_vfs_ops > 0);
6633 vp = mp->mnt_rootvnode;
6636 mp->mnt_rootvnode = NULL;
6644 MPASS(mp->mnt_vfs_ops > 0);
6646 mp->mnt_rootvnode = vp;
6664 KASSERT((*mvp)->v_mount == mp, (
"marker vnode mount list mismatch"));
6665 for (vp = TAILQ_NEXT(*mvp, v_nmntvnodes); vp != NULL;
6666 vp = TAILQ_NEXT(vp, v_nmntvnodes)) {
6668 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp))
6671 if (VN_IS_DOOMED(vp)) {
6680 mtx_assert(MNT_MTX(mp), MA_NOTOWNED);
6683 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
6684 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
6698 TAILQ_FOREACH(vp, &mp->mnt_nvnodelist, v_nmntvnodes) {
6700 if (vp->v_type == VMARKER || VN_IS_DOOMED(vp))
6703 if (VN_IS_DOOMED(vp)) {
6716 TAILQ_INSERT_AFTER(&mp->mnt_nvnodelist, vp, *mvp, v_nmntvnodes);
6730 mtx_assert(MNT_MTX(mp), MA_OWNED);
6732 KASSERT((*mvp)->v_mount == mp, (
"marker vnode mount list mismatch"));
6733 TAILQ_REMOVE(&mp->mnt_nvnodelist, *mvp, v_nmntvnodes);
6748 KASSERT((*mvp)->v_mount == mp, (
"marker vnode mount list mismatch"));
6771 VNASSERT(mvp->v_mount == mp && mvp->v_type == VMARKER &&
6772 TAILQ_NEXT(mvp, v_lazylist) != NULL, mvp,
6773 (
"%s: bad marker", __func__));
6774 VNASSERT(vp->v_mount == mp && vp->v_type != VMARKER, vp,
6775 (
"%s: inappropriate vnode", __func__));
6776 ASSERT_VI_UNLOCKED(vp, __func__);
6777 mtx_assert(&mp->mnt_listmtx, MA_OWNED);
6779 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, mvp, v_lazylist);
6780 TAILQ_INSERT_BEFORE(vp, mvp, v_lazylist);
6789 mtx_unlock(&mp->mnt_listmtx);
6791 if (VN_IS_DOOMED(vp)) {
6792 VNPASS((vp->v_mflag & VMP_LAZYLIST) == 0, vp);
6795 VNPASS(vp->v_mflag & VMP_LAZYLIST, vp);
6799 if (!refcount_release_if_not_last(&vp->v_holdcnt))
6801 mtx_lock(&mp->mnt_listmtx);
6806 mtx_lock(&mp->mnt_listmtx);
6810static struct vnode *
6816 mtx_assert(&mp->mnt_listmtx, MA_OWNED);
6817 KASSERT((*mvp)->v_mount == mp, (
"marker vnode mount list mismatch"));
6819 vp = TAILQ_NEXT(*mvp, v_lazylist);
6820 while (vp != NULL) {
6821 if (vp->v_type == VMARKER) {
6822 vp = TAILQ_NEXT(vp, v_lazylist);
6830 VNPASS(!VN_IS_DOOMED(vp), vp);
6831 if (!cb(vp, cbarg)) {
6833 vp = TAILQ_NEXT(vp, v_lazylist);
6836 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp,
6838 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp,
6840 mtx_unlock(&mp->mnt_listmtx);
6842 mtx_lock(&mp->mnt_listmtx);
6848 if (!VI_TRYLOCK(vp) &&
6851 KASSERT(vp->v_type != VMARKER, (
"locked marker %p", vp));
6852 KASSERT(vp->v_mount == mp || vp->v_mount == NULL,
6853 (
"alien vnode on the lazy list %p %p", vp, mp));
6854 VNPASS(vp->v_mount == mp, vp);
6855 VNPASS(!VN_IS_DOOMED(vp), vp);
6858 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist);
6862 mtx_unlock(&mp->mnt_listmtx);
6866 TAILQ_INSERT_AFTER(&mp->mnt_lazyvnodelist, vp, *mvp, v_lazylist);
6867 mtx_unlock(&mp->mnt_listmtx);
6868 ASSERT_VI_LOCKED(vp,
"lazy iter");
6879 mtx_lock(&mp->mnt_listmtx);
6889 if (TAILQ_EMPTY(&mp->mnt_lazyvnodelist))
6897 mtx_lock(&mp->mnt_listmtx);
6898 vp = TAILQ_FIRST(&mp->mnt_lazyvnodelist);
6900 mtx_unlock(&mp->mnt_listmtx);
6904 TAILQ_INSERT_BEFORE(vp, *mvp, v_lazylist);
6915 mtx_lock(&mp->mnt_listmtx);
6916 TAILQ_REMOVE(&mp->mnt_lazyvnodelist, *mvp, v_lazylist);
6917 mtx_unlock(&mp->mnt_listmtx);
6925 if ((cnp->cn_flags & NOEXECCHECK) != 0) {
6926 cnp->cn_flags &= ~NOEXECCHECK;
6930 return (VOP_ACCESS(vp, VEXEC, cnp->cn_cred, curthread));
6941 ASSERT_VI_LOCKED(vp, __func__);
6942 VNPASS(vp->v_holdcnt > 0, vp);
6943 VNPASS(vp->v_seqc_users >= 0, vp);
6945 if (vp->v_seqc_users == 1)
6946 seqc_sleepable_write_begin(&vp->v_seqc);
6962 ASSERT_VI_LOCKED(vp, __func__);
6963 VNPASS(vp->v_seqc_users > 0, vp);
6965 if (vp->v_seqc_users == 0)
6966 seqc_sleepable_write_end(&vp->v_seqc);
6989 vp->v_seqc_users = 0;
6996 VNPASS(seqc_in_modify(vp->v_seqc), vp);
6997 VNPASS(vp->v_seqc_users == 1, vp);
7005 ASSERT_VI_LOCKED(vp, __func__);
7006 flags = vn_irflag_read(vp);
7007 VNASSERT((
flags & toset) == 0, vp,
7008 (
"%s: some of the passed flags already set (have %d, passed %d)\n",
7009 __func__,
flags, toset));
7010 atomic_store_short(&vp->v_irflag,
flags | toset);
7027 ASSERT_VI_LOCKED(vp, __func__);
7028 flags = vn_irflag_read(vp);
7029 atomic_store_short(&vp->v_irflag,
flags | toset);
7046 ASSERT_VI_LOCKED(vp, __func__);
7047 flags = vn_irflag_read(vp);
7048 VNASSERT((
flags & tounset) == tounset, vp,
7049 (
"%s: some of the passed flags not set (have %d, passed %d)\n",
7050 __func__,
flags, tounset));
7051 atomic_store_short(&vp->v_irflag,
flags & ~tounset);
device_property_type_t type
static struct bt_table bt
void cv_init(struct cv *cvp, const char *desc)
void knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
void knlist_init(struct knlist *knl, void *lock, void(*kl_lock)(void *), void(*kl_unlock)(void *), void(*kl_assert_lock)(void *, int))
void knlist_add(struct knlist *knl, struct knote *kn, int islocked)
void knlist_destroy(struct knlist *knl)
void knote(struct knlist *list, long hint, int lockflags)
void knlist_init_mtx(struct knlist *knl, struct mtx *lock)
int prison_check(struct ucred *cred1, struct ucred *cred2)
int prison_allow(struct ucred *cred, unsigned flag)
int kproc_resume(struct proc *p)
void kproc_suspend_check(struct proc *p)
void kproc_start(const void *udata)
void lockdestroy(struct lock *lk)
void lockmgr_printinfo(const struct lock *lk)
void lockinit(struct lock *lk, int pri, const char *wmesg, int timo, int flags)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void * realloc(void *addr, size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
int priv_check_cred_vfs_lookup_nomac(struct ucred *cred)
int priv_check_cred(struct ucred *cred, int priv)
int priv_check(struct thread *td, int priv)
int groupmember(gid_t gid, struct ucred *cred)
void rangelock_destroy(struct rangelock *lock)
void rangelock_init(struct rangelock *lock)
void kproc_shutdown(void *arg, int howto)
void panic(const char *fmt,...)
void kern_yield(int prio)
void wakeup(const void *ident)
int sysctl_wire_old_buffer(struct sysctl_req *req, size_t len)
int sysctl_handle_long(SYSCTL_HANDLER_ARGS)
int sysctl_handle_int(SYSCTL_HANDLER_ARGS)
void getbinuptime(struct bintime *bt)
volatile time_t time_uptime
volatile time_t time_second
void bintime(struct bintime *bt)
void getnanotime(struct timespec *tsp)
void microtime(struct timeval *tvp)
void nanotime(struct timespec *tsp)
void sched_prio(struct thread *td, u_char prio)
void kasan_mark(const void *addr, size_t size, size_t redzsize, uint8_t code)
__read_mostly cap_rights_t cap_fcntl_rights
counter_u64_t counter_u64_alloc(int flags)
void * hashinit(int elements, struct malloc_type *type, u_long *hashmask)
void kdb_enter(const char *why, const char *msg)
size_t pctrie_node_size(void)
int pctrie_zone_init(void *mem, int size __unused, int flags __unused)
int printf(const char *fmt,...)
int vprintf(const char *fmt, va_list ap)
int snprintf(char *str, size_t size, const char *format,...)
void log(int level, const char *fmt,...)
int uiomove(void *cp, int n, struct uio *uio)
void seldrain(struct selinfo *sip)
void selrecord(struct thread *selector, struct selinfo *sip)
void vfs_unp_reclaim(struct vnode *vp)
void bremfree(struct buf *bp)
void bufobj_init(struct bufobj *bo, void *private)
int bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
void bawrite(struct buf *bp)
struct buf_ops buf_ops_bio
void brelse(struct buf *bp)
void cache_changesize(u_long newmaxvnodes)
void cache_vnode_init(struct vnode *vp)
void cache_purge_vgone(struct vnode *vp)
int vop_stdislocked(struct vop_islocked_args *ap)
int vop_stdunlock(struct vop_unlock_args *ap)
int vop_stdlock(struct vop_lock1_args *ap)
int vop_stdneed_inactive(struct vop_need_inactive_args *ap)
void vfs_hash_changesize(u_long newmaxvnodes)
struct vfsconfhead vfsconf
void() NDFREE(struct nameidata *ndp, const u_int flags)
int namei(struct nameidata *ndp)
struct mtx_padalign __exclusive_cache_line mountlist_mtx
void vfs_op_barrier_wait(struct mount *mp)
int dounmount(struct mount *mp, uint64_t flags, struct thread *td)
void vfs_rel(struct mount *mp)
void vfs_ref(struct mount *mp)
int vfs_mount_fetch_counter(struct mount *mp, enum mount_counter which)
struct vnode * vnlru_alloc_marker(void)
static int sysctl_vfs_worklist_len(SYSCTL_HANDLER_ARGS)
void vop_whiteout_post(void *ap, int rc)
static __inline void vfs_freevnodes_dec(void)
static void vdropl_recycle(struct vnode *vp)
static struct filterops vfswrite_filtops
static int sync_vnode_count
static void __always_inline vdropl_impl(struct vnode *vp, bool enqueue)
void vhold(struct vnode *vp)
int vfs_kqfilter(struct vop_kqfilter_args *ap)
static void vdefer_inactive_unlocked(struct vnode *vp)
void vop_symlink_pre(void *ap)
void vop_rename_fail(struct vop_rename_args *ap)
static void vn_free_marker(struct vnode *vp)
static struct proc * vnlruproc
static struct filterops vfsvnode_filtops
static int filt_vfsread(struct knote *kn, long hint)
void vop_mknod_pre(void *ap)
static u_long vnlru_read_freevnodes(void)
struct vnode * __mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, void *cbarg)
static int sync_vnode(struct synclist *slp, struct bufobj **bo, struct thread *td)
int getnewvnode(const char *tag, struct mount *mp, struct vop_vector *vops, struct vnode **vpp)
void v_addpollinfo(struct vnode *vp)
static counter_u64_t vnodes_created
enum vgetstate vget_prep_smr(struct vnode *vp)
struct buf * gbincore_unlocked(struct bufobj *bo, daddr_t lblkno)
static int filt_fsevent(struct knote *kn, long hint)
int vn_dir_check_exec(struct vnode *vp, struct componentname *cnp)
static int sysctl_vfs_conflist(SYSCTL_HANDLER_ARGS)
static bool vhold_recycle_free(struct vnode *)
static int vfs_sysctl(SYSCTL_HANDLER_ARGS)
static u_long __exclusive_cache_line numvnodes
static int vfsconf2x(struct sysctl_req *req, struct vfsconf *vfsp)
static struct proc * updateproc
static struct vnode * mnt_vnode_next_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, void *cbarg)
static int filt_vfswrite(struct knote *kn, long hint)
static void vunlazy(struct vnode *vp)
void bgetvp(struct vnode *vp, struct buf *bp)
static void __noinline vdbatch_process(struct vdbatch *vd)
void vfs_unmountall(void)
void vop_read_post(void *ap, int rc)
int vrecycle(struct vnode *vp)
static int sync_inactive(struct vop_inactive_args *)
void vfs_allocate_syncvnode(struct mount *mp)
SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_FIRST, vntblinit, NULL)
static int vfs_periodic_msync_inactive_filter(struct vnode *vp, void *arg __unused)
static struct vop_vector sync_vnodeops
int vrecyclel(struct vnode *vp)
int vtruncbuf(struct vnode *vp, off_t length, int blksize)
SYSCTL_ULONG(_vfs, OID_AUTO, numvnodes, CTLFLAG_RD, &numvnodes, 0, "Number of vnodes in existence")
static void unmount_or_warn(struct mount *mp)
void vfs_periodic(struct mount *mp, int flags)
void vop_readdir_post(void *ap, int rc)
static void mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp)
void vref(struct vnode *vp)
static int vfs_periodic_inactive_filter(struct vnode *vp, void *arg)
void vfs_event_signal(fsid_t *fsid, uint32_t event, intptr_t data __unused)
static struct vnode * vn_alloc(struct mount *mp)
void vop_rmdir_pre(void *ap)
int bnoreuselist(struct bufv *bufv, struct bufobj *bo, daddr_t startn, daddr_t endn)
void __mnt_vnode_markerfree_all(struct vnode **mvp, struct mount *mp)
bool vn_isdisk_error(struct vnode *vp, int *errp)
static void destroy_vpollinfo_free(struct vpollinfo *vi)
void vn_printf(struct vnode *vp, const char *fmt,...)
static int sysctl_wantfreevnodes(SYSCTL_HANDLER_ARGS)
void syncer_suspend(void)
void vop_rename_post(void *ap, int rc)
void vop_link_post(void *ap, int rc)
static int sysctl_ftry_reclaim_vnode(SYSCTL_HANDLER_ARGS)
static void vfs_knllock(void *arg)
static int v_inval_buf_range_locked(struct vnode *vp, struct bufobj *bo, daddr_t startlbn, daddr_t endlbn)
static bool vnlru_under_unlocked(u_long rnumvnodes, u_long limit)
static void __noinline vdropl_final(struct vnode *vp)
static int sync_fsync(struct vop_fsync_args *)
static struct filterops vfsread_filtops
void vop_remove_pre(void *ap)
void vn_irflag_set(struct vnode *vp, short toset)
void vget_finish_ref(struct vnode *vp, enum vgetstate vs)
void vnlru_free_marker(struct vnode *mvp)
static struct vnode * vn_alloc_marker(struct mount *mp)
void getnewvnode_drop_reserve(void)
void vop_link_pre(void *ap)
static int __noinline vfs_cache_root_fallback(struct mount *mp, int flags, struct vnode **vpp)
static int filt_fsattach(struct knote *kn)
void vn_irflag_unset_locked(struct vnode *vp, short tounset)
void vn_seqc_write_begin(struct vnode *vp)
enum vtype iftovt_tab[16]
void vop_setacl_post(void *ap, int rc __unused)
void brelvp(struct buf *bp)
void vn_irflag_set_cond(struct vnode *vp, short toset)
void vn_irflag_set_cond_locked(struct vnode *vp, short toset)
static void vdbatch_dequeue(struct vnode *vp)
int vget_finish(struct vnode *vp, int flags, enum vgetstate vs)
int vn_pollrecord(struct vnode *vp, struct thread *td, int events)
void vop_deleteextattr_post(void *ap, int rc)
static MALLOC_DEFINE(M_VNODE_MARKER, "vnodemarker", "vnode marker")
u_quad_t init_va_filerev(void)
int insmntque(struct vnode *vp, struct mount *mp)
void vop_mkdir_pre(void *ap)
void vop_whiteout_pre(void *ap)
static void vdrop_recycle(struct vnode *vp)
static void delmntque(struct vnode *vp)
static int max_vnlru_free
void __mnt_vnode_markerfree_lazy(struct vnode **mvp, struct mount *mp)
void vfs_unbusy(struct mount *mp)
static volatile int vsmalltrigger
static int vlrureclaim(bool reclaim_nc_src, int trigger, u_long target)
void v_inval_buf_range(struct vnode *vp, daddr_t startlbn, daddr_t endlbn, int blksize)
void vop_mknod_post(void *ap, int rc)
static void vunlazy_gone(struct vnode *vp)
void vget_abort(struct vnode *vp, enum vgetstate vs)
int vn_need_pageq_flush(struct vnode *vp)
static void __noinline vfs_periodic_inactive(struct mount *mp, int flags)
static void vnlru_proc(void)
void vop_setextattr_post(void *ap, int rc)
static void __noinline vfs_periodic_msync_inactive(struct mount *mp, int flags)
int vflush(struct mount *mp, int rootrefs, int flags, struct thread *td)
static void filt_vfsdetach(struct knote *kn)
void vop_deleteextattr_pre(void *ap)
void vfs_cache_root_set(struct mount *mp, struct vnode *vp)
void vnlru_free_vfsops(int count, struct vfsops *mnt_op, struct vnode *mvp)
struct vnode * __mnt_vnode_next_all(struct vnode **mvp, struct mount *mp)
void vop_setattr_post(void *ap, int rc)
static int vinactivef(struct vnode *vp)
int vinactive(struct vnode *vp)
void vop_setacl_pre(void *ap)
static void vfs_knl_assert_lock(void *arg, int what)
void vn_irflag_set_locked(struct vnode *vp, short toset)
static void vnode_fini(void *mem, int size)
SYSCTL_COUNTER_U64(_vfs, OID_AUTO, vnodes_created, CTLFLAG_RD, &vnodes_created, "Number of vnodes created by getnewvnode")
static int syncer_worklist_len
int vfs_suser(struct mount *mp, struct thread *td)
void vrefact(struct vnode *vp)
static int sysctl_try_reclaim_vnode(SYSCTL_HANDLER_ARGS)
static void vdefer_inactive(struct vnode *vp)
static __inline void vfs_freevnodes_inc(void)
int vfs_read_dirent(struct vop_readdir_args *ap, struct dirent *dp, off_t off)
static void destroy_vpollinfo(struct vpollinfo *vi)
void vop_rename_pre(void *ap)
enum vgetstate vget_prep(struct vnode *vp)
static struct vnode *__noinline vn_alloc_hard(struct mount *mp)
static int flushbuflist(struct bufv *bufv, int flags, struct bufobj *bo, int slpflag, int slptimeo)
DPCPU_DEFINE_STATIC(struct vdbatch, vd)
struct filterops fs_filtops
void vop_open_post(void *ap, int rc)
static void buf_trie_free(struct pctrie *ptree, void *node)
static int sysctl_maxvnodes(SYSCTL_HANDLER_ARGS)
int vaccess(enum vtype type, mode_t file_mode, uid_t file_uid, gid_t file_gid, accmode_t accmode, struct ucred *cred)
void vgone(struct vnode *vp)
static enum @14 syncer_state
void vop_setattr_pre(void *ap)
static int sysctl_ovfs_conf(SYSCTL_HANDLER_ARGS)
static int sync_reclaim(struct vop_reclaim_args *)
#define SYNCER_SHUTDOWN_SPEEDUP
bool vn_isdisk(struct vnode *vp)
static int filt_vfsvnode(struct knote *kn, long hint)
void vop_close_post(void *ap, int rc)
static struct kproc_desc up_kp
void vrele(struct vnode *vp)
int bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
void vattr_null(struct vattr *vap)
static int vnlru_free_impl(int count, struct vfsops *mnt_op, struct vnode *mvp)
static SYSCTL_NODE(_vfs, VFS_GENERIC, generic, CTLFLAG_RD|CTLFLAG_SKIP|CTLFLAG_MPSAFE, vfs_sysctl, "Generic filesystem")
static bool mnt_vnode_next_lazy_relock(struct vnode *mvp, struct mount *mp, struct vnode *vp)
struct mount * vfs_getvfs(fsid_t *fsid)
static int vtryrecycle(struct vnode *vp)
_Static_assert(sizeof(struct vnode) >=1UL<< vnsz2log &&sizeof(struct vnode)< 1UL<<(vnsz2log+1), "vnsz2log needs to be updated")
static u_long vn_alloc_cyclecount
void vfs_timestamp(struct timespec *tsp)
static void vn_seqc_init(struct vnode *)
void vn_seqc_write_end_locked(struct vnode *vp)
void vput(struct vnode *vp)
static void vnlru_recalc(void)
struct vnode * vfs_cache_root_clear(struct mount *mp)
void reassignbuf(struct buf *bp)
static int insmntque1_int(struct vnode *vp, struct mount *mp, bool dtr)
static int timestamp_precision
struct vnode * __mnt_vnode_first_lazy(struct vnode **mvp, struct mount *mp, mnt_lazy_cb_t *cb, void *cbarg)
int vfs_busy(struct mount *mp, int flags)
static void buf_vlist_remove(struct buf *bp)
int vinvalbuf(struct vnode *vp, int flags, int slpflag, int slptimeo)
void vop_symlink_post(void *ap, int rc)
static void vntblinit(void *dummy __unused)
bool vhold_smr(struct vnode *vp)
static void filt_fsdetach(struct knote *kn)
static void syncer_shutdown(void *arg, int howto)
static void vnlru_kick(void)
static bool vfs_want_msync(struct vnode *vp)
void vholdnz(struct vnode *vp)
int vaccess_vexec_smr(mode_t file_mode, uid_t file_uid, gid_t file_gid, struct ucred *cred)
void vn_irflag_unset(struct vnode *vp, short tounset)
static TAILQ_HEAD(freelst, vnode)
void vop_reclaim_post(void *ap, int rc)
static void vn_syncer_add_to_worklist(struct bufobj *bo, int delay)
static void vdbatch_enqueue(struct vnode *vp)
static void vn_free(struct vnode *vp)
int vfs_emptydir(struct vnode *vp)
static int vnode_init(void *mem, int size, int flags)
void vn_seqc_write_end(struct vnode *vp)
void vop_create_post(void *ap, int rc)
SYSCTL_INT(_debug, OID_AUTO, vnlru_nowhere, CTLFLAG_RW, &vnlru_nowhere, 0, "Number of times the vnlru process ran without success")
static struct kproc_desc vnlru_kp
static bool vnlru_under(u_long rnumvnodes, u_long limit)
static void sched_sync(void)
static struct knlist fs_knlist
struct buf * gbincore(struct bufobj *bo, daddr_t lblkno)
void vunref(struct vnode *vp)
void vlazy(struct vnode *vp)
static int vnlru_free_locked(int count)
void vop_create_pre(void *ap)
struct mount * vfs_busyfs(fsid_t *fsid)
static void vfs_deferred_inactive(struct vnode *vp, int lkflags)
PCTRIE_DEFINE_SMR(BUF, buf, b_lblkno, buf_trie_alloc, buf_trie_free, buf_trie_smr)
void vfs_getnewfsid(struct mount *mp)
void vop_setextattr_pre(void *ap)
#define VNLRU_FREEVNODES_SLOP
void vop_rmdir_post(void *ap, int rc)
void vn_seqc_write_begin_locked(struct vnode *vp)
int extattr_check_cred(struct vnode *vp, int attrnamespace, struct ucred *cred, struct thread *td, accmode_t accmode)
static void * buf_trie_alloc(struct pctrie *ptree)
static void vput_final(struct vnode *vp, enum vput_op func)
void vdrop(struct vnode *vp)
void vop_remove_post(void *ap, int rc)
static void __noinline freevnode(struct vnode *vp)
void vop_mkdir_post(void *ap, int rc)
void vfs_notify_upper(struct vnode *vp, enum vfs_notify_upper_type event)
static void buf_vlist_add(struct buf *bp, struct bufobj *bo, b_xflags_t xflags)
SYSCTL_PROC(_kern, KERN_MAXVNODES, maxvnodes, CTLTYPE_ULONG|CTLFLAG_MPSAFE|CTLFLAG_RW, NULL, 0, sysctl_maxvnodes, "LU", "Target for maximum number of vnodes")
int vfs_cache_root(struct mount *mp, int flags, struct vnode **vpp)
void vfs_deallocate_syncvnode(struct mount *mp)
struct vnode * __mnt_vnode_first_all(struct vnode **mvp, struct mount *mp)
void vdropl(struct vnode *vp)
VFS_VOP_VECTOR_REGISTER(sync_vnodeops)
static void vfs_knlunlock(void *arg)
static void vn_seqc_write_end_free(struct vnode *vp)
int vfs_unixify_accmode(accmode_t *accmode)
static void vfs_event_init(void *arg)
void getnewvnode_reserve(void)
static void v_init_counters(struct vnode *)
int vget(struct vnode *vp, int flags)
void vop_read_pgcache_post(void *ap, int rc)
int insmntque1(struct vnode *vp, struct mount *mp)
static int sysctl_vfs_ctl(SYSCTL_HANDLER_ARGS)
static void vgonel(struct vnode *)
int getvnode(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
void vn_finished_secondary_write(struct mount *mp)
int vn_start_write(struct vnode *vp, struct mount **mpp, int flags)
int vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
void vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
void vn_finished_write(struct mount *mp)