48#include "opt_hwpmc_hooks.h"
61#include <sys/limits.h>
71#include <sys/resourcevar.h>
72#include <sys/rwlock.h>
75#include <sys/sleepqueue.h>
76#include <sys/sysctl.h>
77#include <sys/ttycom.h>
79#include <sys/syslog.h>
80#include <sys/unistd.h>
82#include <sys/ktrace.h>
84#include <security/audit/audit.h>
85#include <security/mac/mac_framework.h>
88#include <vm/vm_extern.h>
91#include <vm/vm_object.h>
92#include <vm/vm_page.h>
93#include <vm/vm_pager.h>
96#include <sys/pmckern.h>
128 .fo_flags = DFLAG_PASSABLE | DFLAG_SEEKABLE
141 "Enable copying from page cache for reads, avoiding fs");
147SYSCTL_INT(_security_bsd, OID_AUTO, allow_read_dir, CTLFLAG_RW,
149 "Enable read(2) of directory by root for filesystems that support it");
160 return (uio->uio_segflg == UIO_USERSPACE && vp->v_type == VREG &&
161 (mp = vp->v_mount) != NULL &&
177 struct fop_args_tag {
181 struct vop_args_tag {
187static int vn_io_fault1(
struct vnode *vp,
struct uio *uio,
191vn_open(
struct nameidata *ndp,
int *flagp,
int cmode,
struct file *fp)
193 struct thread *td = curthread;
195 return (
vn_open_cred(ndp, flagp, cmode, 0, td->td_ucred, fp));
203 res = ISOPEN | LOCKLEAF;
204 if ((fmode & O_RESOLVE_BENEATH) != 0)
206 if ((fmode & O_EMPTY_PATH) != 0)
208 if ((fmode & FREAD) != 0)
210 if ((fmode & FWRITE) != 0)
212 if ((vn_open_flags & VN_OPEN_NOAUDIT) == 0)
214 if ((vn_open_flags & VN_OPEN_NOCAPCHECK) != 0)
228vn_open_cred(
struct nameidata *ndp,
int *flagp,
int cmode, u_int vn_open_flags,
229 struct ucred *cred,
struct file *fp)
234 struct vattr *vap = &vat;
241 if ((fmode & (O_CREAT | O_EXCL | O_DIRECTORY)) == (O_CREAT |
242 O_EXCL | O_DIRECTORY) ||
243 (fmode & (O_CREAT | O_EMPTY_PATH)) == (O_CREAT | O_EMPTY_PATH))
245 else if ((fmode & (O_CREAT | O_DIRECTORY)) == O_CREAT) {
246 ndp->ni_cnd.cn_nameiop = CREATE;
247 ndp->ni_cnd.cn_flags =
open2nameif(fmode, vn_open_flags);
255 ndp->ni_cnd.cn_flags |= LOCKPARENT | NOCACHE | NC_KEEPPOSENTRY;
256 if ((fmode & O_EXCL) == 0 && (fmode & O_NOFOLLOW) == 0)
257 ndp->ni_cnd.cn_flags |= FOLLOW;
258 if ((vn_open_flags & VN_OPEN_INVFS) == 0)
260 if ((error =
namei(ndp)) != 0)
262 if (ndp->ni_vp == NULL) {
265 vap->va_mode = cmode;
267 vap->va_vaflags |= VA_EXCLUSIVE;
269 NDFREE(ndp, NDF_ONLY_PNBUF);
272 V_XSLEEP | PCATCH)) != 0)
277 if ((vn_open_flags & VN_OPEN_NAMECACHE) != 0)
278 ndp->ni_cnd.cn_flags |= MAKEENTRY;
280 error = mac_vnode_check_create(cred, ndp->ni_dvp,
284 error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
287 if (error == 0 && (fmode & O_EXCL) != 0 &&
288 (fmode & (O_EXLOCK | O_SHLOCK)) != 0) {
290 vp->v_iflag |= VI_FOPENING;
294 VOP_VPUT_PAIR(ndp->ni_dvp, error == 0 ? &vp : NULL,
298 NDFREE(ndp, NDF_ONLY_PNBUF);
299 if (error == ERELOOKUP) {
307 if (ndp->ni_dvp == ndp->ni_vp)
313 if (fmode & O_EXCL) {
317 if (vp->v_type == VDIR) {
324 ndp->ni_cnd.cn_nameiop = LOOKUP;
325 ndp->ni_cnd.cn_flags =
open2nameif(fmode, vn_open_flags);
326 ndp->ni_cnd.cn_flags |= (fmode & O_NOFOLLOW) != 0 ? NOFOLLOW :
328 if ((fmode & FWRITE) == 0)
329 ndp->ni_cnd.cn_flags |= LOCKSHARED;
330 if ((error =
namei(ndp)) != 0)
337 vp->v_iflag &= ~VI_FOPENING;
346 NDFREE(ndp, NDF_ONLY_PNBUF);
357 int error, lock_flags,
type;
359 ASSERT_VOP_LOCKED(vp,
"vn_open_vnode_advlock");
360 if ((fmode & (O_EXLOCK | O_SHLOCK)) == 0)
362 KASSERT(fp != NULL, (
"open with flock requires fp"));
363 if (fp->f_type != DTYPE_NONE && fp->f_type != DTYPE_VNODE)
366 lock_flags = VOP_ISLOCKED(vp);
369 lf.l_whence = SEEK_SET;
372 lf.l_type = (fmode & O_EXLOCK) != 0 ? F_WRLCK : F_RDLCK;
374 if ((fmode & FNONBLOCK) == 0)
376 if ((fmode & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL))
378 error = VOP_ADVLOCK(vp, (caddr_t)fp, F_SETLK, &lf,
type);
380 fp->f_flag |= FHASLOCK;
382 vn_lock(vp, lock_flags | LK_RETRY);
392 struct thread *td,
struct file *fp)
397 if (vp->v_type == VLNK) {
398 if ((fmode & O_PATH) == 0 || (fmode & FEXEC) != 0)
401 if (vp->v_type != VDIR && fmode & O_DIRECTORY)
405 if ((fmode & O_PATH) == 0) {
406 if (vp->v_type == VSOCK)
408 if ((fmode & (FWRITE | O_TRUNC)) != 0) {
409 if (vp->v_type == VDIR)
413 if ((fmode & FREAD) != 0)
415 if ((fmode & O_APPEND) && (fmode & FWRITE))
418 if ((fmode & O_CREAT) != 0)
422 if ((fmode & FEXEC) != 0)
425 if ((fmode & O_VERIFY) != 0)
427 error = mac_vnode_check_open(cred, vp,
accmode);
431 accmode &= ~(VCREAT | VVERIFY);
433 if ((fmode & O_CREAT) == 0 &&
accmode != 0) {
434 error = VOP_ACCESS(vp,
accmode, cred, td);
438 if ((fmode & O_PATH) != 0) {
439 if (vp->v_type != VFIFO && vp->v_type != VSOCK &&
440 VOP_ACCESS(vp, VREAD, cred, td) == 0)
441 fp->f_flag |= FKQALLOWED;
445 if (vp->v_type == VFIFO && VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
446 vn_lock(vp, LK_UPGRADE | LK_RETRY);
447 error = VOP_OPEN(vp, fmode, cred, td, fp);
452 if (error == 0 && (fmode & FWRITE) != 0) {
453 error = VOP_ADD_WRITECOUNT(vp, 1);
455 CTR3(KTR_VFS,
"%s: vp %p v_writecount increased to %d",
456 __func__, vp, vp->v_writecount);
472 fp->f_flag |= FOPENFAILED;
475 fp->f_type = DTYPE_VNODE;
484 if (vp->v_type != VFIFO && (fmode & FWRITE) != 0 &&
485 !MNT_EXTENDED_SHARED(vp->v_mount) &&
486 VOP_ISLOCKED(vp) != LK_EXCLUSIVE)
487 vn_lock(vp, LK_UPGRADE | LK_RETRY);
488 (void)VOP_CLOSE(vp, fmode & (FREAD | FWRITE | FEXEC),
493 ASSERT_VOP_LOCKED(vp,
"vn_open_vnode");
507 ASSERT_VOP_LOCKED(vp,
"vn_writechk");
524 struct thread *td,
bool keep_ref)
527 int error, lock_flags;
529 if (vp->v_type != VFIFO && (
flags & FWRITE) == 0 &&
530 MNT_EXTENDED_SHARED(vp->v_mount))
531 lock_flags = LK_SHARED;
533 lock_flags = LK_EXCLUSIVE;
536 vn_lock(vp, lock_flags | LK_RETRY);
537 AUDIT_ARG_VNODE1(vp);
538 if ((
flags & (FWRITE | FOPENFAILED)) == FWRITE) {
539 VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
540 CTR3(KTR_VFS,
"%s: vp %p v_writecount decreased to %d",
541 __func__, vp, vp->v_writecount);
543 error = VOP_CLOSE(vp,
flags, file_cred, td);
568 ASSERT_VOP_LOCKED(fp->f_vnode, __func__);
571 if (fp->f_flag & FRDAHEAD)
572 return (fp->f_seqcount[rw] << IO_SEQSHIFT);
581 if ((uio->uio_offset == 0 && fp->f_seqcount[rw] > 0) ||
582 uio->uio_offset == fp->f_nextoff[rw]) {
592 if (uio->uio_resid >= IO_SEQMAX * 16384)
593 fp->f_seqcount[rw] = IO_SEQMAX;
595 fp->f_seqcount[rw] += howmany(uio->uio_resid, 16384);
596 if (fp->f_seqcount[rw] > IO_SEQMAX)
597 fp->f_seqcount[rw] = IO_SEQMAX;
599 return (fp->f_seqcount[rw] << IO_SEQSHIFT);
603 if (fp->f_seqcount[rw] > 1)
604 fp->f_seqcount[rw] = 1;
606 fp->f_seqcount[rw] = 0;
614vn_rdwr(
enum uio_rw rw,
struct vnode *vp,
void *base,
int len, off_t offset,
615 enum uio_seg segflg,
int ioflg,
struct ucred *active_cred,
616 struct ucred *file_cred, ssize_t *aresid,
struct thread *td)
624 int error, lock_flags;
626 if (offset < 0 && vp->v_type != VCHR)
628 auio.uio_iov = &aiov;
630 aiov.iov_base = base;
632 auio.uio_resid = len;
633 auio.uio_offset = offset;
634 auio.uio_segflg = segflg;
639 if ((ioflg & IO_NODELOCKED) == 0) {
640 if ((ioflg & IO_RANGELOCKED) == 0) {
641 if (rw == UIO_READ) {
642 rl_cookie = vn_rangelock_rlock(
vp, offset,
644 }
else if ((ioflg & IO_APPEND) != 0) {
645 rl_cookie = vn_rangelock_wlock(
vp, 0, OFF_MAX);
647 rl_cookie = vn_rangelock_wlock(
vp, offset,
653 if (rw == UIO_WRITE) {
654 if (
vp->v_type != VCHR &&
660 lock_flags = LK_SHARED;
661 vn_lock(
vp, lock_flags | LK_RETRY);
665 ASSERT_VOP_LOCKED(
vp,
"IO_NODELOCKED with no vp lock held");
667 if ((ioflg & IO_NOMACCHECK) == 0) {
669 error = mac_vnode_check_read(active_cred, file_cred,
672 error = mac_vnode_check_write(active_cred, file_cred,
677 if (file_cred != NULL)
685 args.args.vop_args.vp =
vp;
687 }
else if (rw == UIO_READ) {
688 error = VOP_READ(
vp, &auio, ioflg,
cred);
690 error = VOP_WRITE(
vp, &auio, ioflg,
cred);
694 *aresid = auio.uio_resid;
696 if (auio.uio_resid && error == 0)
698 if ((ioflg & IO_NODELOCKED) == 0) {
704 if (rl_cookie != NULL)
705 vn_rangelock_unlock(
vp, rl_cookie);
719 off_t offset,
enum uio_seg segflg,
int ioflg,
struct ucred *active_cred,
720 struct ucred *file_cred,
size_t *aresid,
struct thread *td)
734 chunk = MAXBSIZE - (uoff_t)offset % MAXBSIZE;
738 if (rw != UIO_READ &&
vp->v_type == VREG)
741 error =
vn_rdwr(rw,
vp, base, chunk, offset, segflg,
742 ioflg, active_cred, file_cred, &iaresid, td);
747 base = (
char *)base + chunk;
751 *aresid = len + iaresid;
755#if OFF_MAX <= LONG_MAX
759 volatile short *flagsp;
763 KASSERT((
flags & FOF_OFFSET) == 0, (
"FOF_OFFSET passed"));
765 if ((
flags & FOF_NOLOCK) != 0)
766 return (atomic_load_long(&
fp->f_offset));
772 flagsp = &
fp->f_vnread_flags;
773 if (atomic_cmpset_acq_16(flagsp, 0, FOFFSET_LOCKED))
774 return (atomic_load_long(&
fp->f_offset));
777 state = atomic_load_16(flagsp);
779 if ((state & FOFFSET_LOCKED) == 0) {
780 if (!atomic_fcmpset_acq_16(flagsp, &state,
785 if ((state & FOFFSET_LOCK_WAITING) == 0) {
786 if (!atomic_fcmpset_acq_16(flagsp, &state,
787 state | FOFFSET_LOCK_WAITING))
791 sleepq_add(&
fp->f_vnread_flags, NULL,
"vofflock", 0, 0);
795 state = atomic_load_16(flagsp);
797 res = atomic_load_long(&
fp->f_offset);
805 volatile short *flagsp;
808 KASSERT((
flags & FOF_OFFSET) == 0, (
"FOF_OFFSET passed"));
810 if ((
flags & FOF_NOUPDATE) == 0)
811 atomic_store_long(&
fp->f_offset, val);
812 if ((
flags & FOF_NEXTOFF_R) != 0)
813 fp->f_nextoff[UIO_READ] = val;
814 if ((
flags & FOF_NEXTOFF_W) != 0)
815 fp->f_nextoff[UIO_WRITE] = val;
817 if ((
flags & FOF_NOLOCK) != 0)
820 flagsp = &
fp->f_vnread_flags;
821 state = atomic_load_16(flagsp);
822 if ((state & FOFFSET_LOCK_WAITING) == 0 &&
823 atomic_cmpset_rel_16(flagsp, state, 0))
827 MPASS((
fp->f_vnread_flags & FOFFSET_LOCKED) != 0);
828 MPASS((
fp->f_vnread_flags & FOFFSET_LOCK_WAITING) != 0);
829 fp->f_vnread_flags = 0;
840 KASSERT((
flags & FOF_OFFSET) == 0, (
"FOF_OFFSET passed"));
844 if ((
flags & FOF_NOLOCK) == 0) {
845 while (fp->f_vnread_flags & FOFFSET_LOCKED) {
846 fp->f_vnread_flags |= FOFFSET_LOCK_WAITING;
847 msleep(&fp->f_vnread_flags, mtxp, PUSER -1,
850 fp->f_vnread_flags |= FOFFSET_LOCKED;
862 KASSERT((
flags & FOF_OFFSET) == 0, (
"FOF_OFFSET passed"));
866 if ((
flags & FOF_NOUPDATE) == 0)
868 if ((
flags & FOF_NEXTOFF_R) != 0)
869 fp->f_nextoff[UIO_READ] = val;
870 if ((
flags & FOF_NEXTOFF_W) != 0)
871 fp->f_nextoff[UIO_WRITE] = val;
872 if ((
flags & FOF_NOLOCK) == 0) {
873 KASSERT((fp->f_vnread_flags & FOFFSET_LOCKED) != 0,
874 (
"Lost FOFFSET_LOCKED"));
875 if (fp->f_vnread_flags & FOFFSET_LOCK_WAITING)
876 wakeup(&fp->f_vnread_flags);
877 fp->f_vnread_flags = 0;
887 if ((
flags & FOF_OFFSET) == 0)
895 if ((
flags & FOF_OFFSET) == 0)
905 ret = POSIX_FADV_NORMAL;
906 if (fp->f_advice == NULL || fp->f_vnode->v_type != VREG)
911 if (fp->f_advice != NULL &&
912 uio->uio_offset >= fp->f_advice->fa_start &&
913 uio->uio_offset + uio->uio_resid <= fp->f_advice->fa_end)
914 ret = fp->f_advice->fa_advice;
928 mp = atomic_load_ptr(&vp->v_mount);
930 if ((fp->f_flag & O_DIRECT) != 0)
933 if ((fp->f_flag & O_FSYNC) != 0 ||
934 (mp != NULL && (mp->mnt_flag & MNT_SYNCHRONOUS) != 0))
942 if ((fp->f_flag & O_DSYNC) != 0)
943 ioflag |= IO_SYNC | IO_DATASYNC;
958 obj = atomic_load_ptr(&vp->v_object);
960 return (EJUSTRETURN);
965 vm_object_pip_add(obj, 1);
966 if ((obj->flags & OBJ_DEAD) != 0) {
981 resid = uio->uio_resid;
982 off = uio->uio_offset;
983 for (i = 0; resid > 0; i++) {
985 ma[i] = vm_page_grab_unlocked(obj, atop(off),
986 VM_ALLOC_NOCREAT | VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY |
995 if (vm_page_none_valid(ma[i])) {
996 vm_page_sunbusy(ma[i]);
1004 error = EJUSTRETURN;
1013 if (VN_IS_DOOMED(vp)) {
1014 error = EJUSTRETURN;
1018 resid = PAGE_SIZE - (uio->uio_offset & PAGE_MASK) + ptoa(i - 1);
1019 if (resid > uio->uio_resid)
1020 resid = uio->uio_resid;
1028#if defined(__powerpc__) && !defined(__powerpc64__)
1029 vsz = obj->un_pager.vnp.vnp_size;
1031 vsz = atomic_load_64(&obj->un_pager.vnp.vnp_size);
1033 if (uio->uio_offset >= vsz) {
1034 error = EJUSTRETURN;
1037 if (uio->uio_offset + resid > vsz)
1038 resid = vsz - uio->uio_offset;
1043 for (j = 0; j < i; j++) {
1045 vm_page_reference(ma[j]);
1046 vm_page_sunbusy(ma[j]);
1049 vm_object_pip_wakeup(obj);
1052 return (uio->uio_resid == 0 ? 0 : EJUSTRETURN);
1059vn_read(
struct file *fp,
struct uio *uio,
struct ucred *active_cred,
int flags,
1067 KASSERT(uio->uio_td == td, (
"uio_td %p is not td %p",
1069 KASSERT(
flags & FOF_OFFSET, (
"No FOF_OFFSET"));
1072 if (fp->f_flag & FNONBLOCK)
1073 ioflag |= IO_NDELAY;
1074 if (fp->f_flag & O_DIRECT)
1075 ioflag |= IO_DIRECT;
1082 (vn_irflag_read(vp) & (VIRF_DOOMED | VIRF_PGREAD)) == VIRF_PGREAD) {
1083 error = VOP_READ_PGCACHE(vp, uio, ioflag, fp->f_cred);
1085 fp->f_nextoff[UIO_READ] = uio->uio_offset;
1088 if (error != EJUSTRETURN)
1093 vn_lock(vp, LK_SHARED | LK_RETRY);
1096 case POSIX_FADV_NORMAL:
1097 case POSIX_FADV_SEQUENTIAL:
1098 case POSIX_FADV_NOREUSE:
1101 case POSIX_FADV_RANDOM:
1105 orig_offset = uio->uio_offset;
1108 error = mac_vnode_check_read(active_cred, fp->f_cred, vp);
1111 error = VOP_READ(vp, uio, ioflag, fp->f_cred);
1112 fp->f_nextoff[UIO_READ] = uio->uio_offset;
1114 if (error == 0 && advice == POSIX_FADV_NOREUSE &&
1115 orig_offset != uio->uio_offset)
1121 error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
1122 POSIX_FADV_DONTNEED);
1138 bool need_finished_write;
1140 KASSERT(uio->uio_td == td, (
"uio_td %p is not td %p",
1142 KASSERT(
flags & FOF_OFFSET, (
"No FOF_OFFSET"));
1144 if (vp->v_type == VREG)
1147 if (vp->v_type == VREG && (fp->f_flag & O_APPEND) != 0)
1148 ioflag |= IO_APPEND;
1149 if ((fp->f_flag & FNONBLOCK) != 0)
1150 ioflag |= IO_NDELAY;
1154 need_finished_write =
false;
1155 if (vp->v_type != VCHR) {
1159 need_finished_write =
true;
1166 case POSIX_FADV_NORMAL:
1167 case POSIX_FADV_SEQUENTIAL:
1168 case POSIX_FADV_NOREUSE:
1171 case POSIX_FADV_RANDOM:
1175 orig_offset = uio->uio_offset;
1178 error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
1181 error = VOP_WRITE(vp, uio, ioflag, fp->f_cred);
1182 fp->f_nextoff[UIO_WRITE] = uio->uio_offset;
1184 if (need_finished_write)
1186 if (error == 0 && advice == POSIX_FADV_NOREUSE &&
1187 orig_offset != uio->uio_offset)
1193 error = VOP_ADVISE(vp, orig_offset, uio->uio_offset - 1,
1194 POSIX_FADV_DONTNEED);
1243 save = vm_fault_disable_pagefaults();
1244 switch (args->
kind) {
1245 case VN_IO_FAULT_FOP:
1249 case VN_IO_FAULT_VOP:
1250 if (uio->uio_rw == UIO_READ) {
1253 }
else if (uio->uio_rw == UIO_WRITE) {
1259 panic(
"vn_io_fault_doio: unknown kind of io %d %d",
1260 args->
kind, uio->uio_rw);
1262 vm_fault_enable_pagefaults(save);
1272 if (r == -1 || (uio->uio_rw == UIO_READ && subyte(base, r) == -1))
1281 const struct iovec *iov;
1286 KASSERT(uio->uio_segflg == UIO_USERSPACE,
1287 (
"vn_io_fault_prefault userspace"));
1291 resid = uio->uio_resid;
1292 base = iov->iov_base;
1298 if (len < PAGE_SIZE) {
1305 if (++i >= uio->uio_iovcnt)
1307 iov = uio->uio_iov + i;
1308 base = iov->iov_base;
1331 struct uio *uio_clone, short_uio;
1332 struct iovec short_iovec[1];
1333 vm_page_t *prev_td_ma;
1335 vm_offset_t
addr, end;
1338 int error, cnt, saveheld, prev_td_ma_cnt;
1346 prot = uio->uio_rw == UIO_READ ? VM_PROT_WRITE : VM_PROT_READ;
1358 resid = uio->uio_resid;
1360 short_uio.uio_segflg = UIO_USERSPACE;
1361 short_uio.uio_rw = uio->uio_rw;
1362 short_uio.uio_td = uio->uio_td;
1365 if (error != EFAULT)
1369 uio_clone->uio_segflg = UIO_NOCOPY;
1370 uiomove(NULL, resid - uio->uio_resid, uio_clone);
1371 uio_clone->uio_segflg = uio->uio_segflg;
1373 saveheld = curthread_pflags_set(TDP_UIOHELD);
1374 prev_td_ma = td->td_ma;
1375 prev_td_ma_cnt = td->td_ma_cnt;
1377 while (uio_clone->uio_resid != 0) {
1378 len = uio_clone->uio_iov->iov_len;
1380 KASSERT(uio_clone->uio_iovcnt >= 1,
1381 (
"iovcnt underflow"));
1382 uio_clone->uio_iov++;
1383 uio_clone->uio_iovcnt--;
1388 addr = (uintptr_t)uio_clone->uio_iov->iov_base;
1389 end = round_page(
addr + len);
1394 cnt = atop(end - trunc_page(
addr));
1400 cnt = vm_fault_quick_hold_pages(&td->td_proc->p_vmspace->vm_map,
1406 short_uio.uio_iov = &short_iovec[0];
1407 short_iovec[0].iov_base = (
void *)
addr;
1408 short_uio.uio_iovcnt = 1;
1409 short_uio.uio_resid = short_iovec[0].iov_len = len;
1410 short_uio.uio_offset = uio_clone->uio_offset;
1412 td->td_ma_cnt = cnt;
1415 vm_page_unhold_pages(ma, cnt);
1416 adv = len - short_uio.uio_resid;
1418 uio_clone->uio_iov->iov_base =
1419 (
char *)uio_clone->uio_iov->iov_base + adv;
1420 uio_clone->uio_iov->iov_len -= adv;
1421 uio_clone->uio_resid -= adv;
1422 uio_clone->uio_offset += adv;
1424 uio->uio_resid -= adv;
1425 uio->uio_offset += adv;
1427 if (error != 0 || adv == 0)
1430 td->td_ma = prev_td_ma;
1431 td->td_ma_cnt = prev_td_ma_cnt;
1432 curthread_pflags_restore(saveheld);
1434 free(uio_clone, M_IOV);
1439vn_io_fault(
struct file *fp,
struct uio *uio,
struct ucred *active_cred,
1440 int flags,
struct thread *td)
1459 if (
vp->v_type == VDIR) {
1460 KASSERT(uio->uio_rw == UIO_READ,
1461 (
"illegal write attempted on a directory"));
1464 if ((error =
priv_check(td, PRIV_VFS_READ_DIR)) != 0)
1471 args.args.fop_args.fp =
fp;
1473 args.cred = active_cred;
1475 if (uio->uio_rw == UIO_READ) {
1476 rl_cookie = vn_rangelock_rlock(
vp, uio->uio_offset,
1477 uio->uio_offset + uio->uio_resid);
1478 }
else if ((
fp->f_flag & O_APPEND) != 0 ||
1479 (
flags & FOF_OFFSET) == 0) {
1481 rl_cookie = vn_rangelock_wlock(
vp, 0, OFF_MAX);
1483 rl_cookie = vn_rangelock_wlock(
vp, uio->uio_offset,
1484 uio->uio_offset + uio->uio_resid);
1487 vn_rangelock_unlock(
vp, rl_cookie);
1489 error =
doio(
fp, uio, active_cred,
flags | FOF_OFFSET, td);
1510 struct uio transp_uio;
1511 struct iovec transp_iov[1];
1517 if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1518 uio->uio_segflg != UIO_USERSPACE)
1521 KASSERT(uio->uio_iovcnt == 1, (
"uio_iovcnt %d", uio->uio_iovcnt));
1522 transp_iov[0].iov_base =
data;
1523 transp_uio.uio_iov = &transp_iov[0];
1524 transp_uio.uio_iovcnt = 1;
1525 if (xfersize > uio->uio_resid)
1526 xfersize = uio->uio_resid;
1527 transp_uio.uio_resid = transp_iov[0].iov_len = xfersize;
1528 transp_uio.uio_offset = 0;
1529 transp_uio.uio_segflg = UIO_SYSSPACE;
1536 switch (uio->uio_rw) {
1538 transp_uio.uio_rw = UIO_READ;
1541 transp_uio.uio_rw = UIO_WRITE;
1544 transp_uio.uio_td = uio->uio_td;
1545 error = uiomove_fromphys(td->td_ma,
1546 ((vm_offset_t)uio->uio_iov->iov_base) & PAGE_MASK,
1547 xfersize, &transp_uio);
1548 adv = xfersize - transp_uio.uio_resid;
1550 (((vm_offset_t)uio->uio_iov->iov_base + adv) >> PAGE_SHIFT) -
1551 (((vm_offset_t)uio->uio_iov->iov_base) >> PAGE_SHIFT);
1553 KASSERT(td->td_ma_cnt >= pgadv, (
"consumed pages %d %d", td->td_ma_cnt,
1555 td->td_ma_cnt -= pgadv;
1556 uio->uio_iov->iov_base = (
char *)uio->uio_iov->iov_base + adv;
1557 uio->uio_iov->iov_len -= adv;
1558 uio->uio_resid -= adv;
1559 uio->uio_offset += adv;
1568 vm_offset_t iov_base;
1572 if ((td->td_pflags & TDP_UIOHELD) == 0 ||
1573 uio->uio_segflg != UIO_USERSPACE)
1574 return (uiomove_fromphys(ma, offset, xfersize, uio));
1576 KASSERT(uio->uio_iovcnt == 1, (
"uio_iovcnt %d", uio->uio_iovcnt));
1577 cnt = xfersize > uio->uio_resid ? uio->uio_resid : xfersize;
1578 iov_base = (vm_offset_t)uio->uio_iov->iov_base;
1579 switch (uio->uio_rw) {
1581 pmap_copy_pages(td->td_ma, iov_base & PAGE_MASK, ma,
1585 pmap_copy_pages(ma, offset, td->td_ma, iov_base & PAGE_MASK,
1589 pgadv = ((iov_base + cnt) >> PAGE_SHIFT) - (iov_base >> PAGE_SHIFT);
1591 KASSERT(td->td_ma_cnt >= pgadv, (
"consumed pages %d %d", td->td_ma_cnt,
1593 td->td_ma_cnt -= pgadv;
1594 uio->uio_iov->iov_base = (
char *)(iov_base + cnt);
1595 uio->uio_iov->iov_len -= cnt;
1596 uio->uio_resid -= cnt;
1597 uio->uio_offset += cnt;
1620 rl_cookie = vn_rangelock_wlock(vp, 0, OFF_MAX);
1624 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1625 AUDIT_ARG_VNODE1(vp);
1626 if (vp->v_type == VDIR) {
1631 error = mac_vnode_check_write(active_cred, fp->f_cred, vp);
1641 vn_rangelock_unlock(vp, rl_cookie);
1642 if (error == ERELOOKUP)
1657 error = VOP_ADD_WRITECOUNT(vp, 1);
1660 vattr.va_size = length;
1662 vattr.va_vaflags |= VA_SYNC;
1663 error = VOP_SETATTR(vp, &vattr, cred);
1664 VOP_ADD_WRITECOUNT_CHECKED(vp, -1);
1673vn_statfile(
struct file *fp,
struct stat *sb,
struct ucred *active_cred)
1675 struct vnode *vp = fp->f_vnode;
1678 vn_lock(vp, LK_SHARED | LK_RETRY);
1679 error = VOP_STAT(vp, sb, active_cred, fp->f_cred);
1689vn_ioctl(
struct file *fp, u_long com,
void *
data,
struct ucred *active_cred,
1694 struct fiobmap2_arg *bmarg;
1698 switch (vp->v_type) {
1703 vn_lock(vp, LK_SHARED | LK_RETRY);
1704 error = VOP_GETATTR(vp, &vattr, active_cred);
1707 *(
int *)
data = vattr.va_size - fp->f_offset;
1710 bmarg = (
struct fiobmap2_arg *)
data;
1711 vn_lock(vp, LK_SHARED | LK_RETRY);
1713 error = mac_vnode_check_read(active_cred, fp->f_cred,
1717 error = VOP_BMAP(vp, bmarg->bn, NULL,
1718 &bmarg->bn, &bmarg->runp, &bmarg->runb);
1725 return (VOP_IOCTL(vp, com,
data, fp->f_flag,
1730 return (VOP_IOCTL(vp, com,
data, fp->f_flag,
1741vn_poll(
struct file *fp,
int events,
struct ucred *active_cred,
1748#if defined(MAC) || defined(AUDIT)
1749 if (AUDITING_TD(td) || mac_vnode_check_poll_enabled()) {
1750 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1751 AUDIT_ARG_VNODE1(vp);
1752 error = mac_vnode_check_poll(active_cred, fp->f_cred, vp);
1758 error = VOP_POLL(vp, events, fp->f_cred, td);
1766static int __noinline
1771 KASSERT((
flags & LK_RETRY) == 0 || error == 0,
1772 (
"vn_lock: error %d incompatible with flags %#x", error,
flags));
1775 VNASSERT(VN_IS_DOOMED(vp), vp, (
"vnode not doomed"));
1777 if ((
flags & LK_RETRY) == 0) {
1796 flags &= ~LK_INTERLOCK;
1798 error = VOP_LOCK1(vp,
flags, file, line);
1799 }
while (error != 0);
1808 VNASSERT((
flags & LK_TYPE_MASK) != 0, vp,
1809 (
"vn_lock: no locktype (%d passed)",
flags));
1810 VNPASS(vp->v_holdcnt > 0, vp);
1811 error = VOP_LOCK1(vp,
flags, file, line);
1812 if (__predict_false(error != 0 || VN_IS_DOOMED(vp)))
1830 ref = (fp->f_flag & FHASLOCK) != 0;
1832 error =
vn_close1(vp, fp->f_flag, fp->f_cred, td, ref);
1834 if (__predict_false(ref)) {
1835 lf.l_whence = SEEK_SET;
1838 lf.l_type = F_UNLCK;
1839 (void) VOP_ADVLOCK(vp, fp, F_UNLCK, &lf, F_FLOCK);
1854 struct mount_pcpu *mpcpu;
1857 if (__predict_true(!mplocked) && (
flags & V_XSLEEP) == 0 &&
1858 vfs_op_thread_enter(mp, mpcpu)) {
1859 MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
1860 vfs_mp_count_add_pcpu(mpcpu, writeopcount, 1);
1861 vfs_op_thread_exit(mp, mpcpu);
1866 mtx_assert(MNT_MTX(mp), MA_OWNED);
1875 if ((curthread->td_pflags & TDP_IGNSUSP) == 0 ||
1876 mp->mnt_susp_owner != curthread) {
1877 mflags = ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ?
1878 (
flags & PCATCH) : 0) | (PUSER - 1);
1879 while ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
1880 if (
flags & V_NOWAIT) {
1881 error = EWOULDBLOCK;
1884 error = msleep(&mp->mnt_flag, MNT_MTX(mp), mflags,
1890 if (
flags & V_XSLEEP)
1892 mp->mnt_writeopcount++;
1894 if (error != 0 || (
flags & V_XSLEEP) != 0)
1906 KASSERT((
flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1907 (
"V_MNTREF requires mp"));
1915 if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1917 if (error != EOPNOTSUPP)
1922 if ((mp = *mpp) == NULL)
1932 if (vp == NULL && (
flags & V_MNTREF) == 0)
1951 KASSERT((
flags & V_MNTREF) == 0 || (*mpp != NULL && vp == NULL),
1952 (
"V_MNTREF requires mp"));
1956 if ((error = VOP_GETWRITEMOUNT(vp, mpp)) != 0) {
1958 if (error != EOPNOTSUPP)
1967 if ((mp = *mpp) == NULL)
1978 if (vp == NULL && (
flags & V_MNTREF) == 0)
1980 if ((mp->mnt_kern_flag & (MNTK_SUSPENDED | MNTK_SUSPEND2)) == 0) {
1981 mp->mnt_secondary_writes++;
1982 mp->mnt_secondary_accwrites++;
1986 if (
flags & V_NOWAIT) {
1989 return (EWOULDBLOCK);
1994 error = msleep(&mp->mnt_flag, MNT_MTX(mp), (PUSER - 1) | PDROP |
1995 ((mp->mnt_vfc->vfc_flags & VFCF_SBDRY) != 0 ? (
flags & PCATCH) : 0),
2011 struct mount_pcpu *mpcpu;
2017 if (vfs_op_thread_enter(mp, mpcpu)) {
2018 vfs_mp_count_sub_pcpu(mpcpu, writeopcount, 1);
2019 vfs_mp_count_sub_pcpu(mpcpu, ref, 1);
2020 vfs_op_thread_exit(mp, mpcpu);
2025 vfs_assert_mount_counters(mp);
2027 c = --mp->mnt_writeopcount;
2028 if (mp->mnt_vfs_ops == 0) {
2029 MPASS((mp->mnt_kern_flag & MNTK_SUSPEND) == 0);
2034 vfs_dump_mount_counters(mp);
2035 if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 && c == 0)
2036 wakeup(&mp->mnt_writeopcount);
2052 mp->mnt_secondary_writes--;
2053 if (mp->mnt_secondary_writes < 0)
2054 panic(
"vn_finished_secondary_write: neg cnt");
2055 if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0 &&
2056 mp->mnt_secondary_writes <= 0)
2057 wakeup(&mp->mnt_secondary_writes);
2072 vfs_assert_mount_counters(mp);
2073 if (mp->mnt_susp_owner == curthread) {
2078 while (mp->mnt_kern_flag & MNTK_SUSPEND)
2079 msleep(&mp->mnt_flag, MNT_MTX(mp), PUSER - 1,
"wsuspfs", 0);
2089 if ((
flags & VS_SKIP_UNMOUNT) != 0 &&
2090 (mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) {
2096 mp->mnt_kern_flag |= MNTK_SUSPEND;
2097 mp->mnt_susp_owner = curthread;
2098 if (mp->mnt_writeopcount > 0)
2099 (void) msleep(&mp->mnt_writeopcount,
2100 MNT_MTX(mp), (PUSER - 1)|PDROP,
"suspwt", 0);
2103 if ((error = VFS_SYNC(mp, MNT_SUSPEND)) != 0) {
2118 if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) {
2119 KASSERT(mp->mnt_susp_owner == curthread, (
"mnt_susp_owner"));
2120 mp->mnt_kern_flag &= ~(MNTK_SUSPEND | MNTK_SUSPEND2 |
2122 mp->mnt_susp_owner = NULL;
2123 wakeup(&mp->mnt_writeopcount);
2125 curthread->td_pflags &= ~TDP_IGNSUSP;
2126 if ((
flags & VR_START_WRITE) != 0) {
2128 mp->mnt_writeopcount++;
2131 if ((
flags & VR_NO_SUSPCLR) == 0)
2134 }
else if ((
flags & VR_START_WRITE) != 0) {
2151 KASSERT((curthread->td_pflags & TDP_IGNSUSP) == 0,
2152 (
"vfs_write_suspend_umnt: recursed"));
2163 if ((mp->mnt_kern_flag & MNTK_SUSPENDED) != 0)
2168 mp->mnt_kern_flag &= ~(MNTK_SUSPENDED | MNTK_SUSPEND2);
2171 curthread->td_pflags |= TDP_IGNSUSP;
2182 return (VOP_KQFILTER(fp->f_vnode, kn));
2188 if ((fp->f_flag & FKQALLOWED) == 0)
2200 const char *attrname,
int *buflen,
char *
buf,
struct thread *td)
2206 iov.iov_len = *buflen;
2209 auio.uio_iov = &iov;
2210 auio.uio_iovcnt = 1;
2211 auio.uio_rw = UIO_READ;
2212 auio.uio_segflg = UIO_SYSSPACE;
2214 auio.uio_offset = 0;
2215 auio.uio_resid = *buflen;
2217 if ((ioflg & IO_NODELOCKED) == 0)
2218 vn_lock(vp, LK_SHARED | LK_RETRY);
2220 ASSERT_VOP_LOCKED(vp,
"IO_NODELOCKED with no vp lock held");
2223 error = VOP_GETEXTATTR(vp,
attrnamespace, attrname, &auio, NULL, NULL,
2226 if ((ioflg & IO_NODELOCKED) == 0)
2230 *buflen = *buflen - auio.uio_resid;
2241 const char *attrname,
int buflen,
char *
buf,
struct thread *td)
2248 iov.iov_len = buflen;
2251 auio.uio_iov = &iov;
2252 auio.uio_iovcnt = 1;
2253 auio.uio_rw = UIO_WRITE;
2254 auio.uio_segflg = UIO_SYSSPACE;
2256 auio.uio_offset = 0;
2257 auio.uio_resid = buflen;
2259 if ((ioflg & IO_NODELOCKED) == 0) {
2262 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2265 ASSERT_VOP_LOCKED(vp,
"IO_NODELOCKED with no vp lock held");
2268 error = VOP_SETEXTATTR(vp,
attrnamespace, attrname, &auio, NULL, td);
2270 if ((ioflg & IO_NODELOCKED) == 0) {
2280 const char *attrname,
struct thread *td)
2285 if ((ioflg & IO_NODELOCKED) == 0) {
2288 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2291 ASSERT_VOP_LOCKED(vp,
"IO_NODELOCKED with no vp lock held");
2294 error = VOP_DELETEEXTATTR(vp,
attrnamespace, attrname, NULL, td);
2295 if (error == EOPNOTSUPP)
2299 if ((ioflg & IO_NODELOCKED) == 0) {
2312 return (VFS_VGET(mp, *(ino_t *)arg, lkflags, rvp));
2316vn_vget_ino(
struct vnode *vp, ino_t ino,
int lkflags,
struct vnode **rvp)
2325 int lkflags,
struct vnode **rvp)
2330 ASSERT_VOP_LOCKED(vp,
"vn_vget_ino_get");
2332 ltype = VOP_ISLOCKED(vp);
2333 KASSERT(ltype == LK_EXCLUSIVE || ltype == LK_SHARED,
2334 (
"vn_vget_ino: vp not locked"));
2340 vn_lock(vp, ltype | LK_RETRY);
2344 if (VN_IS_DOOMED(vp)) {
2350 error = alloc(mp, alloc_arg, lkflags, rvp);
2352 if (error != 0 || *rvp != vp)
2353 vn_lock(vp, ltype | LK_RETRY);
2354 if (VN_IS_DOOMED(vp)) {
2380 ktr_write = (td->td_pflags & TDP_INKTRACE) != 0;
2381 lim =
lim_cur(td, RLIMIT_FSIZE);
2382 if (__predict_false(ktr_write))
2383 lim = td->td_ktr_io_lim;
2384 if (__predict_true((uoff_t)uio->uio_offset + uio->uio_resid <= lim))
2390 if (vp->v_type != VREG ||
2391 (td->td_pflags2 & TDP2_ACCT) != 0)
2394 if (!ktr_write || ktr_filesize_limit_signal) {
2395 PROC_LOCK(td->td_proc);
2397 PROC_UNLOCK(td->td_proc);
2410 vn_lock(vp, LK_SHARED | LK_RETRY);
2411 AUDIT_ARG_VNODE1(vp);
2418vn_chown(
struct file *fp, uid_t uid, gid_t gid,
struct ucred *active_cred,
2425 vn_lock(vp, LK_SHARED | LK_RETRY);
2426 AUDIT_ARG_VNODE1(vp);
2429 return (
setfown(td, active_cred, vp, uid, gid));
2441 if ((
object = vp->v_object) == NULL)
2443 VM_OBJECT_WLOCK(
object);
2444 vm_object_page_remove(
object,
start, end, 0);
2445 VM_OBJECT_WUNLOCK(
object);
2459 if ((
object = vp->v_object) == NULL)
2461 VM_OBJECT_WLOCK(
object);
2462 vm_object_page_remove(
object,
start, end, OBJPR_VALIDONLY);
2463 VM_OBJECT_WUNLOCK(
object);
2476 KASSERT(cmd == FIOSEEKHOLE || cmd == FIOSEEKDATA,
2477 (
"%s: Wrong command %lu", __func__, cmd));
2478 ASSERT_VOP_LOCKED(vp,
"vn_bmap_seekhole_locked");
2480 if (vp->v_type != VREG) {
2484 error = VOP_GETATTR(vp, &va, cred);
2488 if (noff >= va.va_size) {
2492 bsize = vp->v_mount->mnt_stat.f_iosize;
2493 for (bn = noff / bsize; noff < va.va_size; bn++, noff += bsize -
2495 error = VOP_BMAP(vp, bn, NULL, &bnp, NULL, NULL);
2496 if (error == EOPNOTSUPP) {
2500 if ((bnp == -1 && cmd == FIOSEEKHOLE) ||
2501 (bnp != -1 && cmd == FIOSEEKDATA)) {
2508 if (noff > va.va_size)
2511 if (cmd == FIOSEEKDATA)
2524 KASSERT(cmd == FIOSEEKHOLE || cmd == FIOSEEKDATA,
2525 (
"%s: Wrong command %lu", __func__, cmd));
2527 if (vn_lock(vp, LK_SHARED) != 0)
2535vn_seek(
struct file *fp, off_t offset,
int whence,
struct thread *td)
2540 off_t foffset, size;
2543 cred = td->td_ucred;
2546 noneg = (vp->v_type != VCHR);
2552 (offset > 0 && foffset > OFF_MAX - offset))) {
2559 vn_lock(vp, LK_SHARED | LK_RETRY);
2560 error = VOP_GETATTR(vp, &vattr, cred);
2570 if (vattr.va_size == 0 && vp->v_type == VCHR &&
2571 fo_ioctl(fp, DIOCGMEDIASIZE, &size, cred, td) == 0)
2572 vattr.va_size = size;
2574 (vattr.va_size > OFF_MAX ||
2575 (offset > 0 && vattr.va_size > OFF_MAX - offset))) {
2579 offset += vattr.va_size;
2584 error = fo_ioctl(fp, FIOSEEKDATA, &offset, cred, td);
2585 if (error == ENOTTY)
2589 error = fo_ioctl(fp, FIOSEEKHOLE, &offset, cred, td);
2590 if (error == ENOTTY)
2596 if (error == 0 && noneg && offset < 0)
2600 VFS_KNOTE_UNLOCKED(vp, 0);
2601 td->td_uretoff.tdu_off = offset;
2624 error = VOP_ACCESSX(vp, VWRITE_ATTRIBUTES, cred, td);
2625 if (error != 0 && (vap->va_vaflags & VA_UTIMES_NULL) != 0)
2626 error = VOP_ACCESS(vp, VWRITE, cred, td);
2636 if (fp->f_type == DTYPE_FIFO)
2637 kif->kf_type = KF_TYPE_FIFO;
2639 kif->kf_type = KF_TYPE_VNODE;
2642 FILEDESC_SUNLOCK(fdp);
2645 FILEDESC_SLOCK(fdp);
2658 len = (arc4random() % (
sizeof(kif->kf_path) - 2)) + 1;
2659 olen = strlen(kif->kf_path);
2661 strcpy(&kif->kf_path[len - 1],
"$");
2663 for (; olen < len; olen++)
2664 strcpy(&kif->kf_path[olen],
"A");
2671 char *fullpath, *freepath;
2679 strlcpy(kif->kf_path, fullpath,
sizeof(kif->kf_path));
2681 if (freepath != NULL)
2682 free(freepath, M_TEMP);
2684 KFAIL_POINT_CODE(DEBUG_FP, fill_kinfo_vnode__random_path,
2691 va.va_fsid = VNOVAL;
2693 vn_lock(vp, LK_SHARED | LK_RETRY);
2694 error = VOP_GETATTR(vp, &va, curthread->td_ucred);
2698 if (va.va_fsid != VNOVAL)
2699 kif->kf_un.kf_file.kf_file_fsid = va.va_fsid;
2701 kif->kf_un.kf_file.kf_file_fsid =
2702 vp->v_mount->mnt_stat.f_fsid.val[0];
2703 kif->kf_un.kf_file.kf_file_fsid_freebsd11 =
2704 kif->kf_un.kf_file.kf_file_fsid;
2705 kif->kf_un.kf_file.kf_file_fileid = va.va_fileid;
2706 kif->kf_un.kf_file.kf_file_mode = MAKEIMODE(va.va_type, va.va_mode);
2707 kif->kf_un.kf_file.kf_file_size = va.va_size;
2708 kif->kf_un.kf_file.kf_file_rdev = va.va_rdev;
2709 kif->kf_un.kf_file.kf_file_rdev_freebsd11 =
2710 kif->kf_un.kf_file.kf_file_rdev;
2715vn_mmap(
struct file *fp, vm_map_t map, vm_offset_t *
addr, vm_size_t size,
2716 vm_prot_t prot, vm_prot_t cap_maxprot,
int flags, vm_ooffset_t foff,
2720 struct pmckern_map_in pkm;
2726 boolean_t writecounted;
2729#if defined(COMPAT_FREEBSD7) || defined(COMPAT_FREEBSD6) || \
2730 defined(COMPAT_FREEBSD5) || defined(COMPAT_FREEBSD4)
2739 if ((fp->f_flag & FPOSIXSHM) != 0)
2740 flags |= MAP_NOSYNC;
2754 if (mp != NULL && (mp->mnt_flag & MNT_NOEXEC) != 0) {
2755 maxprot = VM_PROT_NONE;
2756 if ((prot & VM_PROT_EXECUTE) != 0)
2759 maxprot = VM_PROT_EXECUTE;
2760 if ((fp->f_flag & FREAD) != 0)
2761 maxprot |= VM_PROT_READ;
2762 else if ((prot & VM_PROT_READ) != 0)
2770 if ((
flags & MAP_SHARED) != 0) {
2771 if ((fp->f_flag & FWRITE) != 0)
2772 maxprot |= VM_PROT_WRITE;
2773 else if ((prot & VM_PROT_WRITE) != 0)
2776 maxprot |= VM_PROT_WRITE;
2777 cap_maxprot |= VM_PROT_WRITE;
2779 maxprot &= cap_maxprot;
2796 foff > OFF_MAX - size)
2799 writecounted = FALSE;
2800 error = vm_mmap_vnode(td, size, prot, &maxprot, &
flags, vp,
2801 &foff, &
object, &writecounted);
2804 error = vm_mmap_object(map,
addr, size, prot, maxprot,
flags,
object,
2805 foff, writecounted, td);
2812 vm_pager_release_writecount(
object, 0, size);
2813 vm_object_deallocate(
object);
2817 if (PMC_HOOK_INSTALLED(PMC_FN_MMAP)) {
2818 if ((prot & VM_PROT_EXECUTE) != 0 && error == 0) {
2820 pkm.pm_address = (uintptr_t) *
addr;
2821 PMC_CALL_HOOK_UNLOCKED(td, PMC_FN_MMAP, (
void *) &pkm);
2833 f = &vp->v_mount->mnt_stat.f_fsid;
2834 va->va_fsid = (uint32_t)f->val[1];
2835 va->va_fsid <<=
sizeof(f->val[1]) * NBBY;
2836 va->va_fsid += (uint32_t)f->val[0];
2842 struct buf *bp, *nbp;
2845 int error, maxretry;
2850 if (vp->v_type == VCHR) {
2852 mp = vp->v_rdev->si_mountpt;
2861 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) {
2862 bp->b_vflags &= ~BV_SCANNED;
2870 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) {
2871 if ((bp->b_vflags & BV_SCANNED) != 0)
2873 bp->b_vflags |= BV_SCANNED;
2874 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) {
2875 if (waitfor != MNT_WAIT)
2878 LK_EXCLUSIVE | LK_INTERLOCK | LK_SLEEPFAIL,
2879 BO_LOCKPTR(bo)) != 0) {
2886 KASSERT(bp->b_bufobj == bo,
2887 (
"bp %p wrong b_bufobj %p should be %p",
2888 bp, bp->b_bufobj, bo));
2889 if ((bp->b_flags & B_DELWRI) == 0)
2890 panic(
"fsync: not dirty");
2891 if ((vp->v_object != NULL) && (bp->b_flags & B_CLUSTEROK)) {
2897 if (maxretry < 1000)
2898 pause(
"dirty",
hz < 1000 ? 1 :
hz / 1000);
2909 if (waitfor == MNT_WAIT) {
2911 if (bo->bo_dirty.bv_cnt > 0) {
2917 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs)
2918 if ((error = bp->b_error) != 0)
2920 if ((mp != NULL && mp->mnt_secondary_writes > 0) ||
2921 (error == 0 && --maxretry >= 0))
2929 vn_printf(vp,
"fsync: giving up on dirty (error = %d) ", error);
2944 off_t *outoffp,
size_t *lenp,
unsigned int flags,
struct ucred *incred,
2945 struct ucred *outcred,
struct thread *fsize_td)
2956 if (invp->v_type == VDIR || outvp->v_type == VDIR)
2958 else if (*inoffp < 0 || *outoffp < 0 ||
2959 invp->v_type != VREG || outvp->v_type != VREG)
2967 if (uval > INT64_MAX)
2968 len = INT64_MAX - *inoffp;
2971 if (uval > INT64_MAX)
2972 len = INT64_MAX - *outoffp;
2982 if (invp->v_mount == outvp->v_mount)
2983 error = VOP_COPY_FILE_RANGE(invp, inoffp, outvp, outoffp,
2984 lenp,
flags, incred, outcred, fsize_td);
2987 outoffp, lenp,
flags, incred, outcred, fsize_td);
3004 for (p = dat; len > 0; len -=
sizeof(*p), p++) {
3005 if (len >=
sizeof(*p)) {
3009 cp = (
const char *)p;
3010 for (i = 0; i < len; i++, cp++)
3025vn_skip_hole(
struct vnode *outvp, off_t xfer2, off_t *outoffp, off_t *xferp,
3026 off_t *dataoffp, off_t *holeoffp,
struct ucred *cred)
3031 if (*holeoffp == 0 || *holeoffp <= *outoffp) {
3032 *dataoffp = *outoffp;
3033 error = VOP_IOCTL(outvp, FIOSEEKDATA, dataoffp, 0, cred,
3036 *holeoffp = *dataoffp;
3037 error = VOP_IOCTL(outvp, FIOSEEKHOLE, holeoffp, 0, cred,
3040 if (error != 0 || *holeoffp == *dataoffp) {
3053 KASSERT(*dataoffp >= *outoffp,
3054 (
"vn_skip_hole: dataoff=%jd < outoff=%jd",
3055 (intmax_t)*dataoffp, (intmax_t)*outoffp));
3056 KASSERT(*holeoffp > *dataoffp,
3057 (
"vn_skip_hole: holeoff=%jd <= dataoff=%jd",
3058 (intmax_t)*holeoffp, (intmax_t)*dataoffp));
3065 if (*dataoffp > *outoffp) {
3066 delta = *dataoffp - *outoffp;
3067 if (delta >= *xferp) {
3075 xfer2 = MIN(xfer2, *xferp);
3085 if (*holeoffp > *outoffp && *holeoffp < *outoffp + xfer2)
3086 xfer2 = *holeoffp - *outoffp;
3101 u_long blksize,
bool growfile,
bool checkhole,
struct ucred *cred)
3104 off_t dataoff, holeoff, xfer2;
3116 xfer2 = MIN(xfer, blksize);
3123 &dataoff, &holeoff, cred);
3128 KASSERT(xfer2 > 0, (
"vn_write_outvp: xfer2=%jd",
3137 error = vn_lock(outvp, LK_EXCLUSIVE);
3146 error =
vn_rdwr(UIO_WRITE, outvp, dat, xfer2,
3147 outoff, UIO_SYSSPACE, IO_NODELOCKED,
3148 curthread->td_ucred, cred, NULL, curthread);
3156 }
while (!growfile && xfer > 0 && error == 0);
3168 struct vnode *outvp, off_t *outoffp,
size_t *lenp,
unsigned int flags,
3169 struct ucred *incred,
struct ucred *outcred,
struct thread *fsize_td)
3171 struct vattr va, inva;
3174 off_t startoff, endoff, xfer, xfer2;
3176 int error, interrupted;
3177 bool cantseek, readzeros, eof, lastblock, holetoeof;
3179 size_t copylen, len, rem, savlen;
3181 long holein, holeout;
3182 struct timespec curts, endts;
3184 holein = holeout = 0;
3185 savlen = len = *lenp;
3190 error = vn_lock(invp, LK_SHARED);
3193 if (VOP_PATHCONF(invp, _PC_MIN_HOLE_SIZE, &holein) != 0)
3196 error = VOP_GETATTR(invp, &inva, incred);
3204 error = vn_lock(outvp, LK_EXCLUSIVE);
3210 if (fsize_td != NULL) {
3211 io.uio_offset = *outoffp;
3217 if (VOP_PATHCONF(outvp, _PC_MIN_HOLE_SIZE, &holeout) != 0)
3226 error = VOP_GETATTR(outvp, &va, outcred);
3227 if (error == 0 && va.va_size > *outoffp && va.va_size <=
3230 error = mac_vnode_check_write(curthread->td_ucred,
3237 va.va_size = *outoffp;
3253 blksize = MAX(holein, holeout);
3257 rem = *inoffp % blksize;
3259 rem = blksize - rem;
3260 if (len > rem && len - rem > blksize)
3261 len = savlen = rounddown(len - rem, blksize) + rem;
3265 blksize = MAX(invp->v_mount->mnt_stat.f_iosize,
3266 outvp->v_mount->mnt_stat.f_iosize);
3269 else if (blksize > 1024 * 1024)
3270 blksize = 1024 * 1024;
3271 dat =
malloc(blksize, M_TEMP, M_WAITOK);
3283 if ((
flags & COPY_FILE_RANGE_TIMEO1SEC) != 0) {
3287 timespecclear(&endts);
3288 holetoeof = eof =
false;
3289 while (len > 0 && error == 0 && !eof && interrupted == 0) {
3307 error = VOP_IOCTL(invp, FIOSEEKDATA, &startoff, 0,
3309 if (error == ENXIO) {
3310 startoff = endoff = inva.va_size;
3311 eof = holetoeof =
true;
3315 if (error == 0 && !holetoeof) {
3317 error = VOP_IOCTL(invp, FIOSEEKHOLE, &endoff, 0,
3328 if (error == 0 && startoff == endoff)
3332 if (startoff > *inoffp) {
3334 xfer = MIN(startoff - *inoffp, len);
3335 if (*outoffp < va.va_size) {
3337 xfer2 = MIN(va.va_size - *outoffp,
3339 memset(dat, 0, MIN(xfer2, blksize));
3341 *outoffp, xfer2, blksize,
false,
3342 holeout > 0, outcred);
3345 if (error == 0 && *outoffp + xfer >
3346 va.va_size && (xfer == len || holetoeof)) {
3349 *outoffp, xfer, blksize,
true,
3358 if (timespecisset(&endts) &&
3361 if (timespeccmp(&curts,
3369 copylen = MIN(len, endoff - startoff);
3385 xfer -= (*inoffp % blksize);
3388 while (copylen > 0 && error == 0 && !eof && interrupted == 0) {
3391 error = vn_lock(invp, LK_SHARED);
3394 error =
vn_rdwr(UIO_READ, invp, dat, xfer,
3395 startoff, UIO_SYSSPACE, IO_NODELOCKED,
3396 curthread->td_ucred, incred, &aresid,
3400 if (error == 0 && aresid > 0) {
3412 readzeros = cantseek ?
mem_iszero(dat, xfer) :
3416 if (!cantseek || *outoffp < va.va_size ||
3417 lastblock || !readzeros)
3419 *outoffp, xfer, blksize,
3420 readzeros && lastblock &&
3421 *outoffp >= va.va_size,
false,
3431 if (timespecisset(&endts) &&
3434 if (timespeccmp(&curts,
3446 *lenp = savlen - len;
3456 off_t olen, ooffset;
3459 int audited_vnode1 = 0;
3463 if (vp->v_type != VREG)
3476 error = vn_lock(vp, LK_EXCLUSIVE);
3482 if (!audited_vnode1) {
3483 AUDIT_ARG_VNODE1(vp);
3488 error = mac_vnode_check_write(td->td_ucred, fp->f_cred, vp);
3491 error = VOP_ALLOCATE(vp, &offset, &len, 0,
3496 if (olen + ooffset != offset + len) {
3497 panic(
"offset + len changed from %jx/%jx to %jx/%jx",
3498 ooffset, olen, offset, len);
3500 if (error != 0 || len == 0)
3502 KASSERT(olen > len, (
"Iteration did not make progress?"));
3511 int ioflag,
struct ucred *cred,
struct ucred *active_cred,
3512 struct ucred *file_cred)
3519 bool audited_vnode1 =
false;
3528 if ((ioflag & (IO_NODELOCKED | IO_RANGELOCKED)) == 0)
3529 rl_cookie = vn_rangelock_wlock(vp, off, off + len);
3530 while (len > 0 && error == 0) {
3538 if ((ioflag & IO_NODELOCKED) == 0) {
3541 V_WAIT | PCATCH)) != 0)
3546 if (!audited_vnode1) {
3547 AUDIT_ARG_VNODE1(vp);
3548 audited_vnode1 =
true;
3553 if ((ioflag & IO_NOMACCHECK) == 0)
3554 error = mac_vnode_check_write(active_cred, file_cred,
3558 error = VOP_DEALLOCATE(vp, &off, &len,
flags, ioflag,
3561 if ((ioflag & IO_NODELOCKED) == 0) {
3568 if (error == 0 && len != 0)
3572 if (rl_cookie != NULL)
3573 vn_rangelock_unlock(vp, rl_cookie);
3585 int ioflag,
struct ucred *active_cred,
struct ucred *file_cred)
3589 if (*offset < 0 || *length <= 0 || *length > OFF_MAX - *offset ||
3592 if (vp->v_type != VREG)
3595 cred = file_cred != NOCRED ? file_cred : active_cred;
3597 active_cred, file_cred));
3602 struct ucred *active_cred,
struct thread *td)
3610 if (cmd != SPACECTL_DEALLOC || *offset < 0 || *length <= 0 ||
3611 *length > OFF_MAX - *offset ||
flags != 0)
3613 if (vp->v_type != VREG)
3619 case SPACECTL_DEALLOC:
3621 active_cred, active_cred, fp->f_cred);
3624 panic(
"vn_fspacectl: unknown cmd %d", cmd);
3633 "Count of vn_lock_pair deadlocks");
3638 "Max ticks for vn_lock_pair deadlock avoidance sleep");
3664 if (vp1 == NULL && vp2 == NULL)
3668 ASSERT_VOP_ELOCKED(vp1,
"vp1");
3670 ASSERT_VOP_UNLOCKED(vp1,
"vp1");
3676 ASSERT_VOP_ELOCKED(vp2,
"vp2");
3678 ASSERT_VOP_UNLOCKED(vp2,
"vp2");
3682 if (!vp1_locked && !vp2_locked) {
3683 vn_lock(vp1, LK_EXCLUSIVE | LK_RETRY);
3688 if (vp1_locked && vp2_locked)
3690 if (vp1_locked && vp2 != NULL) {
3692 error = VOP_LOCK1(vp2, LK_EXCLUSIVE | LK_NOWAIT,
3693 __FILE__, __LINE__);
3700 vn_lock(vp2, LK_EXCLUSIVE | LK_RETRY);
3703 if (vp2_locked && vp1 != NULL) {
3705 error = VOP_LOCK1(vp1, LK_EXCLUSIVE | LK_NOWAIT,
3706 __FILE__, __LINE__);
3713 vn_lock(vp1, LK_EXCLUSIVE | LK_RETRY);
3718 ASSERT_VOP_ELOCKED(vp1,
"vp1 ret");
3720 ASSERT_VOP_ELOCKED(vp2,
"vp2 ret");
3726 if (MNT_SHARED_WRITES(mp) ||
3727 (mp == NULL && MNT_SHARED_WRITES(vp->v_mount)))
3729 return (LK_EXCLUSIVE);
device_property_type_t type
int vntype_to_kinfo(int vtype)
struct fileops badfileops
void knote(struct knlist *list, long hint, int lockflags)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void free(void *addr, struct malloc_type *mtp)
struct mtx_pool __read_mostly * mtxpool_sleep
struct mtx * mtx_pool_find(struct mtx_pool *pool, void *ptr)
int priv_check(struct thread *td, int priv)
rlim_t() lim_cur(struct thread *td, int which)
int vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio, struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags, struct thread *td)
void panic(const char *fmt,...)
void kern_psignal(struct proc *p, int sig)
void kern_yield(int prio)
void wakeup(const void *ident)
void getnanouptime(struct timespec *tsp)
enum vn_io_fault_args::@16 kind
union vn_io_fault_args::@17 args
struct vn_io_fault_args::@17::vop_args_tag vop_args
struct vn_io_fault_args::@17::fop_args_tag fop_args
uint32_t prng32_bounded(uint32_t bound)
void sleepq_release(const void *wchan)
int sleepq_broadcast(const void *wchan, int flags, int pri, int queue)
void sleepq_add(const void *wchan, struct lock_object *lock, const char *wmesg, int flags, int queue)
void sleepq_wait(const void *wchan, int pri)
void sleepq_lock(const void *wchan)
struct uio * cloneuio(struct uio *uiop)
int uiomove(void *cp, int n, struct uio *uio)
void bremfree(struct buf *bp)
int bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
void bawrite(struct buf *bp)
int vfs_bio_awrite(struct buf *bp)
int vn_fullpath(struct vnode *vp, char **retbuf, char **freebuf)
void() NDFREE(struct nameidata *ndp, const u_int flags)
int namei(struct nameidata *ndp)
void vfs_op_enter(struct mount *mp)
void vfs_rel(struct mount *mp)
void vfs_op_exit_locked(struct mount *mp)
void vfs_op_exit(struct mount *mp)
void vfs_ref(struct mount *mp)
void vref(struct vnode *vp)
void vn_printf(struct vnode *vp, const char *fmt,...)
void vfs_unbusy(struct mount *mp)
void vrele(struct vnode *vp)
void vput(struct vnode *vp)
int vfs_busy(struct mount *mp, int flags)
void vunref(struct vnode *vp)
int setfown(struct thread *td, struct ucred *cred, struct vnode *vp, uid_t uid, gid_t gid)
int setfmode(struct thread *td, struct ucred *cred, struct vnode *vp, int mode)
static int vn_io_fault_prefault_user(const struct uio *uio)
static int vn_deallocate_impl(struct vnode *vp, off_t *offset, off_t *length, int flags, int ioflag, struct ucred *cred, struct ucred *active_cred, struct ucred *file_cred)
int vn_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
static int vn_write_outvp(struct vnode *outvp, char *dat, off_t outoff, off_t xfer, u_long blksize, bool growfile, bool checkhole, struct ucred *cred)
void foffset_unlock_uio(struct file *fp, struct uio *uio, int flags)
static void vn_lock_pair_pause(const char *wmesg)
int vn_vget_ino(struct vnode *vp, ino_t ino, int lkflags, struct vnode **rvp)
void vn_finished_secondary_write(struct mount *mp)
int vn_copy_file_range(struct vnode *invp, off_t *inoffp, struct vnode *outvp, off_t *outoffp, size_t *lenp, unsigned int flags, struct ucred *incred, struct ucred *outcred, struct thread *fsize_td)
int vn_extattr_get(struct vnode *vp, int ioflg, int attrnamespace, const char *attrname, int *buflen, char *buf, struct thread *td)
static int __noinline _vn_lock_fallback(struct vnode *vp, int flags, const char *file, int line, int error)
static int vn_close1(struct vnode *vp, int flags, struct ucred *file_cred, struct thread *td, bool keep_ref)
static int get_write_ioflag(struct file *fp)
static int get_advice(struct file *fp, struct uio *uio)
u_int vn_lock_pair_pause_max
static int vn_io_fault_enable
int vfs_write_suspend(struct mount *mp, int flags)
static int vn_io_fault_touch(char *base, const struct uio *uio)
int vn_start_write(struct vnode *vp, struct mount **mpp, int flags)
static u_long vn_lock_pair_pause_cnt
static int vfs_allow_read_dir
static int vn_start_write_refed(struct mount *mp, int flags, bool mplocked)
static int vn_io_fault1(struct vnode *vp, struct uio *uio, struct vn_io_fault_args *args, struct thread *td)
int vn_rdwr(enum uio_rw rw, struct vnode *vp, void *base, int len, off_t offset, enum uio_seg segflg, int ioflg, struct ucred *active_cred, struct ucred *file_cred, ssize_t *aresid, struct thread *td)
int vn_vget_ino_gen(struct vnode *vp, vn_get_ino_t alloc, void *alloc_arg, int lkflags, struct vnode **rvp)
int vn_bmap_seekhole_locked(struct vnode *vp, u_long cmd, off_t *off, struct ucred *cred)
int vn_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred, struct thread *td)
int vn_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
static u_long vn_io_faults_cnt
int vn_rlimit_fsize(const struct vnode *vp, const struct uio *uio, struct thread *td)
static fo_ioctl_t vn_ioctl
int vn_writechk(struct vnode *vp)
int vn_generic_copy_file_range(struct vnode *invp, off_t *inoffp, struct vnode *outvp, off_t *outoffp, size_t *lenp, unsigned int flags, struct ucred *incred, struct ucred *outcred, struct thread *fsize_td)
int vn_start_secondary_write(struct vnode *vp, struct mount **mpp, int flags)
static off_t vn_skip_hole(struct vnode *outvp, off_t xfer2, off_t *outoffp, off_t *xferp, off_t *dataoffp, off_t *holeoffp, struct ucred *cred)
static uint64_t open2nameif(int fmode, u_int vn_open_flags)
SYSCTL_INT(_debug, OID_AUTO, vn_io_fault_enable, CTLFLAG_RWTUN, &vn_io_fault_enable, 0, "Enable vn_io_fault lock avoidance")
int vn_lktype_write(struct mount *mp, struct vnode *vp)
int vn_bmap_seekhole(struct vnode *vp, u_long cmd, off_t *off, struct ucred *cred)
int vn_extattr_rm(struct vnode *vp, int ioflg, int attrnamespace, const char *attrname, struct thread *td)
void vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
int vn_rdwr_inchunks(enum uio_rw rw, struct vnode *vp, void *base, size_t len, off_t offset, enum uio_seg segflg, int ioflg, struct ucred *active_cred, struct ucred *file_cred, size_t *aresid, struct thread *td)
int vn_deallocate(struct vnode *vp, off_t *offset, off_t *length, int flags, int ioflag, struct ucred *active_cred, struct ucred *file_cred)
void foffset_lock_uio(struct file *fp, struct uio *uio, int flags)
int vn_utimes_perm(struct vnode *vp, struct vattr *vap, struct ucred *cred, struct thread *td)
int vn_statfile(struct file *fp, struct stat *sb, struct ucred *active_cred)
static int vn_io_fault_doio(struct vn_io_fault_args *args, struct uio *uio, struct thread *td)
int _vn_lock(struct vnode *vp, int flags, const char *file, int line)
static int vn_io_pgcache_read_enable
static fo_fallocate_t vn_fallocate
SYSCTL_ULONG(_debug, OID_AUTO, vn_io_faults, CTLFLAG_RD, &vn_io_faults_cnt, 0, "Count of vn_io_fault lock avoidance triggers")
int vn_fill_kinfo_vnode(struct vnode *vp, struct kinfo_file *kif)
static bool do_vn_io_fault(struct vnode *vp, struct uio *uio)
static bool mem_iszero(void *dat, int len)
int vn_io_fault_uiomove(char *data, int xfersize, struct uio *uio)
int vn_seek(struct file *fp, off_t offset, int whence, struct thread *td)
void foffset_unlock(struct file *fp, off_t val, int flags)
int vn_open(struct nameidata *ndp, int *flagp, int cmode, struct file *fp)
int vn_kqfilter_opath(struct file *fp, struct knote *kn)
int vn_open_cred(struct nameidata *ndp, int *flagp, int cmode, u_int vn_open_flags, struct ucred *cred, struct file *fp)
int vn_io_fault_pgmove(vm_page_t ma[], vm_offset_t offset, int xfersize, struct uio *uio)
int vn_fsync_buf(struct vnode *vp, int waitfor)
static int vn_get_ino_alloc_vget(struct mount *mp, void *arg, int lkflags, struct vnode **rvp)
static fo_rdwr_t vn_write
void vn_pages_remove_valid(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
void vn_fsid(struct vnode *vp, struct vattr *va)
int vfs_write_suspend_umnt(struct mount *mp)
static fo_truncate_t vn_truncate
int vn_close(struct vnode *vp, int flags, struct ucred *file_cred, struct thread *td)
static fo_fspacectl_t vn_fspacectl
static fo_kqfilter_t vn_kqfilter
static fo_rdwr_t vn_io_fault
void vn_lock_pair(struct vnode *vp1, bool vp1_locked, struct vnode *vp2, bool vp2_locked)
static int vn_open_vnode_advlock(struct vnode *vp, int fmode, struct file *fp)
static void vn_fill_junk(struct kinfo_file *kif)
static int sequential_heuristic(struct uio *uio, struct file *fp)
void vn_finished_write(struct mount *mp)
SYSCTL_UINT(_debug, OID_AUTO, vn_lock_pair_pause_max, CTLFLAG_RW, &vn_lock_pair_pause_max, 0, "Max ticks for vn_lock_pair deadlock avoidance sleep")
void vfs_write_resume(struct mount *mp, int flags)
int vn_read_from_obj(struct vnode *vp, struct uio *uio)
int vn_extattr_set(struct vnode *vp, int ioflg, int attrnamespace, const char *attrname, int buflen, char *buf, struct thread *td)
static fo_close_t vn_closefile
static int vn_io_fault_prefault
int vn_open_vnode(struct vnode *vp, int fmode, struct ucred *cred, struct thread *td, struct file *fp)
off_t foffset_lock(struct file *fp, int flags)
int vn_truncate_locked(struct vnode *vp, off_t length, bool sync, struct ucred *cred)