#include <sys/cdefs.h>
#include "opt_kern_tls.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/capsicum.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/ktls.h>
#include <sys/mutex.h>
#include <sys/malloc.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/mbuf.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/syscallsubr.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
#include <sys/vnode.h>
#include <net/vnet.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <security/audit/audit.h>
#include <security/mac/mac_framework.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
#include <vm/vm_pager.h>

Include dependency graph for kern_sendfile.c:

Data Structures
struct	sf_io

struct	sendfile_sync

Macros
#define	EXT_FLAG_SYNC EXT_FLAG_VENDOR1

#define	EXT_FLAG_NOCACHE EXT_FLAG_VENDOR2

#define	EXT_FLAG_CACHE_LAST EXT_FLAG_VENDOR3

Functions
	__FBSDID ("$FreeBSD$")

static	MALLOC_DEFINE (M_SENDFILE, "sendfile", "sendfile dynamic memory")

static void	sendfile_sync_destroy (struct sendfile_sync *sfs)

static void	sendfile_sync_signal (struct sendfile_sync *sfs)

static void	sfstat_init (const void *unused)

	SYSINIT (sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL)

static int	sfstat_sysctl (SYSCTL_HANDLER_ARGS)

	SYSCTL_PROC (_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE\|CTLFLAG_RW\|CTLFLAG_MPSAFE, NULL, 0, sfstat_sysctl, "I", "sendfile statistics")

static void	sendfile_free_mext (struct mbuf *m)

static void	sendfile_free_mext_pg (struct mbuf *m)

static off_t	xfsize (int i, int n, off_t off, off_t len)

static vm_ooffset_t	vmoff (int i, off_t off)

static void	fixspace (int old, int new, off_t off, int *space)

static void	sendfile_iowait (struct sf_io sfio, const char wmesg)

static void	sendfile_iodone (void arg, vm_page_t pa, int count, int error)

static int	sendfile_swapin (vm_object_t obj, struct sf_io sfio, int nios, off_t off, off_t len, int rhpages, int flags)

static int	sendfile_getobj (struct thread td, struct file fp, vm_object_t obj_res, struct vnode vp_res, struct shmfd shmfd_res, off_t obj_size, int *bsize)

static int	sendfile_getsock (struct thread td, int s, struct file sock_fp, struct socket *so)

int	vn_sendfile (struct file fp, int sockfd, struct uio hdr_uio, struct uio trl_uio, off_t offset, size_t nbytes, off_t sent, int flags, struct thread *td)

static int	sendfile (struct thread td, struct sendfile_args uap, int compat)

int	sys_sendfile (struct thread td, struct sendfile_args uap)

Variables
counter_u64_t	sfstat [sizeof(struct sfstat)/sizeof(uint64_t)]

Macro Definition Documentation

◆ EXT_FLAG_CACHE_LAST

#define EXT_FLAG_CACHE_LAST EXT_FLAG_VENDOR3

Definition at line 72 of file kern_sendfile.c.

◆ EXT_FLAG_NOCACHE

#define EXT_FLAG_NOCACHE EXT_FLAG_VENDOR2

Definition at line 71 of file kern_sendfile.c.

◆ EXT_FLAG_SYNC

#define EXT_FLAG_SYNC EXT_FLAG_VENDOR1

Definition at line 70 of file kern_sendfile.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ fixspace()

static void fixspace	(	int	old,
		int	new,
		off_t	off,
		int *	space
	)

inlinestatic

Definition at line 249 of file kern_sendfile.c.

References xfsize().

Referenced by vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_SENDFILE	,
		"sendfile"	,
		"sendfile dynamic memory"
	)

static

◆ sendfile()

static int sendfile	(	struct thread *	td,
		struct sendfile_args *	uap,
		int	compat
	)

static

Definition at line 1265 of file kern_sendfile.c.

References cap_pread_rights, compat, copyinuio(), fget_read(), and free().

Referenced by sys_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_free_mext()

static void sendfile_free_mext ( struct mbuf * m )

static

Definition at line 166 of file kern_sendfile.c.

References EXT_FLAG_NOCACHE, EXT_FLAG_SYNC, flags, sendfile_sync_signal(), and sf_buf_free().

Referenced by vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_free_mext_pg()

static void sendfile_free_mext_pg ( struct mbuf * m )

static

Definition at line 189 of file kern_sendfile.c.

References EXT_FLAG_CACHE_LAST, EXT_FLAG_NOCACHE, EXT_FLAG_SYNC, flags, and sendfile_sync_signal().

Referenced by vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_getobj()

static int sendfile_getobj	(	struct thread *	td,
		struct file *	fp,
		vm_object_t *	obj_res,
		struct vnode **	vp_res,
		struct shmfd **	shmfd_res,
		off_t *	obj_size,
		int *	bsize
	)

static

Definition at line 563 of file kern_sendfile.c.

Referenced by vn_sendfile().

Here is the caller graph for this function:

◆ sendfile_getsock()

static int sendfile_getsock	(	struct thread *	td,
		int	s,
		struct file **	sock_fp,
		struct socket **	so
	)

static

Definition at line 646 of file kern_sendfile.c.

References cap_send_rights, and getsock_cap().

Referenced by vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_iodone()

static void sendfile_iodone	(	void *	arg,
		vm_page_t *	pa,
		int	count,
		int	error
	)

static

Definition at line 289 of file kern_sendfile.c.

References count, sf_io::error, free(), ktls_enqueue(), sf_io::m, mb_free_notready(), sf_io::nios, sf_io::npages, sf_io::obj, sf_io::pa, sf_io::pindex0, and sf_io::so.

Referenced by sendfile_swapin(), and vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_iowait()

static void sendfile_iowait	(	struct sf_io *	sfio,
		const char *	wmesg
	)

static

Definition at line 279 of file kern_sendfile.c.

References sf_io::nios.

Referenced by sendfile_swapin(), and vn_sendfile().

Here is the caller graph for this function:

◆ sendfile_swapin()

static int sendfile_swapin	(	vm_object_t	obj,
		struct sf_io *	sfio,
		int *	nios,
		off_t	off,
		off_t	len,
		int	rhpages,
		int	flags
	)

static

Definition at line 414 of file kern_sendfile.c.

References count, flags, sf_io::nios, sf_io::npages, sf_io::pa, sf_io::pindex0, sendfile_iodone(), sendfile_iowait(), vmoff(), and xfsize().

Referenced by vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_sync_destroy()

static void sendfile_sync_destroy ( struct sendfile_sync * sfs )

static

Definition at line 113 of file kern_sendfile.c.

References sendfile_sync::count, sendfile_sync::cv, cv_destroy(), free(), and sendfile_sync::mtx.

Referenced by sendfile_sync_signal(), and vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sendfile_sync_signal()

static void sendfile_sync_signal ( struct sendfile_sync * sfs )

static

Definition at line 123 of file kern_sendfile.c.

References sendfile_sync::count, sendfile_sync::cv, cv_signal(), sendfile_sync::mtx, sendfile_sync_destroy(), and sendfile_sync::waiting.

Referenced by sendfile_free_mext(), and sendfile_free_mext_pg().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ sfstat_init()

static void sfstat_init ( const void * unused )

static

Definition at line 142 of file kern_sendfile.c.

References sfstat.

◆ sfstat_sysctl()

static int sfstat_sysctl ( SYSCTL_HANDLER_ARGS )

static

Definition at line 151 of file kern_sendfile.c.

References sfstat.

◆ sys_sendfile()

int sys_sendfile	(	struct thread *	td,
		struct sendfile_args *	uap
	)

Definition at line 1348 of file kern_sendfile.c.

References sendfile().

Here is the call graph for this function:

◆ SYSCTL_PROC()

SYSCTL_PROC	(	_kern_ipc	,
		OID_AUTO	,
		sfstat	,
		CTLTYPE_OPAQUE\|CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		NULL	,
		0	,
		sfstat_sysctl	,
		"I"	,
		"sendfile statistics"
	)

◆ SYSINIT()

SYSINIT	(	sfstat	,
		SI_SUB_MBUF	,
		SI_ORDER_FIRST	,
		sfstat_init	,
		NULL
	)

◆ vmoff()

static vm_ooffset_t vmoff	(	int	i,
		off_t	off
	)

inlinestatic

Definition at line 234 of file kern_sendfile.c.

Referenced by sendfile_swapin(), and vn_sendfile().

Here is the caller graph for this function:

◆ vn_sendfile()

int vn_sendfile	(	struct file *	fp,
		int	sockfd,
		struct uio *	hdr_uio,
		struct uio *	trl_uio,
		off_t	offset,
		size_t	nbytes,
		off_t *	sent,
		int	flags,
		struct thread *	td
	)

Definition at line 674 of file kern_sendfile.c.

References sendfile_sync::count, sendfile_sync::cv, cv_init(), sf_io::error, EXT_FLAG_CACHE_LAST, EXT_FLAG_NOCACHE, EXT_FLAG_SYNC, fixspace(), flags, kern_writev(), ktls_enqueue(), ktls_frame(), sf_io::m, m_freem(), m_length(), m_uiotombuf(), malloc(), maxphys, mb_alloc_ext_pgs(), mb_use_ext_pgs, sendfile_sync::mtx, nbytes, sf_io::nios, sf_io::npages, sf_io::obj, sf_io::pa, sf_io::pindex0, sbwait(), sendfile_free_mext(), sendfile_free_mext_pg(), sendfile_getobj(), sendfile_getsock(), sendfile_iodone(), sendfile_iowait(), sendfile_swapin(), sendfile_sync_destroy(), sf_buf_alloc(), sf_io::so, vmoff(), sendfile_sync::waiting, and xfsize().

Here is the call graph for this function:

◆ xfsize()

static off_t xfsize	(	int	i,
		int	n,
		off_t	off,
		off_t	len
	)

inlinestatic

Definition at line 218 of file kern_sendfile.c.

Referenced by fixspace(), sendfile_swapin(), and vn_sendfile().

Here is the caller graph for this function:

Variable Documentation

◆ sfstat

counter_u64_t sfstat[sizeof(struct sfstat)/sizeof(uint64_t)]

Definition at line 139 of file kern_sendfile.c.

Referenced by sfstat_init(), and sfstat_sysctl().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ EXT_FLAG_CACHE_LAST

◆ EXT_FLAG_NOCACHE

◆ EXT_FLAG_SYNC

Function Documentation

◆ __FBSDID()

◆ fixspace()

◆ MALLOC_DEFINE()

◆ sendfile()

◆ sendfile_free_mext()

◆ sendfile_free_mext_pg()

◆ sendfile_getobj()

◆ sendfile_getsock()

◆ sendfile_iodone()

◆ sendfile_iowait()

◆ sendfile_swapin()

◆ sendfile_sync_destroy()

◆ sendfile_sync_signal()

◆ sfstat_init()

◆ sfstat_sysctl()

◆ sys_sendfile()

◆ SYSCTL_PROC()

◆ SYSINIT()

◆ vmoff()

◆ vn_sendfile()

◆ xfsize()

Variable Documentation

◆ sfstat