FreeBSD kernel kern code
|
#include <sys/cdefs.h>
#include "opt_param.h"
#include "opt_kern_tls.h"
#include <sys/param.h>
#include <sys/conf.h>
#include <sys/domainset.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/domain.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/ktls.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_var.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_map.h>
#include <vm/uma.h>
#include <vm/uma_dbg.h>
Go to the source code of this file.
Functions | |
__FBSDID ("$FreeBSD$") | |
static int | sysctl_mb_use_ext_pgs (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, mb_use_ext_pgs, CTLTYPE_INT|CTLFLAG_RW, &mb_use_ext_pgs, 0, sysctl_mb_use_ext_pgs, "IU", "Use unmapped mbufs for sendfile(2) and TLS offload") | |
SYSCTL_QUAD (_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN|CTLFLAG_NOFETCH, &maxmbufmem, 0, "Maximum real memory allocatable to various mbuf types") | |
SYSCTL_COUNTER_U64 (_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW, &snd_tag_count, "# of active mbuf send tags") | |
static void | tunable_mbinit (void *dummy) |
SYSINIT (tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL) | |
static int | sysctl_nmbclusters (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbclusters, 0, sysctl_nmbclusters, "IU", "Maximum number of mbuf clusters allowed") | |
static int | sysctl_nmbjumbop (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbop, 0, sysctl_nmbjumbop, "IU", "Maximum number of mbuf page size jumbo clusters allowed") | |
static int | sysctl_nmbjumbo9 (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU", "Maximum number of mbuf 9k jumbo clusters allowed") | |
static int | sysctl_nmbjumbo16 (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU", "Maximum number of mbuf 16k jumbo clusters allowed") | |
static int | sysctl_nmbufs (SYSCTL_HANDLER_ARGS) |
SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbufs, 0, sysctl_nmbufs, "IU", "Maximum number of mbufs allowed") | |
static int | mb_ctor_mbuf (void *, int, void *, int) |
static int | mb_ctor_clust (void *, int, void *, int) |
static int | mb_ctor_pack (void *, int, void *, int) |
static void | mb_dtor_mbuf (void *, int, void *) |
static void | mb_dtor_pack (void *, int, void *) |
static int | mb_zinit_pack (void *, int, int) |
static void | mb_zfini_pack (void *, int) |
static void | mb_reclaim (uma_zone_t, int) |
CTASSERT ((((MSIZE - 1) ^ MSIZE)+1) > > 1==MSIZE) | |
_Static_assert (sizeof(struct mbuf)<=MSIZE, "size of mbuf exceeds MSIZE") | |
static void | mbuf_init (void *dummy) |
SYSINIT (mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL) | |
static void | mb_reclaim (uma_zone_t zone __unused, int pending __unused) |
void | mb_free_notready (struct mbuf *m, int count) |
int | mb_unmapped_compress (struct mbuf *m) |
static void | mb_unmapped_free_mext (struct mbuf *m) |
static struct mbuf * | _mb_unmapped_to_ext (struct mbuf *m) |
struct mbuf * | mb_unmapped_to_ext (struct mbuf *top) |
struct mbuf * | mb_alloc_ext_pgs (int how, m_ext_free_t ext_free) |
void | mb_free_ext (struct mbuf *m) |
void | mb_free_extpg (struct mbuf *m) |
int | m_clget (struct mbuf *m, int how) |
void * | m_cljget (struct mbuf *m, int how, int size) |
struct mbuf * | m_get2 (int size, int how, short type, int flags) |
struct mbuf * | m_get3 (int size, int how, short type, int flags) |
struct mbuf * | m_getjcl (int how, short type, int flags, int size) |
struct mbuf * | m_getm2 (struct mbuf *m, int len, int how, short type, int flags) |
void | m_extadd (struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef, void *arg1, void *arg2, int flags, int type) |
void | m_freem (struct mbuf *mb) |
void | m_free_raw (struct mbuf *mb) |
int | m_snd_tag_alloc (struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **mstp) |
void | m_snd_tag_init (struct m_snd_tag *mst, struct ifnet *ifp, const struct if_snd_tag_sw *sw) |
void | m_snd_tag_destroy (struct m_snd_tag *mst) |
void | m_rcvif_serialize (struct mbuf *m) |
struct ifnet * | m_rcvif_restore (struct mbuf *m) |
struct mbuf * | mb_alloc_ext_plus_pages (int len, int how) |
struct mbuf * | mb_mapped_to_unmapped (struct mbuf *mp, int len, int mlen, int how, struct mbuf **mlast) |
Variables | |
int | nmbufs |
int | nmbclusters |
int | nmbjumbop |
int | nmbjumbo9 |
int | nmbjumbo16 |
bool | mb_use_ext_pgs = false |
static quad_t | maxmbufmem |
static counter_u64_t | snd_tag_count |
uma_zone_t | zone_mbuf |
uma_zone_t | zone_clust |
uma_zone_t | zone_pack |
uma_zone_t | zone_jumbop |
uma_zone_t | zone_jumbo9 |
uma_zone_t | zone_jumbo16 |
__FBSDID | ( | "$FreeBSD$" | ) |
|
static |
Definition at line 963 of file kern_mbuf.c.
References m_extadd(), m_freem(), mb_unmapped_free_mext(), and sf_buf_alloc().
Referenced by mb_unmapped_to_ext().
_Static_assert | ( | sizeof(struct mbuf)<= | MSIZE, |
"size of mbuf exceeds MSIZE" | |||
) |
CTASSERT | ( | (((MSIZE - 1) ^ MSIZE)+1) | , |
1 | = =MSIZE |
||
) |
int m_clget | ( | struct mbuf * | m, |
int | how | ||
) |
Definition at line 1323 of file kern_mbuf.c.
References zone_clust, and zone_pack.
void * m_cljget | ( | struct mbuf * | m, |
int | how, | ||
int | size | ||
) |
Definition at line 1350 of file kern_mbuf.c.
void m_extadd | ( | struct mbuf * | mb, |
char * | buf, | ||
u_int | size, | ||
m_ext_free_t | freef, | ||
void * | arg1, | ||
void * | arg2, | ||
int | flags, | ||
int | type | ||
) |
Definition at line 1560 of file kern_mbuf.c.
References buf, flags, and type.
Referenced by _mb_unmapped_to_ext().
void m_free_raw | ( | struct mbuf * | mb | ) |
Definition at line 1599 of file kern_mbuf.c.
References zone_mbuf.
Referenced by ktls_work_thread(), m_get2(), m_get3(), m_getjcl(), mb_free_ext(), and mb_free_extpg().
void m_freem | ( | struct mbuf * | mb | ) |
Definition at line 1587 of file kern_mbuf.c.
Referenced by _mb_unmapped_to_ext(), kern_recvit(), kern_sendit(), ktls_decrypt(), m_adj(), m_copym(), m_copypacket(), m_copyup(), m_defrag(), m_devget(), m_dup(), m_getm2(), m_prepend(), m_pulldown(), m_pullup(), m_sanity(), m_uiotombuf(), m_uiotombuf_nomap(), m_unshare(), mb_done(), mb_mapped_to_unmapped(), mb_unmapped_to_ext(), md_done(), pru_send_notsupp(), sbdrop(), sbdrop_locked(), sbflush_internal(), soopt_getm(), soopt_mcopyin(), soopt_mcopyout(), soreceive_dgram(), soreceive_generic(), soreceive_rcvoob(), sosend_dgram(), sosend_generic(), uipc_send(), unp_externalize(), unp_internalize(), and vn_sendfile().
struct mbuf * m_get2 | ( | int | size, |
int | how, | ||
short | type, | ||
int | flags | ||
) |
Definition at line 1373 of file kern_mbuf.c.
References flags, m_free_raw(), type, zone_jumbop, zone_mbuf, and zone_pack.
Referenced by m_get3(), and sockargs().
struct mbuf * m_get3 | ( | int | size, |
int | how, | ||
short | type, | ||
int | flags | ||
) |
Definition at line 1407 of file kern_mbuf.c.
References flags, m_free_raw(), m_get2(), type, zone_jumbo16, zone_jumbo9, and zone_mbuf.
struct mbuf * m_getjcl | ( | int | how, |
short | type, | ||
int | flags, | ||
int | size | ||
) |
Definition at line 1445 of file kern_mbuf.c.
References flags, m_free_raw(), type, and zone_mbuf.
Referenced by m_getm2().
struct mbuf * m_getm2 | ( | struct mbuf * | m, |
int | len, | ||
int | how, | ||
short | type, | ||
int | flags | ||
) |
Definition at line 1478 of file kern_mbuf.c.
References flags, m_freem(), m_getjcl(), and type.
Referenced by m_uiotombuf().
struct ifnet * m_rcvif_restore | ( | struct mbuf * | m | ) |
Definition at line 1651 of file kern_mbuf.c.
void m_rcvif_serialize | ( | struct mbuf * | m | ) |
Definition at line 1639 of file kern_mbuf.c.
int m_snd_tag_alloc | ( | struct ifnet * | ifp, |
union if_snd_tag_alloc_params * | params, | ||
struct m_snd_tag ** | mstp | ||
) |
Definition at line 1606 of file kern_mbuf.c.
void m_snd_tag_destroy | ( | struct m_snd_tag * | mst | ) |
Definition at line 1628 of file kern_mbuf.c.
References snd_tag_count.
void m_snd_tag_init | ( | struct m_snd_tag * | mst, |
struct ifnet * | ifp, | ||
const struct if_snd_tag_sw * | sw | ||
) |
Definition at line 1616 of file kern_mbuf.c.
References snd_tag_count.
struct mbuf * mb_alloc_ext_pgs | ( | int | how, |
m_ext_free_t | ext_free | ||
) |
Definition at line 1145 of file kern_mbuf.c.
Referenced by m_uiotombuf_nomap(), mb_alloc_ext_plus_pages(), and vn_sendfile().
struct mbuf * mb_alloc_ext_plus_pages | ( | int | len, |
int | how | ||
) |
Definition at line 1669 of file kern_mbuf.c.
References mb_alloc_ext_pgs(), and mb_free_mext_pgs().
Referenced by mb_mapped_to_unmapped().
|
static |
Definition at line 738 of file kern_mbuf.c.
Referenced by mbuf_init().
|
static |
Definition at line 649 of file kern_mbuf.c.
Referenced by mbuf_init().
|
static |
Definition at line 802 of file kern_mbuf.c.
Referenced by mbuf_init().
|
static |
Definition at line 680 of file kern_mbuf.c.
References flags, and m_tag_delete_chain().
Referenced by mbuf_init().
|
static |
Definition at line 698 of file kern_mbuf.c.
References m_tag_delete_chain(), zone_clust, and zone_pack.
Referenced by mbuf_init().
void mb_free_ext | ( | struct mbuf * | m | ) |
Definition at line 1176 of file kern_mbuf.c.
References m_free_raw(), zone_clust, zone_jumbo16, zone_jumbo9, zone_jumbop, and zone_pack.
void mb_free_extpg | ( | struct mbuf * | m | ) |
Definition at line 1269 of file kern_mbuf.c.
References ktls_enqueue_to_free(), and m_free_raw().
Referenced by mb_unmapped_free_mext().
void mb_free_notready | ( | struct mbuf * | m, |
int | count | ||
) |
Definition at line 859 of file kern_mbuf.c.
References count.
Referenced by ktls_encrypt(), ktls_encrypt_async(), ktls_encrypt_cb(), and sendfile_iodone().
struct mbuf * mb_mapped_to_unmapped | ( | struct mbuf * | mp, |
int | len, | ||
int | mlen, | ||
int | how, | ||
struct mbuf ** | mlast | ||
) |
Definition at line 1706 of file kern_mbuf.c.
References m_freem(), and mb_alloc_ext_plus_pages().
|
static |
Definition at line 837 of file kern_mbuf.c.
|
static |
int mb_unmapped_compress | ( | struct mbuf * | m | ) |
Definition at line 882 of file kern_mbuf.c.
References buf, and m_copydata().
Referenced by sbcompress(), and sbready_compress().
|
static |
Definition at line 949 of file kern_mbuf.c.
References mb_free_extpg(), and sf_buf_free().
Referenced by _mb_unmapped_to_ext().
struct mbuf * mb_unmapped_to_ext | ( | struct mbuf * | top | ) |
Definition at line 1094 of file kern_mbuf.c.
References _mb_unmapped_to_ext(), and m_freem().
|
static |
Definition at line 784 of file kern_mbuf.c.
References zone_clust.
Referenced by mbuf_init().
|
static |
Definition at line 764 of file kern_mbuf.c.
References zone_clust.
Referenced by mbuf_init().
|
static |
Definition at line 349 of file kern_mbuf.c.
References counter_u64_alloc(), mb_ctor_clust(), mb_ctor_mbuf(), mb_ctor_pack(), mb_dtor_mbuf(), mb_dtor_pack(), mb_reclaim(), mb_zfini_pack(), mb_zinit_pack(), nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, snd_tag_count, zone_clust, zone_jumbo16, zone_jumbo9, zone_jumbop, zone_mbuf, and zone_pack.
SYSCTL_COUNTER_U64 | ( | _kern_ipc | , |
OID_AUTO | , | ||
num_snd_tags | , | ||
CTLFLAG_RW | , | ||
& | snd_tag_count, | ||
"# of active mbuf send tags" | |||
) |
|
static |
Definition at line 122 of file kern_mbuf.c.
References mb_use_ext_pgs, and sysctl_handle_int().
|
static |
Definition at line 208 of file kern_mbuf.c.
References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_clust.
|
static |
Definition at line 275 of file kern_mbuf.c.
References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbo16.
|
static |
Definition at line 253 of file kern_mbuf.c.
References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbo9.
|
static |
Definition at line 231 of file kern_mbuf.c.
References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbop.
|
static |
Definition at line 297 of file kern_mbuf.c.
References nmbufs, sysctl_handle_int(), and zone_mbuf.
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
mb_use_ext_pgs | , | ||
CTLTYPE_INT| | CTLFLAG_RW, | ||
& | mb_use_ext_pgs, | ||
0 | , | ||
sysctl_mb_use_ext_pgs | , | ||
"IU" | , | ||
"Use unmapped mbufs for sendfile(2) and TLS offload" | |||
) |
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
nmbclusters | , | ||
CTLTYPE_INT|CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
& | nmbclusters, | ||
0 | , | ||
sysctl_nmbclusters | , | ||
"IU" | , | ||
"Maximum number of mbuf clusters allowed" | |||
) |
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
nmbjumbo16 | , | ||
CTLTYPE_INT|CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
& | nmbjumbo16, | ||
0 | , | ||
sysctl_nmbjumbo16 | , | ||
"IU" | , | ||
"Maximum number of mbuf 16k jumbo clusters allowed" | |||
) |
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
nmbjumbo9 | , | ||
CTLTYPE_INT|CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
& | nmbjumbo9, | ||
0 | , | ||
sysctl_nmbjumbo9 | , | ||
"IU" | , | ||
"Maximum number of mbuf 9k jumbo clusters allowed" | |||
) |
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
nmbjumbop | , | ||
CTLTYPE_INT|CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
& | nmbjumbop, | ||
0 | , | ||
sysctl_nmbjumbop | , | ||
"IU" | , | ||
"Maximum number of mbuf page size jumbo clusters allowed" | |||
) |
SYSCTL_PROC | ( | _kern_ipc | , |
OID_AUTO | , | ||
nmbufs | , | ||
CTLTYPE_INT|CTLFLAG_RW| | CTLFLAG_MPSAFE, | ||
& | nmbufs, | ||
0 | , | ||
sysctl_nmbufs | , | ||
"IU" | , | ||
"Maximum number of mbufs allowed" | |||
) |
SYSCTL_QUAD | ( | _kern_ipc | , |
OID_AUTO | , | ||
maxmbufmem | , | ||
CTLFLAG_RDTUN| | CTLFLAG_NOFETCH, | ||
& | maxmbufmem, | ||
0 | , | ||
"Maximum real memory allocatable to various mbuf types" | |||
) |
SYSINIT | ( | mbuf | , |
SI_SUB_MBUF | , | ||
SI_ORDER_FIRST | , | ||
mbuf_init | , | ||
NULL | |||
) |
SYSINIT | ( | tunable_mbinit | , |
SI_SUB_KMEM | , | ||
SI_ORDER_MIDDLE | , | ||
tunable_mbinit | , | ||
NULL | |||
) |
|
static |
Definition at line 154 of file kern_mbuf.c.
References maxmbufmem, mb_use_ext_pgs, nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, realmem, and vm_kmem_size.
|
static |
Definition at line 141 of file kern_mbuf.c.
Referenced by tunable_mbinit().
bool mb_use_ext_pgs = false |
Definition at line 119 of file kern_mbuf.c.
Referenced by sysctl_mb_use_ext_pgs(), tunable_mbinit(), and vn_sendfile().
int nmbclusters |
Definition at line 114 of file kern_mbuf.c.
Referenced by mbuf_init(), sysctl_nmbclusters(), sysctl_nmbjumbo16(), sysctl_nmbjumbo9(), sysctl_nmbjumbop(), and tunable_mbinit().
int nmbjumbo16 |
Definition at line 117 of file kern_mbuf.c.
Referenced by mbuf_init(), sysctl_nmbclusters(), sysctl_nmbjumbo16(), sysctl_nmbjumbo9(), sysctl_nmbjumbop(), and tunable_mbinit().
int nmbjumbo9 |
Definition at line 116 of file kern_mbuf.c.
Referenced by mbuf_init(), sysctl_nmbclusters(), sysctl_nmbjumbo16(), sysctl_nmbjumbo9(), sysctl_nmbjumbop(), and tunable_mbinit().
int nmbjumbop |
Definition at line 115 of file kern_mbuf.c.
Referenced by mbuf_init(), sysctl_nmbclusters(), sysctl_nmbjumbo16(), sysctl_nmbjumbo9(), sysctl_nmbjumbop(), and tunable_mbinit().
int nmbufs |
Definition at line 113 of file kern_mbuf.c.
Referenced by mbuf_init(), sysctl_nmbclusters(), sysctl_nmbjumbo16(), sysctl_nmbjumbo9(), sysctl_nmbjumbop(), sysctl_nmbufs(), and tunable_mbinit().
|
static |
Definition at line 146 of file kern_mbuf.c.
Referenced by m_snd_tag_destroy(), m_snd_tag_init(), and mbuf_init().
uma_zone_t zone_clust |
Definition at line 322 of file kern_mbuf.c.
Referenced by m_clget(), mb_dtor_pack(), mb_free_ext(), mb_zfini_pack(), mb_zinit_pack(), mbuf_init(), and sysctl_nmbclusters().
uma_zone_t zone_jumbo16 |
Definition at line 326 of file kern_mbuf.c.
Referenced by m_get3(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbo16().
uma_zone_t zone_jumbo9 |
Definition at line 325 of file kern_mbuf.c.
Referenced by m_get3(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbo9().
uma_zone_t zone_jumbop |
Definition at line 324 of file kern_mbuf.c.
Referenced by m_get2(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbop().
uma_zone_t zone_mbuf |
Definition at line 321 of file kern_mbuf.c.
Referenced by m_free_raw(), m_get2(), m_get3(), m_getjcl(), mbuf_init(), and sysctl_nmbufs().
uma_zone_t zone_pack |
Definition at line 323 of file kern_mbuf.c.
Referenced by m_clget(), m_get2(), mb_dtor_pack(), mb_free_ext(), and mbuf_init().