FreeBSD kernel kern code
kern_mbuf.c File Reference
#include <sys/cdefs.h>
#include "opt_param.h"
#include "opt_kern_tls.h"
#include <sys/param.h>
#include <sys/conf.h>
#include <sys/domainset.h>
#include <sys/malloc.h>
#include <sys/systm.h>
#include <sys/mbuf.h>
#include <sys/domain.h>
#include <sys/eventhandler.h>
#include <sys/kernel.h>
#include <sys/ktls.h>
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_var.h>
#include <vm/vm.h>
#include <vm/vm_extern.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_map.h>
#include <vm/uma.h>
#include <vm/uma_dbg.h>
Include dependency graph for kern_mbuf.c:

Go to the source code of this file.

Functions

 __FBSDID ("$FreeBSD$")
 
static int sysctl_mb_use_ext_pgs (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, mb_use_ext_pgs, CTLTYPE_INT|CTLFLAG_RW, &mb_use_ext_pgs, 0, sysctl_mb_use_ext_pgs, "IU", "Use unmapped mbufs for sendfile(2) and TLS offload")
 
 SYSCTL_QUAD (_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN|CTLFLAG_NOFETCH, &maxmbufmem, 0, "Maximum real memory allocatable to various mbuf types")
 
 SYSCTL_COUNTER_U64 (_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW, &snd_tag_count, "# of active mbuf send tags")
 
static void tunable_mbinit (void *dummy)
 
 SYSINIT (tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL)
 
static int sysctl_nmbclusters (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbclusters, 0, sysctl_nmbclusters, "IU", "Maximum number of mbuf clusters allowed")
 
static int sysctl_nmbjumbop (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbop, 0, sysctl_nmbjumbop, "IU", "Maximum number of mbuf page size jumbo clusters allowed")
 
static int sysctl_nmbjumbo9 (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU", "Maximum number of mbuf 9k jumbo clusters allowed")
 
static int sysctl_nmbjumbo16 (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU", "Maximum number of mbuf 16k jumbo clusters allowed")
 
static int sysctl_nmbufs (SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC (_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW|CTLFLAG_MPSAFE, &nmbufs, 0, sysctl_nmbufs, "IU", "Maximum number of mbufs allowed")
 
static int mb_ctor_mbuf (void *, int, void *, int)
 
static int mb_ctor_clust (void *, int, void *, int)
 
static int mb_ctor_pack (void *, int, void *, int)
 
static void mb_dtor_mbuf (void *, int, void *)
 
static void mb_dtor_pack (void *, int, void *)
 
static int mb_zinit_pack (void *, int, int)
 
static void mb_zfini_pack (void *, int)
 
static void mb_reclaim (uma_zone_t, int)
 
 CTASSERT ((((MSIZE - 1) ^ MSIZE)+1) > > 1==MSIZE)
 
 _Static_assert (sizeof(struct mbuf)<=MSIZE, "size of mbuf exceeds MSIZE")
 
static void mbuf_init (void *dummy)
 
 SYSINIT (mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
 
static void mb_reclaim (uma_zone_t zone __unused, int pending __unused)
 
void mb_free_notready (struct mbuf *m, int count)
 
int mb_unmapped_compress (struct mbuf *m)
 
static void mb_unmapped_free_mext (struct mbuf *m)
 
static struct mbuf * _mb_unmapped_to_ext (struct mbuf *m)
 
struct mbuf * mb_unmapped_to_ext (struct mbuf *top)
 
struct mbuf * mb_alloc_ext_pgs (int how, m_ext_free_t ext_free)
 
void mb_free_ext (struct mbuf *m)
 
void mb_free_extpg (struct mbuf *m)
 
int m_clget (struct mbuf *m, int how)
 
void * m_cljget (struct mbuf *m, int how, int size)
 
struct mbuf * m_get2 (int size, int how, short type, int flags)
 
struct mbuf * m_get3 (int size, int how, short type, int flags)
 
struct mbuf * m_getjcl (int how, short type, int flags, int size)
 
struct mbuf * m_getm2 (struct mbuf *m, int len, int how, short type, int flags)
 
void m_extadd (struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef, void *arg1, void *arg2, int flags, int type)
 
void m_freem (struct mbuf *mb)
 
void m_free_raw (struct mbuf *mb)
 
int m_snd_tag_alloc (struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **mstp)
 
void m_snd_tag_init (struct m_snd_tag *mst, struct ifnet *ifp, const struct if_snd_tag_sw *sw)
 
void m_snd_tag_destroy (struct m_snd_tag *mst)
 
void m_rcvif_serialize (struct mbuf *m)
 
struct ifnet * m_rcvif_restore (struct mbuf *m)
 
struct mbuf * mb_alloc_ext_plus_pages (int len, int how)
 
struct mbuf * mb_mapped_to_unmapped (struct mbuf *mp, int len, int mlen, int how, struct mbuf **mlast)
 

Variables

int nmbufs
 
int nmbclusters
 
int nmbjumbop
 
int nmbjumbo9
 
int nmbjumbo16
 
bool mb_use_ext_pgs = false
 
static quad_t maxmbufmem
 
static counter_u64_t snd_tag_count
 
uma_zone_t zone_mbuf
 
uma_zone_t zone_clust
 
uma_zone_t zone_pack
 
uma_zone_t zone_jumbop
 
uma_zone_t zone_jumbo9
 
uma_zone_t zone_jumbo16
 

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$"  )

◆ _mb_unmapped_to_ext()

static struct mbuf * _mb_unmapped_to_ext ( struct mbuf *  m)
static

Definition at line 963 of file kern_mbuf.c.

References m_extadd(), m_freem(), mb_unmapped_free_mext(), and sf_buf_alloc().

Referenced by mb_unmapped_to_ext().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ _Static_assert()

_Static_assert ( sizeof(struct mbuf)<=  MSIZE,
"size of mbuf exceeds MSIZE"   
)

◆ CTASSERT()

CTASSERT ( (((MSIZE - 1) ^ MSIZE)+1)  ,
= =MSIZE 
)

◆ m_clget()

int m_clget ( struct mbuf *  m,
int  how 
)

Definition at line 1323 of file kern_mbuf.c.

References zone_clust, and zone_pack.

◆ m_cljget()

void * m_cljget ( struct mbuf *  m,
int  how,
int  size 
)

Definition at line 1350 of file kern_mbuf.c.

◆ m_extadd()

void m_extadd ( struct mbuf *  mb,
char *  buf,
u_int  size,
m_ext_free_t  freef,
void *  arg1,
void *  arg2,
int  flags,
int  type 
)

Definition at line 1560 of file kern_mbuf.c.

References buf, flags, and type.

Referenced by _mb_unmapped_to_ext().

Here is the caller graph for this function:

◆ m_free_raw()

void m_free_raw ( struct mbuf *  mb)

Definition at line 1599 of file kern_mbuf.c.

References zone_mbuf.

Referenced by ktls_work_thread(), m_get2(), m_get3(), m_getjcl(), mb_free_ext(), and mb_free_extpg().

Here is the caller graph for this function:

◆ m_freem()

◆ m_get2()

struct mbuf * m_get2 ( int  size,
int  how,
short  type,
int  flags 
)

Definition at line 1373 of file kern_mbuf.c.

References flags, m_free_raw(), type, zone_jumbop, zone_mbuf, and zone_pack.

Referenced by m_get3(), and sockargs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ m_get3()

struct mbuf * m_get3 ( int  size,
int  how,
short  type,
int  flags 
)

Definition at line 1407 of file kern_mbuf.c.

References flags, m_free_raw(), m_get2(), type, zone_jumbo16, zone_jumbo9, and zone_mbuf.

Here is the call graph for this function:

◆ m_getjcl()

struct mbuf * m_getjcl ( int  how,
short  type,
int  flags,
int  size 
)

Definition at line 1445 of file kern_mbuf.c.

References flags, m_free_raw(), type, and zone_mbuf.

Referenced by m_getm2().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ m_getm2()

struct mbuf * m_getm2 ( struct mbuf *  m,
int  len,
int  how,
short  type,
int  flags 
)

Definition at line 1478 of file kern_mbuf.c.

References flags, m_freem(), m_getjcl(), and type.

Referenced by m_uiotombuf().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ m_rcvif_restore()

struct ifnet * m_rcvif_restore ( struct mbuf *  m)

Definition at line 1651 of file kern_mbuf.c.

◆ m_rcvif_serialize()

void m_rcvif_serialize ( struct mbuf *  m)

Definition at line 1639 of file kern_mbuf.c.

◆ m_snd_tag_alloc()

int m_snd_tag_alloc ( struct ifnet *  ifp,
union if_snd_tag_alloc_params *  params,
struct m_snd_tag **  mstp 
)

Definition at line 1606 of file kern_mbuf.c.

◆ m_snd_tag_destroy()

void m_snd_tag_destroy ( struct m_snd_tag *  mst)

Definition at line 1628 of file kern_mbuf.c.

References snd_tag_count.

◆ m_snd_tag_init()

void m_snd_tag_init ( struct m_snd_tag *  mst,
struct ifnet *  ifp,
const struct if_snd_tag_sw *  sw 
)

Definition at line 1616 of file kern_mbuf.c.

References snd_tag_count.

◆ mb_alloc_ext_pgs()

struct mbuf * mb_alloc_ext_pgs ( int  how,
m_ext_free_t  ext_free 
)

Definition at line 1145 of file kern_mbuf.c.

Referenced by m_uiotombuf_nomap(), mb_alloc_ext_plus_pages(), and vn_sendfile().

Here is the caller graph for this function:

◆ mb_alloc_ext_plus_pages()

struct mbuf * mb_alloc_ext_plus_pages ( int  len,
int  how 
)

Definition at line 1669 of file kern_mbuf.c.

References mb_alloc_ext_pgs(), and mb_free_mext_pgs().

Referenced by mb_mapped_to_unmapped().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_ctor_clust()

static int mb_ctor_clust ( void *  mem,
int  size,
void *  arg,
int  how 
)
static

Definition at line 738 of file kern_mbuf.c.

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mb_ctor_mbuf()

static int mb_ctor_mbuf ( void *  mem,
int  size,
void *  arg,
int  how 
)
static

Definition at line 649 of file kern_mbuf.c.

References flags, and type.

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mb_ctor_pack()

static int mb_ctor_pack ( void *  mem,
int  size,
void *  arg,
int  how 
)
static

Definition at line 802 of file kern_mbuf.c.

References flags, and type.

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mb_dtor_mbuf()

static void mb_dtor_mbuf ( void *  mem,
int  size,
void *  arg 
)
static

Definition at line 680 of file kern_mbuf.c.

References flags, and m_tag_delete_chain().

Referenced by mbuf_init().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_dtor_pack()

static void mb_dtor_pack ( void *  mem,
int  size,
void *  arg 
)
static

Definition at line 698 of file kern_mbuf.c.

References m_tag_delete_chain(), zone_clust, and zone_pack.

Referenced by mbuf_init().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_free_ext()

void mb_free_ext ( struct mbuf *  m)

Definition at line 1176 of file kern_mbuf.c.

References m_free_raw(), zone_clust, zone_jumbo16, zone_jumbo9, zone_jumbop, and zone_pack.

Here is the call graph for this function:

◆ mb_free_extpg()

void mb_free_extpg ( struct mbuf *  m)

Definition at line 1269 of file kern_mbuf.c.

References ktls_enqueue_to_free(), and m_free_raw().

Referenced by mb_unmapped_free_mext().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_free_notready()

void mb_free_notready ( struct mbuf *  m,
int  count 
)

Definition at line 859 of file kern_mbuf.c.

References count.

Referenced by ktls_encrypt(), ktls_encrypt_async(), ktls_encrypt_cb(), and sendfile_iodone().

Here is the caller graph for this function:

◆ mb_mapped_to_unmapped()

struct mbuf * mb_mapped_to_unmapped ( struct mbuf *  mp,
int  len,
int  mlen,
int  how,
struct mbuf **  mlast 
)

Definition at line 1706 of file kern_mbuf.c.

References m_freem(), and mb_alloc_ext_plus_pages().

Here is the call graph for this function:

◆ mb_reclaim() [1/2]

static void mb_reclaim ( uma_zone_t zone  __unused,
int pending  __unused 
)
static

Definition at line 837 of file kern_mbuf.c.

References domain, and pr.

◆ mb_reclaim() [2/2]

static void mb_reclaim ( uma_zone_t  ,
int   
)
static

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mb_unmapped_compress()

int mb_unmapped_compress ( struct mbuf *  m)

Definition at line 882 of file kern_mbuf.c.

References buf, and m_copydata().

Referenced by sbcompress(), and sbready_compress().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_unmapped_free_mext()

static void mb_unmapped_free_mext ( struct mbuf *  m)
static

Definition at line 949 of file kern_mbuf.c.

References mb_free_extpg(), and sf_buf_free().

Referenced by _mb_unmapped_to_ext().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ mb_unmapped_to_ext()

struct mbuf * mb_unmapped_to_ext ( struct mbuf *  top)

Definition at line 1094 of file kern_mbuf.c.

References _mb_unmapped_to_ext(), and m_freem().

Here is the call graph for this function:

◆ mb_zfini_pack()

static void mb_zfini_pack ( void *  mem,
int  size 
)
static

Definition at line 784 of file kern_mbuf.c.

References zone_clust.

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mb_zinit_pack()

static int mb_zinit_pack ( void *  mem,
int  size,
int  how 
)
static

Definition at line 764 of file kern_mbuf.c.

References zone_clust.

Referenced by mbuf_init().

Here is the caller graph for this function:

◆ mbuf_init()

◆ SYSCTL_COUNTER_U64()

SYSCTL_COUNTER_U64 ( _kern_ipc  ,
OID_AUTO  ,
num_snd_tags  ,
CTLFLAG_RW  ,
snd_tag_count,
"# of active mbuf send tags"   
)

◆ sysctl_mb_use_ext_pgs()

static int sysctl_mb_use_ext_pgs ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 122 of file kern_mbuf.c.

References mb_use_ext_pgs, and sysctl_handle_int().

Here is the call graph for this function:

◆ sysctl_nmbclusters()

static int sysctl_nmbclusters ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 208 of file kern_mbuf.c.

References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_clust.

Here is the call graph for this function:

◆ sysctl_nmbjumbo16()

static int sysctl_nmbjumbo16 ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 275 of file kern_mbuf.c.

References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbo16.

Here is the call graph for this function:

◆ sysctl_nmbjumbo9()

static int sysctl_nmbjumbo9 ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 253 of file kern_mbuf.c.

References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbo9.

Here is the call graph for this function:

◆ sysctl_nmbjumbop()

static int sysctl_nmbjumbop ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 231 of file kern_mbuf.c.

References nmbclusters, nmbjumbo16, nmbjumbo9, nmbjumbop, nmbufs, sysctl_handle_int(), and zone_jumbop.

Here is the call graph for this function:

◆ sysctl_nmbufs()

static int sysctl_nmbufs ( SYSCTL_HANDLER_ARGS  )
static

Definition at line 297 of file kern_mbuf.c.

References nmbufs, sysctl_handle_int(), and zone_mbuf.

Here is the call graph for this function:

◆ SYSCTL_PROC() [1/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
mb_use_ext_pgs  ,
CTLTYPE_INT|  CTLFLAG_RW,
mb_use_ext_pgs,
,
sysctl_mb_use_ext_pgs  ,
"IU"  ,
"Use unmapped mbufs for sendfile(2) and TLS offload"   
)

◆ SYSCTL_PROC() [2/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
nmbclusters  ,
CTLTYPE_INT|CTLFLAG_RW|  CTLFLAG_MPSAFE,
nmbclusters,
,
sysctl_nmbclusters  ,
"IU"  ,
"Maximum number of mbuf clusters allowed"   
)

◆ SYSCTL_PROC() [3/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
nmbjumbo16  ,
CTLTYPE_INT|CTLFLAG_RW|  CTLFLAG_MPSAFE,
nmbjumbo16,
,
sysctl_nmbjumbo16  ,
"IU"  ,
"Maximum number of mbuf 16k jumbo clusters allowed"   
)

◆ SYSCTL_PROC() [4/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
nmbjumbo9  ,
CTLTYPE_INT|CTLFLAG_RW|  CTLFLAG_MPSAFE,
nmbjumbo9,
,
sysctl_nmbjumbo9  ,
"IU"  ,
"Maximum number of mbuf 9k jumbo clusters allowed"   
)

◆ SYSCTL_PROC() [5/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
nmbjumbop  ,
CTLTYPE_INT|CTLFLAG_RW|  CTLFLAG_MPSAFE,
nmbjumbop,
,
sysctl_nmbjumbop  ,
"IU"  ,
"Maximum number of mbuf page size jumbo clusters allowed"   
)

◆ SYSCTL_PROC() [6/6]

SYSCTL_PROC ( _kern_ipc  ,
OID_AUTO  ,
nmbufs  ,
CTLTYPE_INT|CTLFLAG_RW|  CTLFLAG_MPSAFE,
nmbufs,
,
sysctl_nmbufs  ,
"IU"  ,
"Maximum number of mbufs allowed"   
)

◆ SYSCTL_QUAD()

SYSCTL_QUAD ( _kern_ipc  ,
OID_AUTO  ,
maxmbufmem  ,
CTLFLAG_RDTUN|  CTLFLAG_NOFETCH,
maxmbufmem,
,
"Maximum real memory allocatable to various mbuf types"   
)

◆ SYSINIT() [1/2]

SYSINIT ( mbuf  ,
SI_SUB_MBUF  ,
SI_ORDER_FIRST  ,
mbuf_init  ,
NULL   
)

◆ SYSINIT() [2/2]

SYSINIT ( tunable_mbinit  ,
SI_SUB_KMEM  ,
SI_ORDER_MIDDLE  ,
tunable_mbinit  ,
NULL   
)

◆ tunable_mbinit()

static void tunable_mbinit ( void *  dummy)
static

Variable Documentation

◆ maxmbufmem

quad_t maxmbufmem
static

Definition at line 141 of file kern_mbuf.c.

Referenced by tunable_mbinit().

◆ mb_use_ext_pgs

bool mb_use_ext_pgs = false

Definition at line 119 of file kern_mbuf.c.

Referenced by sysctl_mb_use_ext_pgs(), tunable_mbinit(), and vn_sendfile().

◆ nmbclusters

◆ nmbjumbo16

◆ nmbjumbo9

◆ nmbjumbop

◆ nmbufs

◆ snd_tag_count

counter_u64_t snd_tag_count
static

Definition at line 146 of file kern_mbuf.c.

Referenced by m_snd_tag_destroy(), m_snd_tag_init(), and mbuf_init().

◆ zone_clust

uma_zone_t zone_clust

◆ zone_jumbo16

uma_zone_t zone_jumbo16

Definition at line 326 of file kern_mbuf.c.

Referenced by m_get3(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbo16().

◆ zone_jumbo9

uma_zone_t zone_jumbo9

Definition at line 325 of file kern_mbuf.c.

Referenced by m_get3(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbo9().

◆ zone_jumbop

uma_zone_t zone_jumbop

Definition at line 324 of file kern_mbuf.c.

Referenced by m_get2(), mb_free_ext(), mbuf_init(), and sysctl_nmbjumbop().

◆ zone_mbuf

uma_zone_t zone_mbuf

Definition at line 321 of file kern_mbuf.c.

Referenced by m_free_raw(), m_get2(), m_get3(), m_getjcl(), mbuf_init(), and sysctl_nmbufs().

◆ zone_pack

uma_zone_t zone_pack

Definition at line 323 of file kern_mbuf.c.

Referenced by m_clget(), m_get2(), mb_dtor_pack(), mb_free_ext(), and mbuf_init().