#include <sys/cdefs.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sockbuf.h>
#include <sys/sysctl.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/ethernet.h>
#include <net/bpf.h>
#include <net/vnet.h>
#include <netinet/in_systm.h>
#include <netinet/in.h>
#include <netinet/ip6.h>
#include <netinet/ip.h>
#include <netinet/ip_var.h>
#include <netinet/in_pcb.h>
#include <netinet6/in6_pcb.h>
#include <netinet/tcp.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_lro.h>
#include <netinet/tcp_var.h>
#include <netinet/tcpip.h>
#include <netinet/tcp_hpts.h>
#include <netinet/tcp_log_buf.h>
#include <netinet/udp.h>
#include <netinet6/ip6_var.h>
#include <machine/in_cksum.h>

Include dependency graph for tcp_lro.c:

Data Structures
struct	vxlan_header

Macros
#define	TCP_LRO_TS_OPTION

Functions
	__FBSDID ("$FreeBSD$")

static	MALLOC_DEFINE (M_LRO, "LRO", "LRO control structures")

static void	tcp_lro_rx_done (struct lro_ctrl *lc)

static int	tcp_lro_rx_common (struct lro_ctrl lc, struct mbuf m, uint32_t csum, bool use_hash)

	SYSCTL_NODE (_net_inet_tcp, OID_AUTO, lro, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "TCP LRO")

	SYSCTL_UINT (_net_inet_tcp_lro, OID_AUTO, entries, CTLFLAG_RDTUN\|CTLFLAG_MPSAFE, &tcp_lro_entries, 0, "default number of LRO entries")

	SYSCTL_UINT (_net_inet_tcp_lro, OID_AUTO, lro_cpu_threshold, CTLFLAG_RDTUN\|CTLFLAG_MPSAFE, &tcp_lro_cpu_set_thresh, 0, "Number of interrups in a row on the same CPU that will make us declare an 'affinity' cpu?")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, fullqueue, CTLFLAG_RD, &tcp_inp_lro_direct_queue, "Number of lro's fully queued to transport")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, wokeup, CTLFLAG_RD, &tcp_inp_lro_wokeup_queue, "Number of lro's where we woke up transport via hpts")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, compressed, CTLFLAG_RD, &tcp_inp_lro_compressed, "Number of lro's compressed and sent to transport")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, lockcnt, CTLFLAG_RD, &tcp_inp_lro_locks_taken, "Number of lro's inp_wlocks taken")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, extra_mbuf, CTLFLAG_RD, &tcp_extra_mbuf, "Number of times we had an extra compressed ack dropped into the tp")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, would_have_but, CTLFLAG_RD, &tcp_would_have_but, "Number of times we would have had an extra compressed, but mget failed")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, with_m_ackcmp, CTLFLAG_RD, &tcp_comp_total, "Number of mbufs queued with M_ACKCMP flags set")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, without_m_ackcmp, CTLFLAG_RD, &tcp_uncomp_total, "Number of mbufs queued without M_ACKCMP")

	SYSCTL_COUNTER_U64 (_net_inet_tcp_lro, OID_AUTO, lro_badcsum, CTLFLAG_RD, &tcp_bad_csums, "Number of packets that the common code saw with bad csums")

void	tcp_lro_reg_mbufq (void)

void	tcp_lro_dereg_mbufq (void)

static __inline void	tcp_lro_active_insert (struct lro_ctrl lc, struct lro_head bucket, struct lro_entry *le)

static __inline void	tcp_lro_active_remove (struct lro_entry *le)

int	tcp_lro_init (struct lro_ctrl *lc)

int	tcp_lro_init_args (struct lro_ctrl lc, struct ifnet ifp, unsigned lro_entries, unsigned lro_mbufs)

static void *	tcp_lro_low_level_parser (void ptr, struct lro_parser parser, bool update_data, bool is_vxlan, int mlen)

static struct lro_parser *	tcp_lro_parser (struct mbuf m, struct lro_parser po, struct lro_parser *pi, bool update_data)

static int	tcp_lro_trim_mbuf_chain (struct mbuf m, const struct lro_parser po)

static struct tcphdr *	tcp_lro_get_th (struct mbuf *m)

static void	lro_free_mbuf_chain (struct mbuf *m)

void	tcp_lro_free (struct lro_ctrl *lc)

static uint16_t	tcp_lro_rx_csum_tcphdr (const struct tcphdr *th)

static uint16_t	tcp_lro_rx_csum_data (const struct lro_parser *pa, uint16_t tcp_csum)

void	tcp_lro_flush_inactive (struct lro_ctrl lc, const struct timeval timeout)

static void	tcp_lro_assign_and_checksum_16 (uint16_t ptr, uint16_t value, uint16_t psum)

static uint16_t	tcp_lro_update_checksum (const struct lro_parser pa, const struct lro_entry le, uint16_t payload_len, uint16_t delta_sum)

static void	tcp_flush_out_entry (struct lro_ctrl lc, struct lro_entry le)

static void	tcp_set_entry_to_mbuf (struct lro_ctrl lc, struct lro_entry le, struct mbuf m, struct tcphdr th)

static void	tcp_push_and_replace (struct lro_ctrl lc, struct lro_entry le, struct mbuf *m)

static void	tcp_lro_mbuf_append_pkthdr (struct lro_entry le, const struct mbuf p)

static void	tcp_lro_condense (struct lro_ctrl lc, struct lro_entry le)

void	tcp_lro_flush (struct lro_ctrl lc, struct lro_entry le)

static uint64_t	tcp_lro_msb_64 (uint64_t x)

static void	tcp_lro_sort (struct lro_mbuf_sort *parray, uint32_t size)

void	tcp_lro_flush_all (struct lro_ctrl *lc)

static struct lro_head *	tcp_lro_rx_get_bucket (struct lro_ctrl lc, struct mbuf m, struct lro_parser *parser)

int	tcp_lro_rx (struct lro_ctrl lc, struct mbuf m, uint32_t csum)

void	tcp_lro_queue_mbuf (struct lro_ctrl lc, struct mbuf mb)

Variables
static long	tcplro_stacks_wanting_mbufq

counter_u64_t	tcp_inp_lro_direct_queue

counter_u64_t	tcp_inp_lro_wokeup_queue

counter_u64_t	tcp_inp_lro_compressed

counter_u64_t	tcp_inp_lro_locks_taken

counter_u64_t	tcp_extra_mbuf

counter_u64_t	tcp_would_have_but

counter_u64_t	tcp_comp_total

counter_u64_t	tcp_uncomp_total

counter_u64_t	tcp_bad_csums

static unsigned	tcp_lro_entries = TCP_LRO_ENTRIES

static uint32_t	tcp_lro_cpu_set_thresh = TCP_LRO_CPU_DECLARATION_THRESH

static const int	vxlan_csum

Macro Definition Documentation

◆ TCP_LRO_TS_OPTION

#define TCP_LRO_TS_OPTION

Value:

ntohl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) | \

(TCPOPT_TIMESTAMP << 8) | TCPOLEN_TIMESTAMP)

Definition at line 78 of file tcp_lro.c.

Function Documentation

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ lro_free_mbuf_chain()

static void lro_free_mbuf_chain ( struct mbuf * m )

static

Definition at line 478 of file tcp_lro.c.

Referenced by tcp_lro_free().

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_LRO	,
		"LRO"	,
		"LRO control structures"
	)

static

◆ SYSCTL_COUNTER_U64() [1/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		compressed	,
		CTLFLAG_RD	,
		&	tcp_inp_lro_compressed,
		"Number of lro's compressed and sent to transport"
	)

◆ SYSCTL_COUNTER_U64() [2/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		extra_mbuf	,
		CTLFLAG_RD	,
		&	tcp_extra_mbuf,
		"Number of times we had an extra compressed ack dropped into the tp"
	)

◆ SYSCTL_COUNTER_U64() [3/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		fullqueue	,
		CTLFLAG_RD	,
		&	tcp_inp_lro_direct_queue,
		"Number of lro's fully queued to transport"
	)

◆ SYSCTL_COUNTER_U64() [4/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		lockcnt	,
		CTLFLAG_RD	,
		&	tcp_inp_lro_locks_taken,
		"Number of lro's inp_wlocks taken"
	)

◆ SYSCTL_COUNTER_U64() [5/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		lro_badcsum	,
		CTLFLAG_RD	,
		&	tcp_bad_csums,
		"Number of packets that the common code saw with bad csums"
	)

◆ SYSCTL_COUNTER_U64() [6/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		with_m_ackcmp	,
		CTLFLAG_RD	,
		&	tcp_comp_total,
		"Number of mbufs queued with M_ACKCMP flags set"
	)

◆ SYSCTL_COUNTER_U64() [7/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		without_m_ackcmp	,
		CTLFLAG_RD	,
		&	tcp_uncomp_total,
		"Number of mbufs queued without M_ACKCMP"
	)

◆ SYSCTL_COUNTER_U64() [8/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		wokeup	,
		CTLFLAG_RD	,
		&	tcp_inp_lro_wokeup_queue,
		"Number of lro's where we woke up transport via hpts"
	)

◆ SYSCTL_COUNTER_U64() [9/9]

SYSCTL_COUNTER_U64	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		would_have_but	,
		CTLFLAG_RD	,
		&	tcp_would_have_but,
		"Number of times we would have had an extra	compressed,
		but mget failed"
	)

◆ SYSCTL_NODE()

SYSCTL_NODE	(	_net_inet_tcp	,
		OID_AUTO	,
		lro	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"TCP LRO"
	)

◆ SYSCTL_UINT() [1/2]

SYSCTL_UINT	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		entries	,
		CTLFLAG_RDTUN\|	CTLFLAG_MPSAFE,
		&	tcp_lro_entries,
		0	,
		"default number of LRO entries"
	)

◆ SYSCTL_UINT() [2/2]

SYSCTL_UINT	(	_net_inet_tcp_lro	,
		OID_AUTO	,
		lro_cpu_threshold	,
		CTLFLAG_RDTUN\|	CTLFLAG_MPSAFE,
		&	tcp_lro_cpu_set_thresh,
		0	,
		"Number of interrups in a row on the same CPU that will make us declare an 'affinity' cpu?"
	)

◆ tcp_flush_out_entry()

static void tcp_flush_out_entry	(	struct lro_ctrl *	lc,
		struct lro_entry *	le
	)

static

Definition at line 820 of file tcp_lro.c.

References lro_parser::data, lro_ctrl::ifp, lro_entry::inner, LRO_FLAG_DECRYPTED, lro_address::lro_flags, lro_ctrl::lro_queued, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV6_TCP, LRO_TYPE_NONE, lro_entry::m_head, lro_entry::needs_merge, lro_entry::outer, tcp_lro_update_checksum(), and lro_parser::total_hdr_len.

Referenced by tcp_lro_flush(), and tcp_push_and_replace().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_active_insert()

static __inline void tcp_lro_active_insert	(	struct lro_ctrl *	lc,
		struct lro_head *	bucket,
		struct lro_entry *	le
	)

static

Definition at line 148 of file tcp_lro.c.

References bucket, lro_ctrl::lro_active, and next.

Referenced by tcp_lro_rx_common().

Here is the caller graph for this function:

◆ tcp_lro_active_remove()

static __inline void tcp_lro_active_remove ( struct lro_entry * le )

static

Definition at line 157 of file tcp_lro.c.

References next.

Referenced by tcp_lro_flush_inactive(), tcp_lro_free(), and tcp_lro_rx_done().

Here is the caller graph for this function:

◆ tcp_lro_assign_and_checksum_16()

static void tcp_lro_assign_and_checksum_16	(	uint16_t *	ptr,
		uint16_t	value,
		uint16_t *	psum
	)

inlinestatic

Definition at line 696 of file tcp_lro.c.

Referenced by tcp_lro_update_checksum().

Here is the caller graph for this function:

◆ tcp_lro_condense()

static void tcp_lro_condense	(	struct lro_ctrl *	lc,
		struct lro_entry *	le
	)

static

Definition at line 993 of file tcp_lro.c.

References lro_entry::ack_seq, lro_entry::flags, lro_ctrl::lro_ackcnt_lim, lro_ctrl::lro_length_lim, lro_entry::m_head, lro_entry::m_tail, lro_entry::needs_merge, lro_entry::next_seq, SEQ_GT, tcp_get_flags(), tcp_lro_get_th(), tcp_lro_mbuf_append_pkthdr(), TCP_LRO_TS_OPTION, tcp_push_and_replace(), lro_entry::tsecr, TSTMP_GT, lro_entry::tsval, WIN_GT, and lro_entry::window.

Referenced by tcp_lro_flush().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_dereg_mbufq()

void tcp_lro_dereg_mbufq ( void )

Definition at line 142 of file tcp_lro.c.

References tcplro_stacks_wanting_mbufq.

Referenced by tcp_addbbr(), and tcp_addrack().

Here is the caller graph for this function:

◆ tcp_lro_flush()

void tcp_lro_flush	(	struct lro_ctrl *	lc,
		struct lro_entry *	le
	)

Definition at line 1383 of file tcp_lro.c.

References lro_ctrl::ifp, lro_ctrl::lro_flushed, lro_ctrl::lro_free, next, tcp_flush_out_entry(), and tcp_lro_condense().

Referenced by tcp_lro_flush_inactive(), and tcp_lro_rx_done().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_flush_all()

void tcp_lro_flush_all ( struct lro_ctrl * lc )

Definition at line 1496 of file tcp_lro.c.

References lro_ctrl::ifp, lro_ctrl::lro_cnt_of_same_cpu, lro_ctrl::lro_cpu_is_set, lro_ctrl::lro_flushed, lro_ctrl::lro_last_cpu, lro_ctrl::lro_last_queue_time, lro_ctrl::lro_mbuf_count, lro_ctrl::lro_mbuf_data, lro_ctrl::lro_queued, lro_mbuf_sort::mb, lro_mbuf_sort::seq, tcp_lro_cpu_set_thresh, tcp_lro_rx_common(), tcp_lro_rx_done(), tcp_lro_sort(), and tcp_run_hpts().

Referenced by tcp_lro_queue_mbuf().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_flush_inactive()

void tcp_lro_flush_inactive	(	struct lro_ctrl *	lc,
		const struct timeval *	timeout
	)

Definition at line 596 of file tcp_lro.c.

References lro_entry::alloc_time, lro_ctrl::lro_active, next, tcp_lro_active_remove(), and tcp_lro_flush().

Here is the call graph for this function:

◆ tcp_lro_free()

void tcp_lro_free ( struct lro_ctrl * lc )

Definition at line 491 of file tcp_lro.c.

References lro_ctrl::lro_active, lro_ctrl::lro_free, lro_free_mbuf_chain(), lro_ctrl::lro_hash, lro_ctrl::lro_hashsz, lro_ctrl::lro_mbuf_count, lro_ctrl::lro_mbuf_data, lro_entry::m_head, lro_mbuf_sort::mb, and tcp_lro_active_remove().

Here is the call graph for this function:

◆ tcp_lro_get_th()

static struct tcphdr * tcp_lro_get_th ( struct mbuf * m )

static

Definition at line 472 of file tcp_lro.c.

Referenced by tcp_lro_condense().

Here is the caller graph for this function:

◆ tcp_lro_init()

int tcp_lro_init ( struct lro_ctrl * lc )

Definition at line 165 of file tcp_lro.c.

References tcp_lro_entries, and tcp_lro_init_args().

Here is the call graph for this function:

◆ tcp_lro_init_args()

int tcp_lro_init_args	(	struct lro_ctrl *	lc,
		struct ifnet *	ifp,
		unsigned	lro_entries,
		unsigned	lro_mbufs
	)

Definition at line 171 of file tcp_lro.c.

References lro_ctrl::ifp, lro_ctrl::lro_ackcnt_lim, lro_ctrl::lro_active, lro_ctrl::lro_bad_csum, lro_ctrl::lro_cnt, lro_ctrl::lro_flushed, lro_ctrl::lro_free, lro_ctrl::lro_hash, lro_ctrl::lro_hashsz, lro_ctrl::lro_length_lim, lro_ctrl::lro_mbuf_count, lro_ctrl::lro_mbuf_data, lro_ctrl::lro_mbuf_max, lro_ctrl::lro_queued, next, TCP_LRO_ACKCNT_MAX, and TCP_LRO_LENGTH_MAX.

Referenced by tcp_lro_init().

Here is the caller graph for this function:

◆ tcp_lro_low_level_parser()

static void * tcp_lro_low_level_parser	(	void *	ptr,
		struct lro_parser *	parser,
		bool	update_data,
		bool	is_vxlan,
		int	mlen
	)

inlinestatic

Definition at line 231 of file tcp_lro.c.

References lro_address::d_addr, lro_address::d_port, lro_parser::data, lro_parser::ip4, lro_parser::ip6, ip6_hdr::ip6_dst, ip6_hdr::ip6_src, ip::ip_dst, ip::ip_hl, IP_MF, ip::ip_off, IP_OFFMASK, ip::ip_p, IPPROTO_TCP, IPPROTO_UDP, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV4_UDP, LRO_TYPE_IPV6_TCP, LRO_TYPE_IPV6_UDP, lro_address::s_addr, lro_address::s_port, lro_parser::tcp, lro_parser::total_hdr_len, lro_parser::udp, udphdr::uh_dport, udphdr::uh_sport, lro_address::v4, lro_address::v6, lro_address::vlan_id, lro_address::vxlan_vni, and vxlan_header::vxlh_vni.

Referenced by tcp_lro_parser().

Here is the caller graph for this function:

◆ tcp_lro_mbuf_append_pkthdr()

static void tcp_lro_mbuf_append_pkthdr	(	struct lro_entry *	le,
		const struct mbuf *	p
	)

static

Definition at line 967 of file tcp_lro.c.

References lro_entry::m_head, and lro_entry::needs_merge.

Referenced by tcp_lro_condense().

Here is the caller graph for this function:

◆ tcp_lro_msb_64()

static uint64_t tcp_lro_msb_64 ( uint64_t x )

inlinestatic

Definition at line 1411 of file tcp_lro.c.

Referenced by tcp_lro_sort().

Here is the caller graph for this function:

◆ tcp_lro_parser()

static struct lro_parser * tcp_lro_parser	(	struct mbuf *	m,
		struct lro_parser *	po,
		struct lro_parser *	pi,
		bool	update_data
	)

inlinestatic

Definition at line 382 of file tcp_lro.c.

References lro_parser::data, LRO_FLAG_DECRYPTED, lro_address::lro_flags, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV4_UDP, LRO_TYPE_IPV6_TCP, LRO_TYPE_IPV6_UDP, tcp_lro_low_level_parser(), lro_parser::total_hdr_len, lro_address::vlan_id, and vxlan_csum.

Referenced by tcp_lro_rx_common(), and tcp_push_and_replace().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_queue_mbuf()

void tcp_lro_queue_mbuf	(	struct lro_ctrl *	lc,
		struct mbuf *	mb
	)

Definition at line 1941 of file tcp_lro.c.

References lro_ctrl::ifp, lro_ctrl::lro_mbuf_count, lro_ctrl::lro_mbuf_data, lro_ctrl::lro_mbuf_max, lro_mbuf_sort::mb, lro_mbuf_sort::seq, and tcp_lro_flush_all().

Here is the call graph for this function:

◆ tcp_lro_reg_mbufq()

void tcp_lro_reg_mbufq ( void )

Definition at line 136 of file tcp_lro.c.

References tcplro_stacks_wanting_mbufq.

Referenced by tcp_addbbr(), and tcp_addrack().

Here is the caller graph for this function:

◆ tcp_lro_rx()

int tcp_lro_rx	(	struct lro_ctrl *	lc,
		struct mbuf *	m,
		uint32_t	csum
	)

Definition at line 1916 of file tcp_lro.c.

References lro_ctrl::ifp, lro_ctrl::lro_last_queue_time, tcp_bad_csums, TCP_LRO_CANNOT, and tcp_lro_rx_common().

Here is the call graph for this function:

◆ tcp_lro_rx_common()

static int tcp_lro_rx_common	(	struct lro_ctrl *	lc,
		struct mbuf *	m,
		uint32_t	csum,
		bool	use_hash
	)

static

Definition at line 1769 of file tcp_lro.c.

References lro_entry::ack_seq, lro_entry::alloc_time, bucket, lro_parser::data, lro_ctrl::ifp, lro_entry::inner, lro_parser::ip4, lro_address_compare(), lro_ctrl::lro_free, lro_ctrl::lro_hash, lro_ctrl::lro_last_queue_time, lro_address::lro_type, LRO_TYPE_IPV4_TCP, lro_entry::m_last_mbuf, next, lro_entry::outer, SEQ_LT, lro_parser::tcp, tcp_bad_csums, tcp_get_flags(), tcp_lro_active_insert(), TCP_LRO_CANNOT, TCP_LRO_NO_ENTRIES, TCP_LRO_NOT_SUPPORTED, tcp_lro_parser(), tcp_lro_rx_csum_data(), tcp_lro_rx_get_bucket(), tcp_lro_trim_mbuf_chain(), tcp_set_entry_to_mbuf(), and V_ipforwarding.

Referenced by tcp_lro_flush_all(), and tcp_lro_rx().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_rx_csum_data()

static uint16_t tcp_lro_rx_csum_data	(	const struct lro_parser *	pa,
		uint16_t	tcp_csum
	)

static

Definition at line 543 of file tcp_lro.c.

References lro_parser::data, in_addword(), in_pseudo(), lro_parser::ip4, lro_parser::ip6, ip::ip_dst, ip::ip_len, IPPROTO_TCP, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV6_TCP, in_addr::s_addr, and lro_parser::tcp.

Referenced by tcp_lro_rx_common().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_rx_csum_tcphdr()

static uint16_t tcp_lro_rx_csum_tcphdr ( const struct tcphdr * th )

static

Definition at line 521 of file tcp_lro.c.

Referenced by tcp_lro_update_checksum().

Here is the caller graph for this function:

◆ tcp_lro_rx_done()

static void tcp_lro_rx_done ( struct lro_ctrl * lc )

static

Definition at line 585 of file tcp_lro.c.

References lro_ctrl::lro_active, tcp_lro_active_remove(), and tcp_lro_flush().

Referenced by tcp_lro_flush_all().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_rx_get_bucket()

static struct lro_head * tcp_lro_rx_get_bucket	(	struct lro_ctrl *	lc,
		struct mbuf *	m,
		struct lro_parser *	parser
	)

static

Definition at line 1755 of file tcp_lro.c.

References lro_parser::data, lro_ctrl::lro_hash, lro_ctrl::lro_hashsz, LRO_RAW_ADDRESS_MAX, and lro_address::raw.

Referenced by tcp_lro_rx_common().

Here is the caller graph for this function:

◆ tcp_lro_sort()

static void tcp_lro_sort	(	struct lro_mbuf_sort *	parray,
		uint32_t	size
	)

static

Definition at line 1433 of file tcp_lro.c.

References lro_mbuf_sort::seq, tcp_lro_msb_64(), and tcp_lro_sort().

Referenced by tcp_lro_flush_all(), and tcp_lro_sort().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_lro_trim_mbuf_chain()

static int tcp_lro_trim_mbuf_chain	(	struct mbuf *	m,
		const struct lro_parser *	po
	)

inlinestatic

Definition at line 437 of file tcp_lro.c.

References lro_parser::data, lro_parser::ip4, lro_parser::ip6, ip::ip_len, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV6_TCP, and TCP_LRO_CANNOT.

Referenced by tcp_lro_rx_common().

Here is the caller graph for this function:

◆ tcp_lro_update_checksum()

static uint16_t tcp_lro_update_checksum	(	const struct lro_parser *	pa,
		const struct lro_entry *	le,
		uint16_t	payload_len,
		uint16_t	delta_sum
	)

static

Definition at line 708 of file tcp_lro.c.

References lro_entry::ack_seq, lro_parser::data, lro_parser::ip4, lro_parser::ip6, ip::ip_hl, ip::ip_len, ip::ip_sum, lro_address::lro_type, LRO_TYPE_IPV4_TCP, LRO_TYPE_IPV4_UDP, LRO_TYPE_IPV6_TCP, LRO_TYPE_IPV6_UDP, lro_parser::tcp, tcp_lro_assign_and_checksum_16(), tcp_lro_rx_csum_tcphdr(), lro_entry::timestamp, lro_entry::tsecr, lro_entry::tsval, lro_parser::udp, udphdr::uh_sum, udphdr::uh_ulen, and lro_entry::window.

Referenced by tcp_flush_out_entry().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_push_and_replace()

static void tcp_push_and_replace	(	struct lro_ctrl *	lc,
		struct lro_entry *	le,
		struct mbuf *	m
	)

static

Definition at line 933 of file tcp_lro.c.

References lro_entry::inner, lro_entry::m_head, lro_entry::outer, lro_parser::tcp, tcp_flush_out_entry(), tcp_lro_parser(), and tcp_set_entry_to_mbuf().

Referenced by tcp_lro_condense().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ tcp_set_entry_to_mbuf()

static void tcp_set_entry_to_mbuf	(	struct lro_ctrl *	lc,
		struct lro_entry *	le,
		struct mbuf *	m,
		struct tcphdr *	th
	)

static

Definition at line 895 of file tcp_lro.c.

References lro_entry::ack_seq, lro_entry::flags, lro_entry::m_head, lro_entry::m_tail, lro_entry::needs_merge, lro_entry::next_seq, tcp_get_flags(), TCP_LRO_TS_OPTION, lro_entry::timestamp, lro_entry::tsecr, lro_entry::tsval, and lro_entry::window.

Referenced by tcp_lro_rx_common(), and tcp_push_and_replace().

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

◆ tcp_bad_csums

counter_u64_t tcp_bad_csums

Definition at line 104 of file tcp_lro.c.

Referenced by tcp_init(), tcp_lro_rx(), and tcp_lro_rx_common().

◆ tcp_comp_total

counter_u64_t tcp_comp_total

Definition at line 102 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_extra_mbuf

counter_u64_t tcp_extra_mbuf

Definition at line 100 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_inp_lro_compressed

counter_u64_t tcp_inp_lro_compressed

Definition at line 98 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_inp_lro_direct_queue

counter_u64_t tcp_inp_lro_direct_queue

Definition at line 96 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_inp_lro_locks_taken

counter_u64_t tcp_inp_lro_locks_taken

Definition at line 99 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_inp_lro_wokeup_queue

counter_u64_t tcp_inp_lro_wokeup_queue

Definition at line 97 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_lro_cpu_set_thresh

uint32_t tcp_lro_cpu_set_thresh = TCP_LRO_CPU_DECLARATION_THRESH

static

Definition at line 111 of file tcp_lro.c.

Referenced by tcp_lro_flush_all().

◆ tcp_lro_entries

unsigned tcp_lro_entries = TCP_LRO_ENTRIES

static

Definition at line 106 of file tcp_lro.c.

Referenced by tcp_lro_init().

◆ tcp_uncomp_total

counter_u64_t tcp_uncomp_total

Definition at line 103 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcp_would_have_but

counter_u64_t tcp_would_have_but

Definition at line 101 of file tcp_lro.c.

Referenced by tcp_init().

◆ tcplro_stacks_wanting_mbufq

long tcplro_stacks_wanting_mbufq

static

Definition at line 95 of file tcp_lro.c.

Referenced by tcp_lro_dereg_mbufq(), and tcp_lro_reg_mbufq().

◆ vxlan_csum

const int vxlan_csum

static

Initial value:

= CSUM_INNER_L3_CALC | CSUM_INNER_L3_VALID |

CSUM_INNER_L4_CALC | CSUM_INNER_L4_VALID

Definition at line 378 of file tcp_lro.c.

Referenced by tcp_lro_parser().

Data Structures

Macros

Functions

Variables

Macro Definition Documentation

◆ TCP_LRO_TS_OPTION

Function Documentation

◆ __FBSDID()

◆ lro_free_mbuf_chain()

◆ MALLOC_DEFINE()

◆ SYSCTL_COUNTER_U64() [1/9]

◆ SYSCTL_COUNTER_U64() [2/9]

◆ SYSCTL_COUNTER_U64() [3/9]

◆ SYSCTL_COUNTER_U64() [4/9]

◆ SYSCTL_COUNTER_U64() [5/9]

◆ SYSCTL_COUNTER_U64() [6/9]

◆ SYSCTL_COUNTER_U64() [7/9]

◆ SYSCTL_COUNTER_U64() [8/9]

◆ SYSCTL_COUNTER_U64() [9/9]

◆ SYSCTL_NODE()

◆ SYSCTL_UINT() [1/2]

◆ SYSCTL_UINT() [2/2]

◆ tcp_flush_out_entry()

◆ tcp_lro_active_insert()

◆ tcp_lro_active_remove()

◆ tcp_lro_assign_and_checksum_16()

◆ tcp_lro_condense()

◆ tcp_lro_dereg_mbufq()

◆ tcp_lro_flush()

◆ tcp_lro_flush_all()

◆ tcp_lro_flush_inactive()

◆ tcp_lro_free()

◆ tcp_lro_get_th()

◆ tcp_lro_init()

◆ tcp_lro_init_args()

◆ tcp_lro_low_level_parser()

◆ tcp_lro_mbuf_append_pkthdr()

◆ tcp_lro_msb_64()

◆ tcp_lro_parser()

◆ tcp_lro_queue_mbuf()

◆ tcp_lro_reg_mbufq()

◆ tcp_lro_rx()

◆ tcp_lro_rx_common()

◆ tcp_lro_rx_csum_data()

◆ tcp_lro_rx_csum_tcphdr()

◆ tcp_lro_rx_done()

◆ tcp_lro_rx_get_bucket()

◆ tcp_lro_sort()

◆ tcp_lro_trim_mbuf_chain()

◆ tcp_lro_update_checksum()

◆ tcp_push_and_replace()

◆ tcp_set_entry_to_mbuf()

Variable Documentation

◆ tcp_bad_csums

◆ tcp_comp_total

◆ tcp_extra_mbuf

◆ tcp_inp_lro_compressed

◆ tcp_inp_lro_direct_queue

◆ tcp_inp_lro_locks_taken

◆ tcp_inp_lro_wokeup_queue

◆ tcp_lro_cpu_set_thresh

◆ tcp_lro_entries

◆ tcp_uncomp_total

◆ tcp_would_have_but

◆ tcplro_stacks_wanting_mbufq

◆ vxlan_csum