#include <sys/cdefs.h>
#include "opt_inet.h"
#include "opt_inet6.h"
#include "opt_kern_tls.h"
#include "opt_ratelimit.h"
#include "opt_rss.h"
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/domainset.h>
#include <sys/endian.h>
#include <sys/ktls.h>
#include <sys/lock.h>
#include <sys/mbuf.h>
#include <sys/mutex.h>
#include <sys/rmlock.h>
#include <sys/proc.h>
#include <sys/protosw.h>
#include <sys/refcount.h>
#include <sys/smp.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/sysctl.h>
#include <sys/taskqueue.h>
#include <sys/kthread.h>
#include <sys/uio.h>
#include <sys/vmmeter.h>
#include <machine/vmparam.h>
#include <net/if.h>
#include <net/if_var.h>
#include <net/route.h>
#include <net/route/nhop.h>
#include <netinet/tcp_var.h>
#include <opencrypto/cryptodev.h>
#include <opencrypto/ktls.h>
#include <vm/uma_dbg.h>
#include <vm/vm.h>
#include <vm/vm_pageout.h>
#include <vm/vm_page.h>
#include <vm/vm_pagequeue.h>

Include dependency graph for uipc_ktls.c:

Data Structures
struct	ktls_wq

struct	ktls_alloc_thread

struct	ktls_domain_info

Functions
	__FBSDID ("$FreeBSD$")

struct ktls_wq	__aligned (CACHE_LINE_SIZE)

	SX_SYSINIT (ktls_init_lock, &ktls_init_lock, "ktls init")

	SYSCTL_NODE (_kern_ipc, OID_AUTO, tls, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "Kernel TLS offload")

	SYSCTL_NODE (_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW\|CTLFLAG_MPSAFE, 0, "Kernel TLS offload stats")

	SYSCTL_INT (_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN, &ktls_bind_threads, 0, "Bind crypto threads to cores (1) or cores and domains (2) at boot")

	SYSCTL_UINT (_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, &ktls_maxlen, 0, "Maximum TLS record size")

	SYSCTL_INT (_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD, &ktls_number_threads, 0, "Number of TLS threads in thread-pool")

	SYSCTL_UINT (_kern_ipc_tls, OID_AUTO, ifnet_max_rexmit_pct, CTLFLAG_RWTUN, &ktls_ifnet_max_rexmit_pct, 2, "Max percent bytes retransmitted before ifnet TLS is disabled")

	SYSCTL_BOOL (_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, &ktls_offload_enable, 0, "Enable support for kernel TLS offload")

	SYSCTL_BOOL (_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RWTUN, &ktls_cbc_enable, 1, "Enable Support of AES-CBC crypto for kernel TLS")

	SYSCTL_BOOL (_kern_ipc_tls, OID_AUTO, sw_buffer_cache, CTLFLAG_RDTUN, &ktls_sw_buffer_cache, 1, "Enable caching of output buffers for SW encryption")

	SYSCTL_INT (_kern_ipc_tls, OID_AUTO, max_alloc, CTLFLAG_RWTUN, &ktls_max_alloc, 128, "Max number of 16k buffers to allocate in thread context")

static	COUNTER_U64_DEFINE_EARLY (ktls_tasks_active)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD, &ktls_tasks_active, "Number of active tasks")

static	COUNTER_U64_DEFINE_EARLY (ktls_cnt_tx_pending)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, sw_tx_pending, CTLFLAG_RD, &ktls_cnt_tx_pending, "Number of TLS 1.0 records waiting for earlier TLS records")

static	COUNTER_U64_DEFINE_EARLY (ktls_cnt_tx_queued)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, sw_tx_inqueue, CTLFLAG_RD, &ktls_cnt_tx_queued, "Number of TLS records in queue to tasks for SW encryption")

static	COUNTER_U64_DEFINE_EARLY (ktls_cnt_rx_queued)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, sw_rx_inqueue, CTLFLAG_RD, &ktls_cnt_rx_queued, "Number of TLS sockets in queue to tasks for SW decryption")

static	COUNTER_U64_DEFINE_EARLY (ktls_offload_total)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, offload_total, CTLFLAG_RD, &ktls_offload_total, "Total successful TLS setups (parameters set)")

static	COUNTER_U64_DEFINE_EARLY (ktls_offload_enable_calls)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, enable_calls, CTLFLAG_RD, &ktls_offload_enable_calls, "Total number of TLS enable calls made")

static	COUNTER_U64_DEFINE_EARLY (ktls_offload_active)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, active, CTLFLAG_RD, &ktls_offload_active, "Total Active TLS sessions")

static	COUNTER_U64_DEFINE_EARLY (ktls_offload_corrupted_records)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, corrupted_records, CTLFLAG_RD, &ktls_offload_corrupted_records, "Total corrupted TLS records received")

static	COUNTER_U64_DEFINE_EARLY (ktls_offload_failed_crypto)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, failed_crypto, CTLFLAG_RD, &ktls_offload_failed_crypto, "Total TLS crypto failures")

static	COUNTER_U64_DEFINE_EARLY (ktls_switch_to_ifnet)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, switch_to_ifnet, CTLFLAG_RD, &ktls_switch_to_ifnet, "TLS sessions switched from SW to ifnet")

static	COUNTER_U64_DEFINE_EARLY (ktls_switch_to_sw)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, switch_to_sw, CTLFLAG_RD, &ktls_switch_to_sw, "TLS sessions switched from ifnet to SW")

static	COUNTER_U64_DEFINE_EARLY (ktls_switch_failed)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, switch_failed, CTLFLAG_RD, &ktls_switch_failed, "TLS sessions unable to switch between SW and ifnet")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_disable_fail)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_failed, CTLFLAG_RD, &ktls_ifnet_disable_fail, "TLS sessions unable to switch to SW from ifnet")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_disable_ok)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_stats, OID_AUTO, ifnet_disable_ok, CTLFLAG_RD, &ktls_ifnet_disable_ok, "TLS sessions able to switch to SW from ifnet")

	SYSCTL_NODE (_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD\|CTLFLAG_MPSAFE, 0, "Software TLS session stats")

	SYSCTL_NODE (_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD\|CTLFLAG_MPSAFE, 0, "Hardware (ifnet) TLS session stats")

static	COUNTER_U64_DEFINE_EARLY (ktls_sw_cbc)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc, "Active number of software TLS sessions using AES-CBC")

static	COUNTER_U64_DEFINE_EARLY (ktls_sw_gcm)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_sw, OID_AUTO, gcm, CTLFLAG_RD, &ktls_sw_gcm, "Active number of software TLS sessions using AES-GCM")

static	COUNTER_U64_DEFINE_EARLY (ktls_sw_chacha20)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_sw, OID_AUTO, chacha20, CTLFLAG_RD, &ktls_sw_chacha20, "Active number of software TLS sessions using Chacha20-Poly1305")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_cbc)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, cbc, CTLFLAG_RD, &ktls_ifnet_cbc, "Active number of ifnet TLS sessions using AES-CBC")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_gcm)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, gcm, CTLFLAG_RD, &ktls_ifnet_gcm, "Active number of ifnet TLS sessions using AES-GCM")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_chacha20)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, chacha20, CTLFLAG_RD, &ktls_ifnet_chacha20, "Active number of ifnet TLS sessions using Chacha20-Poly1305")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_reset)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, reset, CTLFLAG_RD, &ktls_ifnet_reset, "TLS sessions updated to a new ifnet send tag")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_reset_dropped)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, reset_dropped, CTLFLAG_RD, &ktls_ifnet_reset_dropped, "TLS sessions dropped after failing to update ifnet send tag")

static	COUNTER_U64_DEFINE_EARLY (ktls_ifnet_reset_failed)

	SYSCTL_COUNTER_U64 (_kern_ipc_tls_ifnet, OID_AUTO, reset_failed, CTLFLAG_RD, &ktls_ifnet_reset_failed, "TLS sessions that failed to allocate a new ifnet send tag")

	SYSCTL_UINT (_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN, &ktls_ifnet_permitted, 1, "Whether to permit hardware (ifnet) TLS sessions")

static	MALLOC_DEFINE (M_KTLS, "ktls", "Kernel TLS")

static void	ktls_cleanup (struct ktls_session *tls)

static void	ktls_work_thread (void *ctx)

static void	ktls_alloc_thread (void *ctx)

static int	ktls_buffer_import (void arg, void *store, int count, int domain, int flags)

static void	ktls_buffer_release (void arg __unused, void *store, int count)

static void	ktls_free_mext_contig (struct mbuf *m)

static int	ktls_init (void)

static int	ktls_start_kthreads (void)

void	ktls_destroy (struct ktls_session *tls)

void	ktls_seq (struct sockbuf sb, struct mbuf m)

void	ktls_frame (struct mbuf top, struct ktls_session tls, int *enq_cnt, uint8_t record_type)

bool	ktls_permit_empty_frames (struct ktls_session *tls)

void	ktls_check_rx (struct sockbuf *sb)

static struct mbuf *	ktls_detach_record (struct sockbuf *sb, int len)

static int	tls13_find_record_type (struct ktls_session tls, struct mbuf m, int tls_len, int trailer_len, uint8_t record_typep)

static void	ktls_decrypt (struct socket *so)

void	ktls_enqueue_to_free (struct mbuf *m)

static void *	ktls_buffer_alloc (struct ktls_wq wq, struct mbuf m)

static int	ktls_encrypt_record (struct ktls_wq wq, struct mbuf m, struct ktls_session tls, struct ktls_ocf_encrypt_state state)

static u_int	ktls_batched_records (struct mbuf *m)

void	ktls_enqueue (struct mbuf m, struct socket so, int page_count)

static void	ktls_finish_nonanon (struct mbuf m, struct ktls_ocf_encrypt_state state)

static __noinline void	ktls_encrypt (struct ktls_wq wq, struct mbuf top)

void	ktls_encrypt_cb (struct ktls_ocf_encrypt_state *state, int error)

static __noinline void	ktls_encrypt_async (struct ktls_wq wq, struct mbuf top)

static int	ktls_bind_domain (int domain)

Variables
struct mtx	mtx

struct ktls_alloc_thread	__aligned

struct ktls_domain_info	ktls_domains [MAXMEMDOM]

static struct ktls_wq *	ktls_wq

static struct proc *	ktls_proc

static uma_zone_t	ktls_session_zone

static uma_zone_t	ktls_buffer_zone

static uint16_t	ktls_cpuid_lookup [MAXCPU]

static int	ktls_init_state

static struct sx	ktls_init_lock

static int	ktls_bind_threads

static u_int	ktls_maxlen = 16384

static int	ktls_number_threads

unsigned int	ktls_ifnet_max_rexmit_pct = 2

static bool	ktls_offload_enable

static bool	ktls_cbc_enable = true

static bool	ktls_sw_buffer_cache = true

static int	ktls_max_alloc = 128

static int	ktls_ifnet_permitted

Function Documentation

◆ __aligned()

struct ktls_wq __aligned ( CACHE_LINE_SIZE )

◆ __FBSDID()

__FBSDID ( "$FreeBSD$" )

◆ COUNTER_U64_DEFINE_EARLY() [1/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_cnt_rx_queued )

static

◆ COUNTER_U64_DEFINE_EARLY() [2/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_cnt_tx_pending )

static

◆ COUNTER_U64_DEFINE_EARLY() [3/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_cnt_tx_queued )

static

◆ COUNTER_U64_DEFINE_EARLY() [4/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_cbc )

static

◆ COUNTER_U64_DEFINE_EARLY() [5/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_chacha20 )

static

◆ COUNTER_U64_DEFINE_EARLY() [6/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_disable_fail )

static

◆ COUNTER_U64_DEFINE_EARLY() [7/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_disable_ok )

static

◆ COUNTER_U64_DEFINE_EARLY() [8/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_gcm )

static

◆ COUNTER_U64_DEFINE_EARLY() [9/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_reset )

static

◆ COUNTER_U64_DEFINE_EARLY() [10/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_reset_dropped )

static

◆ COUNTER_U64_DEFINE_EARLY() [11/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_ifnet_reset_failed )

static

◆ COUNTER_U64_DEFINE_EARLY() [12/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_offload_active )

static

◆ COUNTER_U64_DEFINE_EARLY() [13/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_offload_corrupted_records )

static

◆ COUNTER_U64_DEFINE_EARLY() [14/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_offload_enable_calls )

static

◆ COUNTER_U64_DEFINE_EARLY() [15/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_offload_failed_crypto )

static

◆ COUNTER_U64_DEFINE_EARLY() [16/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_offload_total )

static

◆ COUNTER_U64_DEFINE_EARLY() [17/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_sw_cbc )

static

◆ COUNTER_U64_DEFINE_EARLY() [18/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_sw_chacha20 )

static

◆ COUNTER_U64_DEFINE_EARLY() [19/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_sw_gcm )

static

◆ COUNTER_U64_DEFINE_EARLY() [20/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_switch_failed )

static

◆ COUNTER_U64_DEFINE_EARLY() [21/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_switch_to_ifnet )

static

◆ COUNTER_U64_DEFINE_EARLY() [22/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_switch_to_sw )

static

◆ COUNTER_U64_DEFINE_EARLY() [23/23]

static COUNTER_U64_DEFINE_EARLY ( ktls_tasks_active )

static

◆ ktls_alloc_thread()

static void ktls_alloc_thread ( void * ctx )

static

Definition at line 2664 of file uipc_ktls.c.

References ktls_domain_info::alloc_td, ktls_alloc_thread::allocs, bootverbose, buf, domain, free(), ktls_bind_domain(), ktls_buffer_zone, ktls_domains, ktls_max_alloc, malloc(), name, printf(), ktls_alloc_thread::running, snprintf(), and ktls_alloc_thread::wakeups.

Here is the call graph for this function:

◆ ktls_batched_records()

static u_int ktls_batched_records ( struct mbuf * m )

static

Definition at line 2314 of file uipc_ktls.c.

Referenced by ktls_enqueue().

Here is the caller graph for this function:

◆ ktls_bind_domain()

static int ktls_bind_domain ( int domain )

static

Definition at line 2652 of file uipc_ktls.c.

References cpuset_domain, cpuset_setthread(), and domain.

Referenced by ktls_alloc_thread(), and ktls_work_thread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_buffer_alloc()

static void * ktls_buffer_alloc	(	struct ktls_wq *	wq,
		struct mbuf *	m
	)

static

Definition at line 2211 of file uipc_ktls.c.

References buf, domain, hz, ktls_buffer_zone, ktls_domains, ticks, and wakeup().

Referenced by ktls_encrypt_record().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_buffer_import()

static int ktls_buffer_import	(	void *	arg,
		void **	store,
		int	count,
		int	domain,
		int	flags
	)

static

Definition at line 341 of file uipc_ktls.c.

References count, domain, flags, and ktls_maxlen.

Referenced by ktls_init().

Here is the caller graph for this function:

◆ ktls_buffer_release()

static void ktls_buffer_release	(	void *arg	__unused,
		void **	store,
		int	count
	)

static

Definition at line 363 of file uipc_ktls.c.

References count, and ktls_maxlen.

Referenced by ktls_init().

Here is the caller graph for this function:

◆ ktls_check_rx()

void ktls_check_rx ( struct sockbuf * sb )

Definition at line 1807 of file uipc_ktls.c.

References m_copydata(), ktls_wq::mtx, and wakeup().

Referenced by socantrcvmore_locked().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_cleanup()

static void ktls_cleanup ( struct ktls_session * tls )

static

Definition at line 781 of file uipc_ktls.c.

References zfree().

Referenced by ktls_destroy().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_decrypt()

static void ktls_decrypt ( struct socket * so )

static

Definition at line 1996 of file uipc_ktls.c.

References data, ktls_detach_record(), m_copydata(), m_freem(), sbappendcontrol_locked(), sbcreatecontrol_how(), and tls13_find_record_type().

Referenced by ktls_work_thread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_destroy()

void ktls_destroy ( struct ktls_session * tls )

Definition at line 1628 of file uipc_ktls.c.

References ktls_cleanup(), and ktls_session_zone.

Here is the call graph for this function:

◆ ktls_detach_record()

static struct mbuf * ktls_detach_record	(	struct sockbuf *	sb,
		int	len
	)

static

Definition at line 1852 of file uipc_ktls.c.

References m_length(), and mb_dupcl().

Referenced by ktls_decrypt().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_encrypt()

static __noinline void ktls_encrypt	(	struct ktls_wq *	wq,
		struct mbuf *	top
	)

static

Definition at line 2463 of file uipc_ktls.c.

References ktls_encrypt_record(), ktls_finish_nonanon(), and mb_free_notready().

Referenced by ktls_work_thread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_encrypt_async()

static __noinline void ktls_encrypt_async	(	struct ktls_wq *	wq,
		struct mbuf *	top
	)

static

Definition at line 2592 of file uipc_ktls.c.

References free(), ktls_encrypt_record(), malloc(), and mb_free_notready().

Referenced by ktls_work_thread().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_encrypt_cb()

void ktls_encrypt_cb	(	struct ktls_ocf_encrypt_state *	state,
		int	error
	)

Definition at line 2543 of file uipc_ktls.c.

References free(), ktls_finish_nonanon(), and mb_free_notready().

Here is the call graph for this function:

◆ ktls_encrypt_record()

static int ktls_encrypt_record	(	struct ktls_wq *	wq,
		struct mbuf *	m,
		struct ktls_session *	tls,
		struct ktls_ocf_encrypt_state *	state
	)

static

Definition at line 2251 of file uipc_ktls.c.

References ktls_buffer_alloc(), ktls_buffer_zone, and ktls_maxlen.

Referenced by ktls_encrypt(), and ktls_encrypt_async().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_enqueue()

void ktls_enqueue	(	struct mbuf *	m,
		struct socket *	so,
		int	page_count
	)

Definition at line 2330 of file uipc_ktls.c.

References ktls_batched_records(), ktls_wq::mtx, and wakeup().

Referenced by sendfile_iodone(), sosend_generic(), and vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_enqueue_to_free()

void ktls_enqueue_to_free ( struct mbuf * m )

Definition at line 2194 of file uipc_ktls.c.

References ktls_wq::mtx, and wakeup().

Referenced by mb_free_extpg().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_finish_nonanon()

static void ktls_finish_nonanon	(	struct mbuf *	m,
		struct ktls_ocf_encrypt_state *	state
	)

static

Definition at line 2434 of file uipc_ktls.c.

References ktls_free_mext_contig(), and mb_free_mext_pgs().

Referenced by ktls_encrypt(), and ktls_encrypt_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_frame()

void ktls_frame	(	struct mbuf *	top,
		struct ktls_session *	tls,
		int *	enq_cnt,
		uint8_t	record_type
	)

Definition at line 1678 of file uipc_ktls.c.

References ktls_permit_empty_frames().

Referenced by sosend_generic(), and vn_sendfile().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_free_mext_contig()

static void ktls_free_mext_contig ( struct mbuf * m )

static

Definition at line 378 of file uipc_ktls.c.

References ktls_buffer_zone.

Referenced by ktls_finish_nonanon().

Here is the caller graph for this function:

◆ ktls_init()

static int ktls_init ( void )

static

Definition at line 385 of file uipc_ktls.c.

References bootverbose, count, ktls_domain_info::count, ktls_domain_info::cpu, cpuset_domain, domain, kproc_kthread_add(), ktls_bind_threads, ktls_buffer_import(), ktls_buffer_release(), ktls_buffer_zone, ktls_cpuid_lookup, ktls_domains, ktls_maxlen, ktls_number_threads, ktls_proc, ktls_session_zone, ktls_sw_buffer_cache, ktls_work_thread(), malloc(), mp_maxid, mtx, pcpu_find(), and printf().

Referenced by ktls_start_kthreads().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ ktls_permit_empty_frames()

bool ktls_permit_empty_frames ( struct ktls_session * tls )

Definition at line 1800 of file uipc_ktls.c.

Referenced by ktls_frame(), and sosend_generic().

Here is the caller graph for this function:

◆ ktls_seq()

void ktls_seq	(	struct sockbuf *	sb,
		struct mbuf *	m
	)

Definition at line 1650 of file uipc_ktls.c.

Referenced by sbappendstream_locked().

Here is the caller graph for this function:

◆ ktls_start_kthreads()

static int ktls_start_kthreads ( void )

static

Definition at line 476 of file uipc_ktls.c.

References ktls_init(), ktls_init_lock, ktls_init_state, and start.

Here is the call graph for this function:

◆ ktls_work_thread()

static void ktls_work_thread ( void * ctx )

static

Definition at line 2728 of file uipc_ktls.c.

References bootverbose, cpuset_setthread(), ktls_bind_domain(), ktls_bind_threads, ktls_decrypt(), ktls_encrypt(), ktls_encrypt_async(), ktls_wq, m_free_raw(), mask, ktls_wq::mtx, pcpu_find(), printf(), and STAILQ_HEAD().

Referenced by ktls_init().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ MALLOC_DEFINE()

static MALLOC_DEFINE	(	M_KTLS	,
		"ktls"	,
		"Kernel TLS"
	)

static

◆ SX_SYSINIT()

SX_SYSINIT	(	ktls_init_lock	,
		&	ktls_init_lock,
		"ktls init"
	)

◆ SYSCTL_BOOL() [1/3]

SYSCTL_BOOL	(	_kern_ipc_tls	,
		OID_AUTO	,
		cbc_enable	,
		CTLFLAG_RWTUN	,
		&	ktls_cbc_enable,
		1	,
		"Enable Support of AES-CBC crypto for kernel TLS"
	)

◆ SYSCTL_BOOL() [2/3]

SYSCTL_BOOL	(	_kern_ipc_tls	,
		OID_AUTO	,
		enable	,
		CTLFLAG_RWTUN	,
		&	ktls_offload_enable,
		0	,
		"Enable support for kernel TLS offload"
	)

◆ SYSCTL_BOOL() [3/3]

SYSCTL_BOOL	(	_kern_ipc_tls	,
		OID_AUTO	,
		sw_buffer_cache	,
		CTLFLAG_RDTUN	,
		&	ktls_sw_buffer_cache,
		1	,
		"Enable caching of output buffers for SW encryption"
	)

◆ SYSCTL_COUNTER_U64() [1/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls	,
		OID_AUTO	,
		tasks_active	,
		CTLFLAG_RD	,
		&	ktls_tasks_active,
		"Number of active tasks"
	)

◆ SYSCTL_COUNTER_U64() [2/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		cbc	,
		CTLFLAG_RD	,
		&	ktls_ifnet_cbc,
		"Active number of ifnet TLS sessions using AES-CBC"
	)

◆ SYSCTL_COUNTER_U64() [3/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		chacha20	,
		CTLFLAG_RD	,
		&	ktls_ifnet_chacha20,
		"Active number of ifnet TLS sessions using Chacha20-Poly1305"
	)

◆ SYSCTL_COUNTER_U64() [4/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		gcm	,
		CTLFLAG_RD	,
		&	ktls_ifnet_gcm,
		"Active number of ifnet TLS sessions using AES-GCM"
	)

◆ SYSCTL_COUNTER_U64() [5/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		reset	,
		CTLFLAG_RD	,
		&	ktls_ifnet_reset,
		"TLS sessions updated to a new ifnet send tag"
	)

◆ SYSCTL_COUNTER_U64() [6/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		reset_dropped	,
		CTLFLAG_RD	,
		&	ktls_ifnet_reset_dropped,
		"TLS sessions dropped after failing to update ifnet send tag"
	)

◆ SYSCTL_COUNTER_U64() [7/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		reset_failed	,
		CTLFLAG_RD	,
		&	ktls_ifnet_reset_failed,
		"TLS sessions that failed to allocate a new ifnet send tag"
	)

◆ SYSCTL_COUNTER_U64() [8/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		active	,
		CTLFLAG_RD	,
		&	ktls_offload_active,
		"Total Active TLS sessions"
	)

◆ SYSCTL_COUNTER_U64() [9/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		corrupted_records	,
		CTLFLAG_RD	,
		&	ktls_offload_corrupted_records,
		"Total corrupted TLS records received"
	)

◆ SYSCTL_COUNTER_U64() [10/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		enable_calls	,
		CTLFLAG_RD	,
		&	ktls_offload_enable_calls,
		"Total number of TLS enable calls made"
	)

◆ SYSCTL_COUNTER_U64() [11/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		failed_crypto	,
		CTLFLAG_RD	,
		&	ktls_offload_failed_crypto,
		"Total TLS crypto failures"
	)

◆ SYSCTL_COUNTER_U64() [12/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		ifnet_disable_failed	,
		CTLFLAG_RD	,
		&	ktls_ifnet_disable_fail,
		"TLS sessions unable to switch to SW from ifnet"
	)

◆ SYSCTL_COUNTER_U64() [13/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		ifnet_disable_ok	,
		CTLFLAG_RD	,
		&	ktls_ifnet_disable_ok,
		"TLS sessions able to switch to SW from ifnet"
	)

◆ SYSCTL_COUNTER_U64() [14/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		offload_total	,
		CTLFLAG_RD	,
		&	ktls_offload_total,
		"Total successful TLS setups (parameters set)"
	)

◆ SYSCTL_COUNTER_U64() [15/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		sw_rx_inqueue	,
		CTLFLAG_RD	,
		&	ktls_cnt_rx_queued,
		"Number of TLS sockets in queue to tasks for SW decryption"
	)

◆ SYSCTL_COUNTER_U64() [16/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		sw_tx_inqueue	,
		CTLFLAG_RD	,
		&	ktls_cnt_tx_queued,
		"Number of TLS records in queue to tasks for SW encryption"
	)

◆ SYSCTL_COUNTER_U64() [17/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		sw_tx_pending	,
		CTLFLAG_RD	,
		&	ktls_cnt_tx_pending,
		"Number of TLS 1.0 records waiting for earlier TLS records"
	)

◆ SYSCTL_COUNTER_U64() [18/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		switch_failed	,
		CTLFLAG_RD	,
		&	ktls_switch_failed,
		"TLS sessions unable to switch between SW and ifnet"
	)

◆ SYSCTL_COUNTER_U64() [19/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		switch_to_ifnet	,
		CTLFLAG_RD	,
		&	ktls_switch_to_ifnet,
		"TLS sessions switched from SW to ifnet"
	)

◆ SYSCTL_COUNTER_U64() [20/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		switch_to_sw	,
		CTLFLAG_RD	,
		&	ktls_switch_to_sw,
		"TLS sessions switched from ifnet to SW"
	)

◆ SYSCTL_COUNTER_U64() [21/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_sw	,
		OID_AUTO	,
		cbc	,
		CTLFLAG_RD	,
		&	ktls_sw_cbc,
		"Active number of software TLS sessions using AES-CBC"
	)

◆ SYSCTL_COUNTER_U64() [22/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_sw	,
		OID_AUTO	,
		chacha20	,
		CTLFLAG_RD	,
		&	ktls_sw_chacha20,
		"Active number of software TLS sessions using Chacha20-Poly1305"
	)

◆ SYSCTL_COUNTER_U64() [23/23]

SYSCTL_COUNTER_U64	(	_kern_ipc_tls_sw	,
		OID_AUTO	,
		gcm	,
		CTLFLAG_RD	,
		&	ktls_sw_gcm,
		"Active number of software TLS sessions using AES-GCM"
	)

◆ SYSCTL_INT() [1/3]

SYSCTL_INT	(	_kern_ipc_tls	,
		OID_AUTO	,
		bind_threads	,
		CTLFLAG_RDTUN	,
		&	ktls_bind_threads,
		0	,
		"Bind crypto threads to cores (1) or cores and domains (2) at boot"
	)

◆ SYSCTL_INT() [2/3]

SYSCTL_INT	(	_kern_ipc_tls	,
		OID_AUTO	,
		max_alloc	,
		CTLFLAG_RWTUN	,
		&	ktls_max_alloc,
		128	,
		"Max number of 16k buffers to allocate in thread context"
	)

◆ SYSCTL_INT() [3/3]

SYSCTL_INT	(	_kern_ipc_tls_stats	,
		OID_AUTO	,
		threads	,
		CTLFLAG_RD	,
		&	ktls_number_threads,
		0	,
		"Number of TLS threads in thread-pool"
	)

◆ SYSCTL_NODE() [1/4]

SYSCTL_NODE	(	_kern_ipc	,
		OID_AUTO	,
		tls	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"Kernel TLS offload"
	)

◆ SYSCTL_NODE() [2/4]

SYSCTL_NODE	(	_kern_ipc_tls	,
		OID_AUTO	,
		ifnet	,
		CTLFLAG_RD\|	CTLFLAG_MPSAFE,
		0	,
		"Hardware (ifnet) TLS session stats"
	)

◆ SYSCTL_NODE() [3/4]

SYSCTL_NODE	(	_kern_ipc_tls	,
		OID_AUTO	,
		stats	,
		CTLFLAG_RW\|	CTLFLAG_MPSAFE,
		0	,
		"Kernel TLS offload stats"
	)

◆ SYSCTL_NODE() [4/4]

SYSCTL_NODE	(	_kern_ipc_tls	,
		OID_AUTO	,
		sw	,
		CTLFLAG_RD\|	CTLFLAG_MPSAFE,
		0	,
		"Software TLS session stats"
	)

◆ SYSCTL_UINT() [1/3]

SYSCTL_UINT	(	_kern_ipc_tls	,
		OID_AUTO	,
		ifnet_max_rexmit_pct	,
		CTLFLAG_RWTUN	,
		&	ktls_ifnet_max_rexmit_pct,
		2	,
		"Max percent bytes retransmitted before ifnet TLS is disabled"
	)

◆ SYSCTL_UINT() [2/3]

SYSCTL_UINT	(	_kern_ipc_tls	,
		OID_AUTO	,
		maxlen	,
		CTLFLAG_RDTUN	,
		&	ktls_maxlen,
		0	,
		"Maximum TLS record size"
	)

◆ SYSCTL_UINT() [3/3]

SYSCTL_UINT	(	_kern_ipc_tls_ifnet	,
		OID_AUTO	,
		permitted	,
		CTLFLAG_RWTUN	,
		&	ktls_ifnet_permitted,
		1	,
		"Whether to permit hardware (ifnet) TLS sessions"
	)

◆ tls13_find_record_type()

static int tls13_find_record_type	(	struct ktls_session *	tls,
		struct mbuf *	m,
		int	tls_len,
		int *	trailer_len,
		uint8_t *	record_typep
	)

static

Definition at line 1963 of file uipc_ktls.c.

Referenced by ktls_decrypt().

Here is the caller graph for this function:

Variable Documentation

◆ __aligned

struct ktls_alloc_thread __aligned

◆ ktls_bind_threads

int ktls_bind_threads

static

Definition at line 124 of file uipc_ktls.c.

Referenced by ktls_init(), and ktls_work_thread().

◆ ktls_buffer_zone

uma_zone_t ktls_buffer_zone

static

Definition at line 110 of file uipc_ktls.c.

Referenced by ktls_alloc_thread(), ktls_buffer_alloc(), ktls_encrypt_record(), ktls_free_mext_contig(), and ktls_init().

◆ ktls_cbc_enable

bool ktls_cbc_enable = true

static

Definition at line 149 of file uipc_ktls.c.

◆ ktls_cpuid_lookup

uint16_t ktls_cpuid_lookup[MAXCPU]

static

Definition at line 111 of file uipc_ktls.c.

Referenced by ktls_init().

◆ ktls_domains

struct ktls_domain_info ktls_domains[MAXMEMDOM]

Definition at line 106 of file uipc_ktls.c.

Referenced by ktls_alloc_thread(), ktls_buffer_alloc(), and ktls_init().

◆ ktls_ifnet_max_rexmit_pct

unsigned int ktls_ifnet_max_rexmit_pct = 2

Definition at line 139 of file uipc_ktls.c.

◆ ktls_ifnet_permitted

int ktls_ifnet_permitted

static

Definition at line 276 of file uipc_ktls.c.

◆ ktls_init_lock

struct sx ktls_init_lock

static

Definition at line 113 of file uipc_ktls.c.

Referenced by ktls_start_kthreads().

◆ ktls_init_state

int ktls_init_state

static

Definition at line 112 of file uipc_ktls.c.

Referenced by ktls_start_kthreads().

◆ ktls_max_alloc

int ktls_max_alloc = 128

static

Definition at line 159 of file uipc_ktls.c.

Referenced by ktls_alloc_thread().

◆ ktls_maxlen

u_int ktls_maxlen = 16384

static

Definition at line 130 of file uipc_ktls.c.

Referenced by ktls_buffer_import(), ktls_buffer_release(), ktls_encrypt_record(), and ktls_init().

◆ ktls_number_threads

int ktls_number_threads

static

Definition at line 134 of file uipc_ktls.c.

Referenced by ktls_init().

◆ ktls_offload_enable

bool ktls_offload_enable

static

Definition at line 144 of file uipc_ktls.c.

◆ ktls_proc

struct proc* ktls_proc

static

Definition at line 108 of file uipc_ktls.c.

Referenced by ktls_init().

◆ ktls_session_zone

uma_zone_t ktls_session_zone

static

Definition at line 109 of file uipc_ktls.c.

Referenced by ktls_destroy(), and ktls_init().

◆ ktls_sw_buffer_cache

bool ktls_sw_buffer_cache = true

static

Definition at line 154 of file uipc_ktls.c.

Referenced by ktls_init().

◆ ktls_wq

struct ktls_wq* ktls_wq

static

Definition at line 107 of file uipc_ktls.c.

Referenced by ktls_work_thread().

◆ mtx

struct mtx mtx

Definition at line 0 of file uipc_ktls.c.

Referenced by __mtx_lock_flags(), __mtx_lock_sleep(), __mtx_lock_spin_flags(), __mtx_trylock_spin_flags(), __mtx_unlock_flags(), __mtx_unlock_sleep(), __mtx_unlock_spin_flags(), _mtx_destroy(), _mtx_init(), _mtx_trylock_flags_(), _thread_lock(), assert_mtx(), bdone(), biodone(), biowait(), busdma_lock_mutex(), bwait(), cache_assert_vnode_locked(), cache_enter_lock(), cache_enter_lock_dd(), cache_lock_vnodes_cel(), cache_lock_vnodes_cel_3(), cache_lookup_dotdot(), cache_lookup_fallback(), cache_neg_evict(), cache_purge_impl(), cache_purge_negative(), cache_purge_vgone(), cache_remove_cnp(), cache_zap_locked_bucket(), cache_zap_locked_vnode_kl2(), cache_zap_negative_locked_vnode_kl(), get_advice(), kern_semctl(), knlist_mtx_assert_lock(), knlist_mtx_lock(), knlist_mtx_unlock(), ktls_init(), lock_mtx(), lock_spin(), msleep_spin_sbt(), mtx_pool_create(), mtx_sysinit(), sched_switch(), selrecord(), semexit_myhook(), seminit(), softclock_thread(), start_softclock(), sys_semop(), thread_lock_block(), thread_lock_flags_(), thread_lock_set(), thread_unblock_switch(), TQ_SLEEP(), uipc_close(), uipc_detach(), unlock_mtx(), unlock_spin(), unp_connectat(), vfs_unp_reclaim(), vn_commname(), vn_dir_dd_ino(), vn_vptocnp(), vop_lock(), and vop_stdlock().

Data Structures

Functions

Variables

Function Documentation

◆ __aligned()

◆ __FBSDID()

◆ COUNTER_U64_DEFINE_EARLY() [1/23]

◆ COUNTER_U64_DEFINE_EARLY() [2/23]

◆ COUNTER_U64_DEFINE_EARLY() [3/23]

◆ COUNTER_U64_DEFINE_EARLY() [4/23]

◆ COUNTER_U64_DEFINE_EARLY() [5/23]

◆ COUNTER_U64_DEFINE_EARLY() [6/23]

◆ COUNTER_U64_DEFINE_EARLY() [7/23]

◆ COUNTER_U64_DEFINE_EARLY() [8/23]

◆ COUNTER_U64_DEFINE_EARLY() [9/23]

◆ COUNTER_U64_DEFINE_EARLY() [10/23]

◆ COUNTER_U64_DEFINE_EARLY() [11/23]

◆ COUNTER_U64_DEFINE_EARLY() [12/23]

◆ COUNTER_U64_DEFINE_EARLY() [13/23]

◆ COUNTER_U64_DEFINE_EARLY() [14/23]

◆ COUNTER_U64_DEFINE_EARLY() [15/23]

◆ COUNTER_U64_DEFINE_EARLY() [16/23]

◆ COUNTER_U64_DEFINE_EARLY() [17/23]

◆ COUNTER_U64_DEFINE_EARLY() [18/23]

◆ COUNTER_U64_DEFINE_EARLY() [19/23]

◆ COUNTER_U64_DEFINE_EARLY() [20/23]

◆ COUNTER_U64_DEFINE_EARLY() [21/23]

◆ COUNTER_U64_DEFINE_EARLY() [22/23]

◆ COUNTER_U64_DEFINE_EARLY() [23/23]

◆ ktls_alloc_thread()

◆ ktls_batched_records()

◆ ktls_bind_domain()

◆ ktls_buffer_alloc()

◆ ktls_buffer_import()

◆ ktls_buffer_release()

◆ ktls_check_rx()

◆ ktls_cleanup()

◆ ktls_decrypt()

◆ ktls_destroy()

◆ ktls_detach_record()

◆ ktls_encrypt()

◆ ktls_encrypt_async()

◆ ktls_encrypt_cb()

◆ ktls_encrypt_record()

◆ ktls_enqueue()

◆ ktls_enqueue_to_free()

◆ ktls_finish_nonanon()

◆ ktls_frame()

◆ ktls_free_mext_contig()

◆ ktls_init()

◆ ktls_permit_empty_frames()

◆ ktls_seq()

◆ ktls_start_kthreads()

◆ ktls_work_thread()

◆ MALLOC_DEFINE()

◆ SX_SYSINIT()

◆ SYSCTL_BOOL() [1/3]

◆ SYSCTL_BOOL() [2/3]

◆ SYSCTL_BOOL() [3/3]

◆ SYSCTL_COUNTER_U64() [1/23]

◆ SYSCTL_COUNTER_U64() [2/23]

◆ SYSCTL_COUNTER_U64() [3/23]

◆ SYSCTL_COUNTER_U64() [4/23]

◆ SYSCTL_COUNTER_U64() [5/23]

◆ SYSCTL_COUNTER_U64() [6/23]

◆ SYSCTL_COUNTER_U64() [7/23]

◆ SYSCTL_COUNTER_U64() [8/23]

◆ SYSCTL_COUNTER_U64() [9/23]

◆ SYSCTL_COUNTER_U64() [10/23]

◆ SYSCTL_COUNTER_U64() [11/23]

◆ SYSCTL_COUNTER_U64() [12/23]

◆ SYSCTL_COUNTER_U64() [13/23]

◆ SYSCTL_COUNTER_U64() [14/23]

◆ SYSCTL_COUNTER_U64() [15/23]

◆ SYSCTL_COUNTER_U64() [16/23]

◆ SYSCTL_COUNTER_U64() [17/23]

◆ SYSCTL_COUNTER_U64() [18/23]

◆ SYSCTL_COUNTER_U64() [19/23]

◆ SYSCTL_COUNTER_U64() [20/23]

◆ SYSCTL_COUNTER_U64() [21/23]