33#include "opt_kern_tls.h"
34#include "opt_ratelimit.h"
38#include <sys/kernel.h>
39#include <sys/domainset.h>
40#include <sys/endian.h>
45#include <sys/rmlock.h>
47#include <sys/protosw.h>
48#include <sys/refcount.h>
50#include <sys/socket.h>
51#include <sys/socketvar.h>
52#include <sys/sysctl.h>
53#include <sys/taskqueue.h>
54#include <sys/kthread.h>
56#include <sys/vmmeter.h>
57#if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
58#include <machine/pcb.h>
60#include <machine/vmparam.h>
62#include <net/if_var.h>
64#include <net/netisr.h>
65#include <net/rss_config.h>
68#include <net/route/nhop.h>
69#if defined(INET) || defined(INET6)
70#include <netinet/in.h>
71#include <netinet/in_pcb.h>
73#include <netinet/tcp_var.h>
75#include <netinet/tcp_offload.h>
77#include <opencrypto/cryptodev.h>
78#include <opencrypto/ktls.h>
79#include <vm/uma_dbg.h>
81#include <vm/vm_pageout.h>
82#include <vm/vm_page.h>
83#include <vm/vm_pagequeue.h>
116SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
117 "Kernel TLS offload");
118SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
119 "Kernel TLS offload stats");
126SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN,
128 "Bind crypto threads to cores (1) or cores and domains (2) at boot");
135SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD,
137 "Number of TLS threads in thread-pool");
140SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, ifnet_max_rexmit_pct, CTLFLAG_RWTUN,
142 "Max percent bytes retransmitted before ifnet TLS is disabled");
147 "Enable support for kernel TLS offload");
152 "Enable Support of AES-CBC crypto for kernel TLS");
155SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, sw_buffer_cache, CTLFLAG_RDTUN,
157 "Enable caching of output buffers for SW encryption");
162 "Max number of 16k buffers to allocate in thread context");
166 &ktls_tasks_active,
"Number of active tasks");
170 &ktls_cnt_tx_pending,
171 "Number of TLS 1.0 records waiting for earlier TLS records");
176 "Number of TLS records in queue to tasks for SW encryption");
181 "Number of TLS sockets in queue to tasks for SW decryption");
185 CTLFLAG_RD, &ktls_offload_total,
186 "Total successful TLS setups (parameters set)");
190 CTLFLAG_RD, &ktls_offload_enable_calls,
191 "Total number of TLS enable calls made");
195 &ktls_offload_active,
"Total Active TLS sessions");
199 &ktls_offload_corrupted_records,
"Total corrupted TLS records received");
203 &ktls_offload_failed_crypto,
"Total TLS crypto failures");
207 &ktls_switch_to_ifnet,
"TLS sessions switched from SW to ifnet");
211 &ktls_switch_to_sw,
"TLS sessions switched from ifnet to SW");
215 &ktls_switch_failed,
"TLS sessions unable to switch between SW and ifnet");
219 &ktls_ifnet_disable_fail,
"TLS sessions unable to switch to SW from ifnet");
223 &ktls_ifnet_disable_ok,
"TLS sessions able to switch to SW from ifnet");
225SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
226 "Software TLS session stats");
227SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
228 "Hardware (ifnet) TLS session stats");
230SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
231 "TOE TLS session stats");
236 "Active number of software TLS sessions using AES-CBC");
240 "Active number of software TLS sessions using AES-GCM");
245 "Active number of software TLS sessions using Chacha20-Poly1305");
250 "Active number of ifnet TLS sessions using AES-CBC");
255 "Active number of ifnet TLS sessions using AES-GCM");
259 &ktls_ifnet_chacha20,
260 "Active number of ifnet TLS sessions using Chacha20-Poly1305");
264 &ktls_ifnet_reset,
"TLS sessions updated to a new ifnet send tag");
268 &ktls_ifnet_reset_dropped,
269 "TLS sessions dropped after failing to update ifnet send tag");
273 &ktls_ifnet_reset_failed,
274 "TLS sessions that failed to allocate a new ifnet send tag");
277SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN,
279 "Whether to permit hardware (ifnet) TLS sessions");
285 "Active number of TOE TLS sessions using AES-CBC");
290 "Active number of TOE TLS sessions using AES-GCM");
295 "Active number of TOE TLS sessions using Chacha20-Poly1305");
301#if defined(INET) || defined(INET6)
302static void ktls_reset_send_tag(
void *context,
int pending);
307#if defined(INET) || defined(INET6)
309ktls_get_cpu(
struct socket *so)
319 cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
320 if (cpuid != NETISR_CPUID_NONE)
332 cpuid = di->
cpu[inp->inp_flowid % di->
count];
347 (
"%s: ktls max length %d is not page size-aligned",
350 req = VM_ALLOC_WIRED | VM_ALLOC_NODUMP | malloc2vm_flags(
flags);
351 for (i = 0; i <
count; i++) {
352 m = vm_page_alloc_noobj_contig_domain(
domain, req,
357 store[i] = (
void *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m));
368 for (i = 0; i <
count; i++) {
369 m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)store[i]));
371 (void)vm_page_unwire_noq(m + j);
395 sizeof(
struct ktls_session),
396 NULL, NULL, NULL, NULL,
401 roundup2(
ktls_maxlen, PAGE_SIZE), NULL, NULL, NULL, NULL,
403 UMA_ZONE_FIRSTTOUCH);
411 STAILQ_INIT(&
ktls_wq[i].m_head);
412 STAILQ_INIT(&
ktls_wq[i].so_head);
413 mtx_init(&
ktls_wq[i].
mtx,
"ktls work queue", NULL, MTX_DEF);
430 for (i = 0; i < vm_ndomains; i++) {
441 &
ktls_proc, &td, 0, 0,
"KTLS",
"thr_%d", i);
443 printf(
"Can't add KTLS thread %d error %d\n", i, error);
454 if (VM_DOMAIN_EMPTY(
domain))
461 0, 0,
"KTLS",
"alloc_%d",
domain);
463 printf(
"Can't add KTLS alloc thread %d error %d\n",
482 if (__predict_true(state > 0))
503#if defined(INET) || defined(INET6)
505ktls_create_session(
struct socket *so,
struct tls_enable *en,
506 struct ktls_session **tlsp)
508 struct ktls_session *tls;
512 if (en->tls_vmajor != TLS_MAJOR_VER_ONE)
514 if (en->tls_vminor < TLS_MINOR_VER_ZERO ||
515 en->tls_vminor > TLS_MINOR_VER_THREE)
518 if (en->auth_key_len < 0 || en->auth_key_len > TLS_MAX_PARAM_SIZE)
520 if (en->cipher_key_len < 0 || en->cipher_key_len > TLS_MAX_PARAM_SIZE)
522 if (en->iv_len < 0 || en->iv_len >
sizeof(tls->params.iv))
526 if (en->cipher_key_len == 0)
534 switch (en->cipher_algorithm) {
535 case CRYPTO_AES_NIST_GCM_16:
540 switch (en->auth_algorithm) {
542#ifdef COMPAT_FREEBSD12
544 case CRYPTO_AES_128_NIST_GMAC:
545 case CRYPTO_AES_192_NIST_GMAC:
546 case CRYPTO_AES_256_NIST_GMAC:
552 if (en->auth_key_len != 0)
554 switch (en->tls_vminor) {
555 case TLS_MINOR_VER_TWO:
556 if (en->iv_len != TLS_AEAD_GCM_LEN)
559 case TLS_MINOR_VER_THREE:
560 if (en->iv_len != TLS_1_3_GCM_IV_LEN)
568 switch (en->auth_algorithm) {
569 case CRYPTO_SHA1_HMAC:
571 case CRYPTO_SHA2_256_HMAC:
572 case CRYPTO_SHA2_384_HMAC:
573 if (en->tls_vminor != TLS_MINOR_VER_TWO)
579 if (en->auth_key_len == 0)
586 switch (en->tls_vminor) {
587 case TLS_MINOR_VER_ZERO:
588 if (en->iv_len != TLS_CBC_IMPLICIT_IV_LEN)
591 case TLS_MINOR_VER_ONE:
592 case TLS_MINOR_VER_TWO:
600 case CRYPTO_CHACHA20_POLY1305:
601 if (en->auth_algorithm != 0 || en->auth_key_len != 0)
603 if (en->tls_vminor != TLS_MINOR_VER_TWO &&
604 en->tls_vminor != TLS_MINOR_VER_THREE)
606 if (en->iv_len != TLS_CHACHA20_IV_LEN)
619 counter_u64_add(ktls_offload_active, 1);
621 refcount_init(&tls->refcount, 1);
622 TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
624 tls->wq_index = ktls_get_cpu(so);
626 tls->params.cipher_algorithm = en->cipher_algorithm;
627 tls->params.auth_algorithm = en->auth_algorithm;
628 tls->params.tls_vmajor = en->tls_vmajor;
629 tls->params.tls_vminor = en->tls_vminor;
630 tls->params.flags = en->flags;
631 tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2,
ktls_maxlen);
634 tls->params.tls_hlen =
sizeof(
struct tls_record_layer);
635 switch (en->cipher_algorithm) {
636 case CRYPTO_AES_NIST_GCM_16:
641 if (en->tls_vminor < TLS_MINOR_VER_THREE)
642 tls->params.tls_hlen +=
sizeof(uint64_t);
643 tls->params.tls_tlen = AES_GMAC_HASH_LEN;
644 tls->params.tls_bs = 1;
647 switch (en->auth_algorithm) {
648 case CRYPTO_SHA1_HMAC:
649 if (en->tls_vminor == TLS_MINOR_VER_ZERO) {
651 tls->sequential_records =
true;
652 tls->next_seqno = be64dec(en->rec_seq);
653 STAILQ_INIT(&tls->pending_records);
655 tls->params.tls_hlen += AES_BLOCK_LEN;
657 tls->params.tls_tlen = AES_BLOCK_LEN +
660 case CRYPTO_SHA2_256_HMAC:
661 tls->params.tls_hlen += AES_BLOCK_LEN;
662 tls->params.tls_tlen = AES_BLOCK_LEN +
665 case CRYPTO_SHA2_384_HMAC:
666 tls->params.tls_hlen += AES_BLOCK_LEN;
667 tls->params.tls_tlen = AES_BLOCK_LEN +
671 panic(
"invalid hmac");
673 tls->params.tls_bs = AES_BLOCK_LEN;
675 case CRYPTO_CHACHA20_POLY1305:
679 tls->params.tls_tlen = POLY1305_HASH_LEN;
680 tls->params.tls_bs = 1;
683 panic(
"invalid cipher");
691 if (en->tls_vminor == TLS_MINOR_VER_THREE)
692 tls->params.tls_tlen +=
sizeof(uint8_t);
694 KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN,
695 (
"TLS header length too long: %d", tls->params.tls_hlen));
696 KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN,
697 (
"TLS trailer length too long: %d", tls->params.tls_tlen));
699 if (en->auth_key_len != 0) {
700 tls->params.auth_key_len = en->auth_key_len;
701 tls->params.auth_key =
malloc(en->auth_key_len, M_KTLS,
703 error = copyin(en->auth_key, tls->params.auth_key,
709 tls->params.cipher_key_len = en->cipher_key_len;
710 tls->params.cipher_key =
malloc(en->cipher_key_len, M_KTLS, M_WAITOK);
711 error = copyin(en->cipher_key, tls->params.cipher_key,
721 if (en->iv_len != 0) {
722 tls->params.iv_len = en->iv_len;
723 error = copyin(en->iv, tls->params.iv, en->iv_len);
734 if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&
735 en->tls_vminor == TLS_MINOR_VER_TWO)
736 arc4rand(tls->params.iv + 8,
sizeof(uint64_t), 0);
747static struct ktls_session *
748ktls_clone_session(
struct ktls_session *tls)
750 struct ktls_session *tls_new;
754 counter_u64_add(ktls_offload_active, 1);
756 refcount_init(&tls_new->refcount, 1);
757 TASK_INIT(&tls_new->reset_tag_task, 0, ktls_reset_send_tag, tls_new);
760 tls_new->params = tls->params;
761 tls_new->wq_index = tls->wq_index;
764 if (tls_new->params.auth_key != NULL) {
765 tls_new->params.auth_key =
malloc(tls->params.auth_key_len,
767 memcpy(tls_new->params.auth_key, tls->params.auth_key,
768 tls->params.auth_key_len);
771 tls_new->params.cipher_key =
malloc(tls->params.cipher_key_len, M_KTLS,
773 memcpy(tls_new->params.cipher_key, tls->params.cipher_key,
774 tls->params.cipher_key_len);
784 counter_u64_add(ktls_offload_active, -1);
786 case TCP_TLS_MODE_SW:
787 switch (tls->params.cipher_algorithm) {
789 counter_u64_add(ktls_sw_cbc, -1);
791 case CRYPTO_AES_NIST_GCM_16:
792 counter_u64_add(ktls_sw_gcm, -1);
794 case CRYPTO_CHACHA20_POLY1305:
795 counter_u64_add(ktls_sw_chacha20, -1);
799 case TCP_TLS_MODE_IFNET:
800 switch (tls->params.cipher_algorithm) {
802 counter_u64_add(ktls_ifnet_cbc, -1);
804 case CRYPTO_AES_NIST_GCM_16:
805 counter_u64_add(ktls_ifnet_gcm, -1);
807 case CRYPTO_CHACHA20_POLY1305:
808 counter_u64_add(ktls_ifnet_chacha20, -1);
811 if (tls->snd_tag != NULL)
812 m_snd_tag_rele(tls->snd_tag);
815 case TCP_TLS_MODE_TOE:
816 switch (tls->params.cipher_algorithm) {
818 counter_u64_add(ktls_toe_cbc, -1);
820 case CRYPTO_AES_NIST_GCM_16:
821 counter_u64_add(ktls_toe_gcm, -1);
823 case CRYPTO_CHACHA20_POLY1305:
824 counter_u64_add(ktls_toe_chacha20, -1);
830 if (tls->ocf_session != NULL)
832 if (tls->params.auth_key != NULL) {
833 zfree(tls->params.auth_key, M_KTLS);
834 tls->params.auth_key = NULL;
835 tls->params.auth_key_len = 0;
837 if (tls->params.cipher_key != NULL) {
838 zfree(tls->params.cipher_key, M_KTLS);
839 tls->params.cipher_key = NULL;
840 tls->params.cipher_key_len = 0;
842 explicit_bzero(tls->params.iv,
sizeof(tls->params.iv));
845#if defined(INET) || defined(INET6)
849ktls_try_toe(
struct socket *so,
struct ktls_session *tls,
int direction)
857 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
861 if (inp->inp_socket == NULL) {
866 if (!(tp->t_flags & TF_TOE)) {
871 error = tcp_offload_alloc_tls_session(tp, tls, direction);
874 tls->mode = TCP_TLS_MODE_TOE;
875 switch (tls->params.cipher_algorithm) {
877 counter_u64_add(ktls_toe_cbc, 1);
879 case CRYPTO_AES_NIST_GCM_16:
880 counter_u64_add(ktls_toe_gcm, 1);
882 case CRYPTO_CHACHA20_POLY1305:
883 counter_u64_add(ktls_toe_chacha20, 1);
898ktls_alloc_snd_tag(
struct inpcb *inp,
struct ktls_session *tls,
bool force,
899 struct m_snd_tag **mstp)
901 union if_snd_tag_alloc_params params;
903 struct nhop_object *nh;
908 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
912 if (inp->inp_socket == NULL) {
937 nh = inp->inp_route.ro_nh;
949 if (tp->t_pacing_rate != -1 &&
950 (ifp->if_capenable & IFCAP_TXTLS_RTLMT) != 0) {
951 params.hdr.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT;
952 params.tls_rate_limit.inp = inp;
953 params.tls_rate_limit.tls = tls;
954 params.tls_rate_limit.max_rate = tp->t_pacing_rate;
956 params.hdr.type = IF_SND_TAG_TYPE_TLS;
957 params.tls.inp = inp;
958 params.tls.tls = tls;
960 params.hdr.flowid = inp->inp_flowid;
961 params.hdr.flowtype = inp->inp_flowtype;
962 params.hdr.numa_domain = inp->inp_numa_domain;
965 if ((ifp->if_capenable & IFCAP_MEXTPG) == 0) {
969 if (inp->inp_vflag & INP_IPV6) {
970 if ((ifp->if_capenable & IFCAP_TXTLS6) == 0) {
975 if ((ifp->if_capenable & IFCAP_TXTLS4) == 0) {
987ktls_try_ifnet(
struct socket *so,
struct ktls_session *tls,
bool force)
989 struct m_snd_tag *mst;
992 error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
994 tls->mode = TCP_TLS_MODE_IFNET;
996 switch (tls->params.cipher_algorithm) {
998 counter_u64_add(ktls_ifnet_cbc, 1);
1000 case CRYPTO_AES_NIST_GCM_16:
1001 counter_u64_add(ktls_ifnet_gcm, 1);
1003 case CRYPTO_CHACHA20_POLY1305:
1004 counter_u64_add(ktls_ifnet_chacha20, 1);
1012ktls_use_sw(
struct ktls_session *tls)
1014 tls->mode = TCP_TLS_MODE_SW;
1015 switch (tls->params.cipher_algorithm) {
1016 case CRYPTO_AES_CBC:
1017 counter_u64_add(ktls_sw_cbc, 1);
1019 case CRYPTO_AES_NIST_GCM_16:
1020 counter_u64_add(ktls_sw_gcm, 1);
1022 case CRYPTO_CHACHA20_POLY1305:
1023 counter_u64_add(ktls_sw_chacha20, 1);
1029ktls_try_sw(
struct socket *so,
struct ktls_session *tls,
int direction)
1033 error = ktls_ocf_try(so, tls, direction);
1069sb_mark_notready(
struct sockbuf *sb)
1076 sb->sb_mbtail = NULL;
1077 sb->sb_lastrecord = NULL;
1078 for (; m != NULL; m = m->m_next) {
1079 KASSERT(m->m_nextpkt == NULL, (
"%s: m_nextpkt != NULL",
1081 KASSERT((m->m_flags & M_NOTAVAIL) == 0, (
"%s: mbuf not avail",
1083 KASSERT(sb->sb_acc >= m->m_len, (
"%s: sb_acc < m->m_len",
1085 m->m_flags |= M_NOTREADY;
1086 sb->sb_acc -= m->m_len;
1087 sb->sb_tlscc += m->m_len;
1088 sb->sb_mtlstail = m;
1090 KASSERT(sb->sb_acc == 0 && sb->sb_tlscc == sb->sb_ccc,
1091 (
"%s: acc %u tlscc %u ccc %u", __func__, sb->sb_acc, sb->sb_tlscc,
1105ktls_pending_rx_info(
struct sockbuf *sb, uint64_t *seqnop,
size_t *residp)
1107 struct tls_record_layer hdr;
1111 u_int offset, record_len;
1113 SOCKBUF_LOCK_ASSERT(sb);
1114 MPASS(sb->sb_flags & SB_TLS_RX);
1115 seqno = sb->sb_tls_seqno;
1116 resid = sb->sb_tlscc;
1129 if (resid <
sizeof(hdr)) {
1131 *residp =
sizeof(hdr) - resid;
1135 m_copydata(m, offset,
sizeof(hdr), (
void *)&hdr);
1137 record_len =
sizeof(hdr) + ntohs(hdr.tls_length);
1138 if (resid <= record_len) {
1140 *residp = record_len - resid;
1143 resid -= record_len;
1145 while (record_len != 0) {
1146 if (m->m_len - offset > record_len) {
1147 offset += record_len;
1151 record_len -= (m->m_len - offset);
1159ktls_enable_rx(
struct socket *so,
struct tls_enable *en)
1161 struct ktls_session *tls;
1166 if (SOLISTENING(so))
1169 counter_u64_add(ktls_offload_enable_calls, 1);
1175 if (so->so_proto->pr_protocol != IPPROTO_TCP)
1182 if (so->so_rcv.sb_tls_info != NULL)
1188 error = ktls_create_session(so, en, &tls);
1192 error = ktls_ocf_try(so, tls, KTLS_RX);
1199 SOCKBUF_LOCK(&so->so_rcv);
1200 so->so_rcv.sb_tls_seqno = be64dec(en->rec_seq);
1201 so->so_rcv.sb_tls_info = tls;
1202 so->so_rcv.sb_flags |= SB_TLS_RX;
1205 sb_mark_notready(&so->so_rcv);
1207 SOCKBUF_UNLOCK(&so->so_rcv);
1210 error = ktls_try_toe(so, tls, KTLS_RX);
1215 counter_u64_add(ktls_offload_total, 1);
1221ktls_enable_tx(
struct socket *so,
struct tls_enable *en)
1223 struct ktls_session *tls;
1229 if (SOLISTENING(so))
1232 counter_u64_add(ktls_offload_enable_calls, 1);
1238 if (so->so_proto->pr_protocol != IPPROTO_TCP)
1245 if (so->so_snd.sb_tls_info != NULL)
1255 error = ktls_create_session(so, en, &tls);
1261 error = ktls_try_toe(so, tls, KTLS_TX);
1264 error = ktls_try_ifnet(so, tls,
false);
1266 error = ktls_try_sw(so, tls, KTLS_TX);
1273 error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
1286 SOCKBUF_LOCK(&so->so_snd);
1287 so->so_snd.sb_tls_seqno = be64dec(en->rec_seq);
1288 so->so_snd.sb_tls_info = tls;
1289 if (tls->mode != TCP_TLS_MODE_SW)
1290 so->so_snd.sb_flags |= SB_TLS_IFNET;
1291 SOCKBUF_UNLOCK(&so->so_snd);
1293 SOCK_IO_SEND_UNLOCK(so);
1295 counter_u64_add(ktls_offload_total, 1);
1301ktls_get_rx_mode(
struct socket *so,
int *modep)
1303 struct ktls_session *tls;
1304 struct inpcb *inp __diagused;
1306 if (SOLISTENING(so))
1309 INP_WLOCK_ASSERT(inp);
1310 SOCK_RECVBUF_LOCK(so);
1311 tls = so->so_rcv.sb_tls_info;
1313 *modep = TCP_TLS_MODE_NONE;
1316 SOCK_RECVBUF_UNLOCK(so);
1334ktls_get_rx_sequence(
struct inpcb *inp, uint32_t *tcpseq, uint64_t *tlsseq)
1340 so = inp->inp_socket;
1341 if (__predict_false(so == NULL)) {
1345 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
1347 return (ECONNRESET);
1350 tp = intotcpcb(inp);
1353 SOCKBUF_LOCK(&so->so_rcv);
1354 *tcpseq = tp->rcv_nxt - so->so_rcv.sb_tlscc;
1355 *tlsseq = so->so_rcv.sb_tls_seqno;
1356 SOCKBUF_UNLOCK(&so->so_rcv);
1364ktls_get_tx_mode(
struct socket *so,
int *modep)
1366 struct ktls_session *tls;
1367 struct inpcb *inp __diagused;
1369 if (SOLISTENING(so))
1372 INP_WLOCK_ASSERT(inp);
1373 SOCK_SENDBUF_LOCK(so);
1374 tls = so->so_snd.sb_tls_info;
1376 *modep = TCP_TLS_MODE_NONE;
1379 SOCK_SENDBUF_UNLOCK(so);
1387ktls_set_tx_mode(
struct socket *so,
int mode)
1389 struct ktls_session *tls, *tls_new;
1393 if (SOLISTENING(so))
1396 case TCP_TLS_MODE_SW:
1397 case TCP_TLS_MODE_IFNET:
1404 INP_WLOCK_ASSERT(inp);
1405 SOCKBUF_LOCK(&so->so_snd);
1406 tls = so->so_snd.sb_tls_info;
1408 SOCKBUF_UNLOCK(&so->so_snd);
1412 if (tls->mode ==
mode) {
1413 SOCKBUF_UNLOCK(&so->so_snd);
1417 tls = ktls_hold(tls);
1418 SOCKBUF_UNLOCK(&so->so_snd);
1421 tls_new = ktls_clone_session(tls);
1423 if (
mode == TCP_TLS_MODE_IFNET)
1424 error = ktls_try_ifnet(so, tls_new,
true);
1426 error = ktls_try_sw(so, tls_new, KTLS_TX);
1428 counter_u64_add(ktls_switch_failed, 1);
1435 error = SOCK_IO_SEND_LOCK(so, SBL_WAIT);
1437 counter_u64_add(ktls_switch_failed, 1);
1448 if (tls != so->so_snd.sb_tls_info) {
1449 counter_u64_add(ktls_switch_failed, 1);
1450 SOCK_IO_SEND_UNLOCK(so);
1458 SOCKBUF_LOCK(&so->so_snd);
1459 so->so_snd.sb_tls_info = tls_new;
1460 if (tls_new->mode != TCP_TLS_MODE_SW)
1461 so->so_snd.sb_flags |= SB_TLS_IFNET;
1462 SOCKBUF_UNLOCK(&so->so_snd);
1463 SOCK_IO_SEND_UNLOCK(so);
1470 KASSERT(tls->refcount >= 2, (
"too few references on old session"));
1474 if (
mode == TCP_TLS_MODE_IFNET)
1475 counter_u64_add(ktls_switch_to_ifnet, 1);
1477 counter_u64_add(ktls_switch_to_sw, 1);
1491ktls_reset_send_tag(
void *context,
int pending)
1493 struct epoch_tracker et;
1494 struct ktls_session *tls;
1495 struct m_snd_tag *old, *
new;
1500 MPASS(pending == 1);
1517 tls->snd_tag = NULL;
1520 m_snd_tag_rele(old);
1522 error = ktls_alloc_snd_tag(inp, tls,
true, &
new);
1528 tls->reset_pending =
false;
1530 if (!in_pcbrele_wlocked(inp))
1533 counter_u64_add(ktls_ifnet_reset, 1);
1540 NET_EPOCH_ENTER(et);
1542 if (!in_pcbrele_wlocked(inp)) {
1543 if (!(inp->inp_flags & INP_TIMEWAIT) &&
1544 !(inp->inp_flags & INP_DROPPED)) {
1545 tp = intotcpcb(inp);
1546 CURVNET_SET(tp->t_vnet);
1547 tp = tcp_drop(tp, ECONNABORTED);
1551 counter_u64_add(ktls_ifnet_reset_dropped, 1);
1557 counter_u64_add(ktls_ifnet_reset_failed, 1);
1569ktls_output_eagain(
struct inpcb *inp,
struct ktls_session *tls)
1575 INP_LOCK_ASSERT(inp);
1582 if (!tls->reset_pending) {
1583 (void) ktls_hold(tls);
1586 tls->reset_pending =
true;
1595ktls_modify_txrtlmt(
struct ktls_session *tls, uint64_t max_pacing_rate)
1597 union if_snd_tag_modify_params params = {
1598 .rate_limit.max_rate = max_pacing_rate,
1599 .rate_limit.flags = M_NOWAIT,
1601 struct m_snd_tag *mst;
1606 MPASS(tls->mode == TCP_TLS_MODE_IFNET);
1608 if (tls->snd_tag == NULL) {
1618 MPASS(tls->snd_tag != NULL);
1619 MPASS(tls->snd_tag->sw->type == IF_SND_TAG_TYPE_TLS_RATE_LIMIT);
1622 return (mst->sw->snd_tag_modify(mst, ¶ms));
1631 if (tls->sequential_records) {
1635 STAILQ_FOREACH_SAFE(m, &tls->pending_records, m_epg_stailq, n) {
1636 page_count = m->m_epg_enc_cnt;
1637 while (page_count > 0) {
1638 KASSERT(page_count >= m->m_epg_nrdy,
1639 (
"%s: too few pages", __func__));
1640 page_count -= m->m_epg_nrdy;
1653 for (; m != NULL; m = m->m_next) {
1654 KASSERT((m->m_flags & M_EXTPG) != 0,
1655 (
"ktls_seq: mapped mbuf %p", m));
1657 m->m_epg_seqno = sb->sb_tls_seqno;
1678ktls_frame(
struct mbuf *top,
struct ktls_session *tls,
int *enq_cnt,
1679 uint8_t record_type)
1681 struct tls_record_layer *tlshdr;
1685 int maxlen __diagused;
1687 maxlen = tls->params.max_frame_len;
1689 for (m = top; m != NULL; m = m->m_next) {
1696 KASSERT(m->m_len <= maxlen && m->m_len >= 0 &&
1698 (
"ktls_frame: m %p len %d", m, m->m_len));
1704 KASSERT((m->m_flags & M_EXTPG) != 0,
1705 (
"ktls_frame: mapped mbuf %p (top = %p)", m, top));
1710 m->m_epg_tls = ktls_hold(tls);
1712 m->m_epg_hdrlen = tls->params.tls_hlen;
1713 m->m_epg_trllen = tls->params.tls_tlen;
1714 if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {
1733 bs = tls->params.tls_bs;
1734 delta = (tls_len + tls->params.tls_tlen) & (bs - 1);
1735 m->m_epg_trllen -= delta;
1737 m->m_len += m->m_epg_hdrlen + m->m_epg_trllen;
1740 tlshdr = (
void *)m->m_epg_hdr;
1741 tlshdr->tls_vmajor = tls->params.tls_vmajor;
1747 if (tls->params.tls_vminor == TLS_MINOR_VER_THREE &&
1748 tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) {
1749 tlshdr->tls_vminor = TLS_MINOR_VER_TWO;
1750 tlshdr->tls_type = TLS_RLTYPE_APP;
1752 m->m_epg_record_type = record_type;
1753 m->m_epg_trail[0] = record_type;
1755 tlshdr->tls_vminor = tls->params.tls_vminor;
1756 tlshdr->tls_type = record_type;
1758 tlshdr->tls_length = htons(m->m_len -
sizeof(*tlshdr));
1770 if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&
1771 tls->params.tls_vminor == TLS_MINOR_VER_TWO) {
1772 noncep = (uint64_t *)(tls->params.iv + 8);
1773 be64enc(tlshdr + 1, *noncep);
1775 }
else if (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&
1776 tls->params.tls_vminor >= TLS_MINOR_VER_ONE)
1777 arc4rand(tlshdr + 1, AES_BLOCK_LEN, 0);
1787 if (tls->mode == TCP_TLS_MODE_SW) {
1788 m->m_flags |= M_NOTREADY;
1789 if (__predict_false(tls_len == 0)) {
1793 m->m_epg_nrdy = m->m_epg_npgs;
1794 *enq_cnt += m->m_epg_nrdy;
1802 return (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&
1803 tls->params.tls_vminor == TLS_MINOR_VER_ZERO);
1809 struct tls_record_layer hdr;
1814 SOCKBUF_LOCK_ASSERT(sb);
1815 KASSERT(sb->sb_flags & SB_TLS_RX, (
"%s: sockbuf %p isn't TLS RX",
1817 so = __containerof(sb,
struct socket, so_rcv);
1819 if (sb->sb_flags & SB_TLS_RX_RUNNING)
1823 if (sb->sb_tlscc <
sizeof(hdr)) {
1824 if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc != 0)
1825 so->so_error = EMSGSIZE;
1829 m_copydata(sb->sb_mtls, 0,
sizeof(hdr), (
void *)&hdr);
1832 if (sb->sb_tlscc <
sizeof(hdr) + ntohs(hdr.tls_length)) {
1833 if ((sb->sb_state & SBS_CANTRCVMORE) != 0)
1834 so->so_error = EMSGSIZE;
1838 sb->sb_flags |= SB_TLS_RX_RUNNING;
1841 wq = &
ktls_wq[so->so_rcv.sb_tls_info->wq_index];
1843 STAILQ_INSERT_TAIL(&wq->so_head, so, so_ktls_rx_list);
1844 running = wq->running;
1845 mtx_unlock(&wq->
mtx);
1848 counter_u64_add(ktls_cnt_rx_queued, 1);
1854 struct mbuf *m, *n, *top;
1857 SOCKBUF_LOCK_ASSERT(sb);
1858 MPASS(len <= sb->sb_tlscc);
1865 if (sb->sb_tlscc == len) {
1867 sb->sb_mtlstail = NULL;
1877 for (m = top; remain > m->m_len; m = m->m_next)
1881 if (remain == m->m_len) {
1882 sb->sb_mtls = m->m_next;
1883 if (sb->sb_mtls == NULL)
1884 sb->sb_mtlstail = NULL;
1893 n = m_get(M_NOWAIT, MT_DATA);
1902 n = m_get(M_WAITOK, MT_DATA);
1905 if (sb->sb_mtls != top) {
1910 n->m_flags |= M_NOTREADY;
1913 n->m_len = m->m_len - remain;
1914 if (m->m_flags & M_EXT) {
1915 n->m_data = m->m_data + remain;
1918 bcopy(mtod(m, caddr_t) + remain, mtod(n, caddr_t), n->m_len);
1922 m->m_len -= n->m_len;
1923 sb->sb_tlscc -= n->m_len;
1924 sb->sb_ccc -= n->m_len;
1927 sballoc_ktls_rx(sb, n);
1931 n->m_next = m->m_next;
1932 if (sb->sb_mtlstail == m)
1933 sb->sb_mtlstail = n;
1940 for (m = top; m != NULL; m = m->m_next)
1941 sbfree_ktls_rx(sb, m);
1942 sb->sb_tlsdcc = len;
1964 int *trailer_len, uint8_t *record_typep)
1967 u_int digest_start, last_offset, m_len, offset;
1968 uint8_t record_type;
1970 digest_start = tls_len - *trailer_len;
1973 for (; m != NULL && offset < digest_start;
1974 offset += m->m_len, m = m->m_next) {
1976 m_len = min(digest_start - offset, m->m_len);
1977 cp = mtod(m,
char *);
1980 while (m_len > 0 && cp[m_len - 1] == 0)
1983 record_type = cp[m_len - 1];
1984 last_offset = offset + m_len;
1987 if (last_offset < tls->params.tls_hlen)
1990 *record_typep = record_type;
1991 *trailer_len = tls_len - last_offset + 1;
1998 char tls_header[MBUF_PEXT_HDR_LEN];
1999 struct ktls_session *tls;
2001 struct tls_record_layer *hdr;
2002 struct tls_get_record tgr;
2003 struct mbuf *control, *
data, *m;
2005 int error, remain, tls_len, trail_len;
2007 uint8_t vminor, record_type;
2009 hdr = (
struct tls_record_layer *)tls_header;
2012 KASSERT(sb->sb_flags & SB_TLS_RX_RUNNING,
2013 (
"%s: socket %p not running", __func__, so));
2015 tls = sb->sb_tls_info;
2018 tls13 = (tls->params.tls_vminor == TLS_MINOR_VER_THREE);
2020 vminor = TLS_MINOR_VER_TWO;
2022 vminor = tls->params.tls_vminor;
2025 if (sb->sb_tlscc < tls->params.tls_hlen)
2028 m_copydata(sb->sb_mtls, 0, tls->params.tls_hlen, tls_header);
2029 tls_len =
sizeof(*hdr) + ntohs(hdr->tls_length);
2031 if (hdr->tls_vmajor != tls->params.tls_vmajor ||
2032 hdr->tls_vminor != vminor)
2034 else if (tls13 && hdr->tls_type != TLS_RLTYPE_APP)
2036 else if (tls_len < tls->params.tls_hlen || tls_len >
2037 tls->params.tls_hlen + TLS_MAX_MSG_SIZE_V10_2 +
2038 tls->params.tls_tlen)
2042 if (__predict_false(error != 0)) {
2049 counter_u64_add(ktls_offload_corrupted_records, 1);
2051 CURVNET_SET(so->so_vnet);
2052 so->so_proto->pr_usrreqs->pru_abort(so);
2053 so->so_error = error;
2059 if (sb->sb_tlscc < tls_len)
2069 MPASS(sb->sb_tlsdcc == tls_len);
2071 seqno = sb->sb_tls_seqno;
2076 error = tls->sw_decrypt(tls, hdr,
data, seqno, &trail_len);
2080 tls_len, &trail_len, &record_type);
2082 record_type = hdr->tls_type;
2085 counter_u64_add(ktls_offload_failed_crypto, 1);
2088 if (sb->sb_tlsdcc == 0) {
2101 sb->sb_ccc -= tls_len;
2104 CURVNET_SET(so->so_vnet);
2105 so->so_error = EBADMSG;
2106 sorwakeup_locked(so);
2116 memset(&tgr, 0,
sizeof(tgr));
2117 tgr.tls_type = record_type;
2118 tgr.tls_vmajor = hdr->tls_vmajor;
2119 tgr.tls_vminor = hdr->tls_vminor;
2120 tgr.tls_length = htobe16(tls_len - tls->params.tls_hlen -
2123 TLS_GET_RECORD, IPPROTO_TCP, M_WAITOK);
2126 if (sb->sb_tlsdcc == 0) {
2128 MPASS(sb->sb_tlscc == 0);
2138 sb->sb_ccc -= tls_len;
2143 if (tgr.tls_length == htobe16(0)) {
2148 remain = tls->params.tls_hlen;
2149 while (remain > 0) {
2150 if (
data->m_len > remain) {
2151 data->m_data += remain;
2152 data->m_len -= remain;
2155 remain -=
data->m_len;
2160 remain = be16toh(tgr.tls_length);
2162 for (m =
data; remain > m->m_len; m = m->m_next) {
2163 m->m_flags &= ~M_NOTREADY;
2169 m->m_flags &= ~M_NOTREADY;
2172 m->m_flags |= M_EOR;
2178 sb->sb_flags &= ~SB_TLS_RX_RUNNING;
2180 if ((sb->sb_state & SBS_CANTRCVMORE) != 0 && sb->sb_tlscc > 0)
2181 so->so_error = EMSGSIZE;
2183 sorwakeup_locked(so);
2186 SOCKBUF_UNLOCK_ASSERT(sb);
2188 CURVNET_SET(so->so_vnet);
2200 m->m_epg_flags |= EPG_FLAG_2FREE;
2201 wq = &
ktls_wq[m->m_epg_tls->wq_index];
2203 STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq);
2204 running = wq->running;
2205 mtx_unlock(&wq->
mtx);
2216 if (m->m_epg_npgs <= 2)
2220 if ((u_int)(
ticks - wq->lastallocfail) <
hz) {
2232 wq->lastallocfail =
ticks;
2241 if (!VM_DOMAIN_EMPTY(
domain)) {
2252 struct ktls_session *tls,
struct ktls_ocf_encrypt_state *state)
2255 int error, i, len, off;
2257 KASSERT((m->m_flags & (M_EXTPG | M_NOTREADY)) == (M_EXTPG | M_NOTREADY),
2258 (
"%p not unready & nomap mbuf\n", m));
2260 (
"page count %d larger than maximum frame length %d", m->m_epg_npgs,
2264 if ((m->m_epg_flags & EPG_FLAG_ANON) != 0)
2265 return (tls->sw_encrypt(state, tls, m, NULL, 0));
2272 len = ptoa(m->m_epg_npgs - 1) + m->m_epg_last_len -
2274 state->dst_iov[0].iov_base = (
char *)state->cbuf +
2276 state->dst_iov[0].iov_len = len;
2277 state->parray[0] = DMAP_TO_PHYS((vm_offset_t)state->cbuf);
2280 off = m->m_epg_1st_off;
2281 for (i = 0; i < m->m_epg_npgs; i++, off = 0) {
2282 pg = vm_page_alloc_noobj(VM_ALLOC_NODUMP |
2283 VM_ALLOC_WIRED | VM_ALLOC_WAITOK);
2284 len = m_epg_pagelen(m, i, off);
2285 state->parray[i] = VM_PAGE_TO_PHYS(pg);
2286 state->dst_iov[i].iov_base =
2287 (
char *)PHYS_TO_DMAP(state->parray[i]) + off;
2288 state->dst_iov[i].iov_len = len;
2291 KASSERT(i + 1 <= nitems(state->dst_iov), (
"dst_iov is too small"));
2292 state->dst_iov[i].iov_base = m->m_epg_trail;
2293 state->dst_iov[i].iov_len = m->m_epg_trllen;
2295 error = tls->sw_encrypt(state, tls, m, state->dst_iov, i + 1);
2297 if (__predict_false(error != 0)) {
2299 if (state->cbuf != NULL)
2302 for (i = 0; i < m->m_epg_npgs; i++) {
2303 pg = PHYS_TO_VM_PAGE(state->parray[i]);
2304 (void)vm_page_unwire_noq(pg);
2316 int page_count, records;
2319 page_count = m->m_epg_enc_cnt;
2320 while (page_count > 0) {
2322 page_count -= m->m_epg_nrdy;
2325 KASSERT(page_count == 0, (
"%s: mismatched page count", __func__));
2332 struct ktls_session *tls;
2337 KASSERT(((m->m_flags & (M_EXTPG | M_NOTREADY)) ==
2338 (M_EXTPG | M_NOTREADY)),
2339 (
"ktls_enqueue: %p not unready & nomap mbuf\n", m));
2340 KASSERT(page_count != 0, (
"enqueueing TLS mbuf with zero page count"));
2342 KASSERT(m->m_epg_tls->mode == TCP_TLS_MODE_SW, (
"!SW TLS mbuf"));
2344 m->m_epg_enc_cnt = page_count;
2356 if (__predict_false(tls->sequential_records)) {
2379 if (m->m_epg_seqno != tls->next_seqno) {
2383 STAILQ_FOREACH(n, &tls->pending_records, m_epg_stailq) {
2384 if (n->m_epg_seqno > m->m_epg_seqno)
2389 STAILQ_INSERT_TAIL(&tls->pending_records, m,
2392 STAILQ_INSERT_HEAD(&tls->pending_records, m,
2395 STAILQ_INSERT_AFTER(&tls->pending_records, p, m,
2397 mtx_unlock(&wq->
mtx);
2398 counter_u64_add(ktls_cnt_tx_pending, 1);
2403 STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq);
2405 while (!STAILQ_EMPTY(&tls->pending_records)) {
2408 n = STAILQ_FIRST(&tls->pending_records);
2409 if (n->m_epg_seqno != tls->next_seqno)
2413 STAILQ_REMOVE_HEAD(&tls->pending_records, m_epg_stailq);
2415 STAILQ_INSERT_TAIL(&wq->m_head, n, m_epg_stailq);
2417 counter_u64_add(ktls_cnt_tx_pending, -(queued - 1));
2419 STAILQ_INSERT_TAIL(&wq->m_head, m, m_epg_stailq);
2421 running = wq->running;
2422 mtx_unlock(&wq->
mtx);
2425 counter_u64_add(ktls_cnt_tx_queued, queued);
2438 MPASS((m->m_epg_flags & EPG_FLAG_ANON) == 0);
2441 m->m_ext.ext_free(m);
2444 if (state->cbuf != NULL) {
2445 for (i = 0; i < m->m_epg_npgs; i++)
2446 m->m_epg_pa[i] = state->parray[0] + ptoa(i);
2451 for (i = 0; i < m->m_epg_npgs; i++)
2452 m->m_epg_pa[i] = state->parray[i];
2459 m->m_epg_flags |= EPG_FLAG_ANON;
2462static __noinline
void
2465 struct ktls_ocf_encrypt_state state;
2466 struct ktls_session *tls;
2469 int error, npages, total_pages;
2472 tls = top->m_epg_tls;
2473 KASSERT(tls != NULL, (
"tls = NULL, top = %p\n", top));
2474 KASSERT(so != NULL, (
"so = NULL, top = %p\n", top));
2476 top->m_epg_so = NULL;
2478 total_pages = top->m_epg_enc_cnt;
2499 for (m = top; npages != total_pages; m = m->m_next) {
2500 KASSERT(m->m_epg_tls == tls,
2501 (
"different TLS sessions in a single mbuf chain: %p vs %p",
2502 tls, m->m_epg_tls));
2503 KASSERT(npages + m->m_epg_npgs <= total_pages,
2504 (
"page count mismatch: top %p, total_pages %d, m %p", top,
2509 counter_u64_add(ktls_offload_failed_crypto, 1);
2513 if ((m->m_epg_flags & EPG_FLAG_ANON) == 0)
2516 npages += m->m_epg_nrdy;
2525 m->m_epg_tls = NULL;
2529 CURVNET_SET(so->so_vnet);
2531 (void)(*so->so_proto->pr_usrreqs->pru_ready)(so, top, npages);
2533 so->so_proto->pr_usrreqs->pru_abort(so);
2545 struct ktls_session *tls;
2552 if ((m->m_epg_flags & EPG_FLAG_ANON) == 0)
2556 free(state, M_KTLS);
2565 m->m_epg_tls = NULL;
2569 counter_u64_add(ktls_offload_failed_crypto, 1);
2571 CURVNET_SET(so->so_vnet);
2572 npages = m->m_epg_nrdy;
2575 (void)(*so->so_proto->pr_usrreqs->pru_ready)(so, m, npages);
2577 so->so_proto->pr_usrreqs->pru_abort(so);
2591static __noinline
void
2594 struct ktls_ocf_encrypt_state *state;
2595 struct ktls_session *tls;
2598 int error, mpages, npages, total_pages;
2601 tls = top->m_epg_tls;
2602 KASSERT(tls != NULL, (
"tls = NULL, top = %p\n", top));
2603 KASSERT(so != NULL, (
"so = NULL, top = %p\n", top));
2605 top->m_epg_so = NULL;
2607 total_pages = top->m_epg_enc_cnt;
2611 for (m = top; npages != total_pages; m = n) {
2612 KASSERT(m->m_epg_tls == tls,
2613 (
"different TLS sessions in a single mbuf chain: %p vs %p",
2614 tls, m->m_epg_tls));
2615 KASSERT(npages + m->m_epg_npgs <= total_pages,
2616 (
"page count mismatch: top %p, total_pages %d, m %p", top,
2619 state =
malloc(
sizeof(*state), M_KTLS, M_WAITOK | M_ZERO);
2624 mpages = m->m_epg_nrdy;
2629 counter_u64_add(ktls_offload_failed_crypto, 1);
2630 free(state, M_KTLS);
2631 CURVNET_SET(so->so_vnet);
2640 CURVNET_SET(so->so_vnet);
2642 so->so_proto->pr_usrreqs->pru_abort(so);
2659 curthread->td_domain.dr_policy = DOMAINSET_PREF(
domain);
2669 struct sysctl_oid *oid;
2671 int domain, error, i, nbufs;
2675 printf(
"Starting KTLS alloc thread for domain %d\n",
domain);
2678 printf(
"Unable to bind KTLS alloc thread for domain %d: error %d\n",
2681 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_kern_ipc_tls), OID_AUTO,
2682 name, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
"");
2683 SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"allocs",
2684 CTLFLAG_RD, &sc->
allocs, 0,
"buffers allocated");
2685 SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"wakeups",
2686 CTLFLAG_RD, &sc->
wakeups, 0,
"thread wakeups");
2687 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO,
"running",
2688 CTLFLAG_RD, &sc->
running, 0,
"thread running");
2693 atomic_store_int(&sc->
running, 0);
2694 tsleep(sc, PZERO | PNOLOCK,
"-", 0);
2695 atomic_store_int(&sc->
running, 1);
2700 buf =
malloc(
sizeof(
void *) * nbufs, M_KTLS,
2716 for (i = 0; i < nbufs; i++) {
2720 for (i = 0; i < nbufs; i++) {
2732 struct socket *so, *son;
2739 printf(
"Starting KTLS worker thread for CPU %d\n", cpu);
2755 CPU_SETOF(cpu, &
mask);
2759 printf(
"Unable to bind KTLS worker thread for CPU %d: error %d\n",
2762#if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
2767 while (STAILQ_EMPTY(&wq->m_head) &&
2768 STAILQ_EMPTY(&wq->so_head)) {
2769 wq->running =
false;
2770 mtx_sleep(wq, &wq->
mtx, 0,
"-", 0);
2774 STAILQ_INIT(&local_m_head);
2775 STAILQ_CONCAT(&local_m_head, &wq->m_head);
2776 STAILQ_INIT(&local_so_head);
2777 STAILQ_CONCAT(&local_so_head, &wq->so_head);
2778 mtx_unlock(&wq->
mtx);
2780 STAILQ_FOREACH_SAFE(m, &local_m_head, m_epg_stailq, n) {
2781 if (m->m_epg_flags & EPG_FLAG_2FREE) {
2782 ktls_free(m->m_epg_tls);
2785 if (m->m_epg_tls->sync_dispatch)
2789 counter_u64_add(ktls_cnt_tx_queued, -1);
2793 STAILQ_FOREACH_SAFE(so, &local_so_head, so_ktls_rx_list, son) {
2795 counter_u64_add(ktls_cnt_rx_queued, -1);
2800#if defined(INET) || defined(INET6)
2802ktls_disable_ifnet_help(
void *context,
int pending __unused)
2804 struct ktls_session *tls;
2815 so = inp->inp_socket;
2817 if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
2821 if (so->so_snd.sb_tls_info != NULL)
2822 err = ktls_set_tx_mode(so, TCP_TLS_MODE_SW);
2826 counter_u64_add(ktls_ifnet_disable_ok, 1);
2828 if ((inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) == 0 &&
2829 (tp = intotcpcb(inp)) != NULL &&
2830 tp->t_fb->tfb_hwtls_change != NULL)
2831 (*tp->t_fb->tfb_hwtls_change)(tp, 0);
2833 counter_u64_add(ktls_ifnet_disable_fail, 1);
2838 if (!in_pcbrele_wlocked(inp))
2857ktls_disable_ifnet(
void *arg)
2862 struct ktls_session *tls;
2866 INP_WLOCK_ASSERT(inp);
2867 so = inp->inp_socket;
2869 tls = so->so_snd.sb_tls_info;
2870 if (tls->disable_ifnet_pending) {
2881 (void)ktls_hold(tls);
2884 tls->disable_ifnet_pending =
true;
2887 TASK_INIT(&tls->disable_ifnet_task, 0, ktls_disable_ifnet_help, tls);
static STAILQ_HEAD(cn_device)
cpuset_t cpuset_domain[MAXMEMDOM]
int cpuset_setthread(lwpid_t id, cpuset_t *mask)
int kproc_kthread_add(void(*func)(void *), void *arg, struct proc **procptr, struct thread **tdptr, int flags, int pages, const char *procname, const char *fmt,...)
void *() malloc(size_t size, struct malloc_type *mtp, int flags)
void zfree(void *addr, struct malloc_type *mtp)
void free(void *addr, struct malloc_type *mtp)
void m_freem(struct mbuf *mb)
void m_free_raw(struct mbuf *mb)
int m_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params, struct m_snd_tag **mstp)
void mb_free_notready(struct mbuf *m, int count)
struct mtx_pool __read_mostly * mtxpool_sleep
void panic(const char *fmt,...)
void wakeup(const void *ident)
struct iommu_domain ** domain
struct ktls_alloc_thread alloc_td
struct pcpu * pcpu_find(u_int cpuid)
int printf(const char *fmt,...)
int snprintf(char *str, size_t size, const char *format,...)
int taskqueue_enqueue(struct taskqueue *queue, struct task *task)
static uint16_t ktls_cpuid_lookup[MAXCPU]
static int ktls_max_alloc
static COUNTER_U64_DEFINE_EARLY(ktls_tasks_active)
void ktls_seq(struct sockbuf *sb, struct mbuf *m)
static void ktls_finish_nonanon(struct mbuf *m, struct ktls_ocf_encrypt_state *state)
SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD, &ktls_tasks_active, "Number of active tasks")
void ktls_enqueue_to_free(struct mbuf *m)
static int ktls_encrypt_record(struct ktls_wq *wq, struct mbuf *m, struct ktls_session *tls, struct ktls_ocf_encrypt_state *state)
void ktls_encrypt_cb(struct ktls_ocf_encrypt_state *state, int error)
static int ktls_start_kthreads(void)
static int ktls_init_state
struct ktls_domain_info ktls_domains[MAXMEMDOM]
static int ktls_number_threads
SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN, &ktls_bind_threads, 0, "Bind crypto threads to cores (1) or cores and domains (2) at boot")
bool ktls_permit_empty_frames(struct ktls_session *tls)
static int ktls_bind_domain(int domain)
void ktls_check_rx(struct sockbuf *sb)
static int tls13_find_record_type(struct ktls_session *tls, struct mbuf *m, int tls_len, int *trailer_len, uint8_t *record_typep)
static uma_zone_t ktls_session_zone
void ktls_destroy(struct ktls_session *tls)
static bool ktls_cbc_enable
static void ktls_free_mext_contig(struct mbuf *m)
void ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
static struct sx ktls_init_lock
static struct ktls_wq * ktls_wq
SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RWTUN, &ktls_offload_enable, 0, "Enable support for kernel TLS offload")
struct ktls_alloc_thread __aligned
static void ktls_cleanup(struct ktls_session *tls)
SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RDTUN, &ktls_maxlen, 0, "Maximum TLS record size")
static struct proc * ktls_proc
static void ktls_alloc_thread(void *ctx)
static __noinline void ktls_encrypt_async(struct ktls_wq *wq, struct mbuf *top)
static struct mbuf * ktls_detach_record(struct sockbuf *sb, int len)
SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "Kernel TLS offload")
static int ktls_buffer_import(void *arg, void **store, int count, int domain, int flags)
static bool ktls_sw_buffer_cache
static bool ktls_offload_enable
static int ktls_init(void)
static __noinline void ktls_encrypt(struct ktls_wq *wq, struct mbuf *top)
static void ktls_work_thread(void *ctx)
static u_int ktls_batched_records(struct mbuf *m)
static uma_zone_t ktls_buffer_zone
void ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt, uint8_t record_type)
static int ktls_ifnet_permitted
static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS")
static void ktls_decrypt(struct socket *so)
unsigned int ktls_ifnet_max_rexmit_pct
SX_SYSINIT(ktls_init_lock, &ktls_init_lock, "ktls init")
static void ktls_buffer_release(void *arg __unused, void **store, int count)
static void * ktls_buffer_alloc(struct ktls_wq *wq, struct mbuf *m)
static int ktls_bind_threads
void mb_free_mext_pgs(struct mbuf *m)
void m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
u_int m_length(struct mbuf *m0, struct mbuf **last)
void mb_dupcl(struct mbuf *n, struct mbuf *m)
void sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control, int flags)
struct mbuf * sbcreatecontrol_how(void *p, int size, int type, int level, int wait)