166#include <sys/cdefs.h>
171#include <sys/param.h>
173#include <sys/kernel.h>
175#include <sys/limits.h>
178#include <sys/rmlock.h>
180#include <sys/socket.h>
181#include <sys/socketvar.h>
182#include <sys/sysctl.h>
183#include <sys/systm.h>
185#include <crypto/siphash/siphash.h>
194#define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH
196#if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
197#error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
204#if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
205#error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
208#define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16
209#define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048
211#define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900
213#if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
214#define TCP_FASTOPEN_MAX_KEYS 2
216#define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS
219#if TCP_FASTOPEN_MAX_KEYS > 10
220#undef TCP_FASTOPEN_MAX_KEYS
221#define TCP_FASTOPEN_MAX_KEYS 10
224#if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
225#define TCP_FASTOPEN_MAX_PSKS 2
227#define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS
230#if TCP_FASTOPEN_MAX_PSKS > 10
231#undef TCP_FASTOPEN_MAX_PSKS
232#define TCP_FASTOPEN_MAX_PSKS 10
257SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
261#define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany)
263 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
264 "Accept any non-empty cookie");
267#define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey)
270 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
272 "Number of seconds between auto-generation of a new key; zero disables");
276 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT,
278 "Max entries per bucket in client cookie cache");
282#define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
284 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
285 "Client cookie cache number of buckets (power of 2)");
290 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
292 "Enable/disable TCP Fast Open client functionality");
296 "Key length in bytes");
300 "Maximum number of keys supported");
304 "Maximum number of pre-shared keys supported");
307#define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys)
309 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
310 "Number of keys installed");
313#define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks)
315 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
316 "Number of pre-shared keys installed");
320#define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
322 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
323 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
326#define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable)
329 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
331 "Enable/disable TCP Fast Open server pre-shared key mode");
336 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
338 "Enable/disable TCP Fast Open server functionality");
342 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
344 "Install a new key");
348 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
350 "Install a new pre-shared key");
354 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
356 "List of all client cookie cache entries");
359#define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock)
361#define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t))
362#define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t))
363#define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock)
364#define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock)
367#define V_tcp_fastopen_keys VNET(tcp_fastopen_keys)
370#define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
373#define V_counter_zone VNET(counter_zone)
375static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE,
"tfo_ccache",
"TFO client cookie cache buckets");
378#define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache)
380#define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx)
381#define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx)
382#define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
390 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
459 (
"%s: TFO ccache zone allocation count not 0", __func__));
471 unsigned int *counter;
484 atomic_subtract_int(counter, 1);
540 SipHash24_Init(&ctx);
541 SipHash_SetKey(&ctx, key);
545 SipHash_Update(&ctx, &inc->inc_faddr,
sizeof(inc->inc_faddr));
550 SipHash_Update(&ctx, &inc->inc6_faddr,
sizeof(inc->inc6_faddr));
554 SipHash_Final((u_int8_t *)&siphash, &ctx);
565 SipHash24_Init(&ctx);
566 SipHash_SetKey(&ctx, psk);
567 SipHash_Update(&ctx,
cookie, cookie_len);
568 SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
576 unsigned int i, psk_index;
588 if (memcmp(wire_cookie, &psk_cookie,
611 unsigned int len, uint64_t *latest_cookie)
613 struct rm_priotracker tracker;
614 unsigned int i, key_index;
645 *latest_cookie = cur_cookie;
668 error = sysctl_handle_int(oidp, &
new, 0, req);
669 if (error == 0 && req->newptr) {
670 if (
new > (INT_MAX / hz))
696 error = sysctl_handle_int(oidp, &
new, 0, req);
697 if (error == 0 && req->newptr) {
723 error = sysctl_handle_int(oidp, &
new, 0, req);
724 if (error == 0 && req->newptr) {
761 if (req->oldptr != NULL || req->oldlen != 0)
763 if (req->newptr == NULL)
765 if (req->newlen !=
sizeof(newkey))
767 error = SYSCTL_IN(req, newkey,
sizeof(newkey));
784 if (req->oldptr != NULL || req->oldlen != 0)
786 if (req->newptr == NULL)
788 if (req->newlen !=
sizeof(newpsk))
790 error = SYSCTL_IN(req, newpsk,
sizeof(newpsk));
810 error = sysctl_handle_int(oidp, &
new, 0, req);
811 if (error == 0 && req->newptr) {
812 if ((
new == 0) || (
new > INT_MAX))
836 error = sysctl_handle_int(oidp, &
new, 0, req);
837 if (error == 0 && req->newptr) {
843 (
"%s: ccb->ccb_num_entries %d is negative",
853 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
854 (
"%s: ccb->ccb_entries not empty", __func__));
856 (
"%s: ccb->ccb_num_entries %d not -1", __func__,
884 TCP_FASTOPEN_PSK_LEN)) {
897 TCP_FASTOPEN_PSK_LEN && psk_cookie) {
910 now = getsbinuptime();
974 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
990 if ((
cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
991 (
cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1027 last_word = inc->inc_fport;
1028 hash = jenkins_hash32(&last_word, 1, hash);
1036 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1039 (((cce->
af == AF_INET) &&
1042 ((cce->
af == AF_INET6) &&
1073 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1079 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1083 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1085 if (cce->
af == AF_INET) {
1093 if ((
cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1094 (
cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1115 unsigned int entries;
1119 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1121 if (entries > limit)
1125 (
"%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1128 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1129 (
"%s: ccb->ccb_entries not empty", __func__));
1142 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1153 sbintime_t now, duration, limit;
1154 const int linesize = 128;
1155 int i, error, num_entries;
1158 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1160 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1163 if (jailed_without_vnet(curthread->td_ucred) != 0)
1167 if (curthread->td_ucred->cr_uid != 0)
1178 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1181 "\nLocal IP address Remote IP address Port MSS"
1182 " Disabled Cookie\n");
1184 now = getsbinuptime();
1189 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1192 if (limit >= duration)
1193 duration = limit - duration;
1199 "%-20s %-20s %5u %5u ",
1201 clt_buf,
sizeof(clt_buf)),
1203 srv_buf,
sizeof(srv_buf)),
1207 sbuf_printf(&sb,
"%7ds ", sbintime_getsec(duration));
1209 sbuf_printf(&sb,
"%8s ",
"No");
1211 sbuf_printf(&sb,
"%02x", cce->
cookie[j]);
1212 sbuf_putc(&sb,
'\n');
1216 error = sbuf_finish(&sb);
1218 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
char * inet_ntop(int, const void *, char *, socklen_t)
const struct encaptab * cookie
struct in_endpoints inc_ie
union in_dependaddr ie_dependladdr
union in_dependaddr ie_dependfaddr
struct in_conninfo inp_inc
uint8_t cookie[TCP_FASTOPEN_MAX_COOKIE_LEN]
union tcp_fastopen_ip_addr cce_client_ip
union tcp_fastopen_ip_addr cce_server_ip
uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN]
uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN]
union tcpcb::@55 t_tfo_cookie
uint8_t t_tfo_client_cookie_len
uint8_t client[TCP_FASTOPEN_MAX_COOKIE_LEN]
#define TCP_FASTOPEN_KEYS_WLOCK()
SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "TCP Fast Open")
#define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT
static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *, struct tcp_fastopen_ccache_bucket *)
static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
VNET_DEFINE(unsigned int, tcp_fastopen_client_enable)
SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets, CTLFLAG_VNET|CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0, "Client cookie cache number of buckets (power of 2)")
static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
#define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT
static uint64_t tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets")
static uint64_t tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
#define V_tcp_fastopen_numkeys
unsigned int * tcp_fastopen_alloc_counter(void)
#define V_tcp_fastopen_autokey_ctx
#define CCB_LOCK_ASSERT(ccb)
int tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie, unsigned int len, uint64_t *latest_cookie)
#define V_tcp_fastopen_acceptany
#define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT
static int tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey, CTLFLAG_VNET|CTLTYPE_UINT|CTLFLAG_RW|CTLFLAG_MPSAFE, NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU", "Number of seconds between auto-generation of a new key; zero disables")
static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
#define V_tcp_fastopen_ccache_buckets
static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *, unsigned int)
static void tcp_fastopen_addkey_locked(uint8_t *key)
#define TCP_FASTOPEN_MAX_PSKS
SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany, CTLFLAG_VNET|CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0, "Accept any non-empty cookie")
VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany)=0
#define V_tcp_fastopen_numpsks
#define V_tcp_fastopen_psk_enable
static struct tcp_fastopen_ccache_entry * tcp_fastopen_ccache_lookup(struct in_conninfo *, struct tcp_fastopen_ccache_bucket **)
static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
void tcp_fastopen_init(void)
#define V_tcp_fastopen_keylock
#define TCP_FASTOPEN_KEY_LEN
void tcp_fastopen_destroy(void)
void tcp_fastopen_disable_path(struct tcpcb *tp)
void tcp_fastopen_decrement_counter(unsigned int *counter)
#define TCP_FASTOPEN_KEYS_WUNLOCK()
static void tcp_fastopen_autokey_locked(void)
#define V_tcp_fastopen_ccache
#define V_tcp_fastopen_keys
static void tcp_fastopen_autokey_callout(void *arg)
#define TCP_FASTOPEN_KEYS_RLOCK(t)
#define TCP_FASTOPEN_KEYS_RUNLOCK(t)
#define TCP_FASTOPEN_MAX_KEYS
static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
#define V_tcp_fastopen_autokey
static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
#define V_tcp_fastopen_path_disable_time
static struct tcp_fastopen_ccache_entry * tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t, uint8_t *)
static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
void tcp_fastopen_connect(struct tcpcb *tp)
void tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
static void tcp_fastopen_addpsk_locked(uint8_t *psk)
#define V_tcp_fastopen_server_enable
#define V_tcp_fastopen_client_enable
#define TCP_FASTOPEN_COOKIE_LEN