77#include <sys/kernel.h>
80#include <sys/malloc.h>
84#include <sys/socket.h>
85#include <sys/socketvar.h>
86#include <sys/sysctl.h>
98 CK_SLIST_HEAD(hc_qhead,
hc_metrics) hch_bucket;
119#ifdef TCP_HC_COUNTERS
141#define TCP_HOSTCACHE_HASHSIZE 512
142#define TCP_HOSTCACHE_BUCKETLIMIT 30
143#define TCP_HOSTCACHE_EXPIRE 60*60
144#define TCP_HOSTCACHE_PRUNE 5*60
147#define V_tcp_hostcache VNET(tcp_hostcache)
150#define V_tcp_hc_callout VNET(tcp_hc_callout)
160 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
164#define V_tcp_use_hostcache VNET(tcp_use_hostcache)
165SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW,
166 &VNET_NAME(tcp_use_hostcache), 0,
167 "Enable the TCP hostcache");
169SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_VNET | CTLFLAG_RDTUN,
171 "Overall entry limit for hostcache");
173SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, hashsize, CTLFLAG_VNET | CTLFLAG_RDTUN,
175 "Size of TCP hostcache hashtable");
179 "Per-bucket hash limit for hostcache");
183 "Current number of entries in hostcache");
187 "Expire time of TCP hostcache entries");
189SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, prune, CTLFLAG_VNET | CTLFLAG_RW,
191 "Time between purge runs");
193SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, purge, CTLFLAG_VNET | CTLFLAG_RW,
195 "Expire all entries on next purge run");
198 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
200 "List of all hostcache entries");
203 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
205 "Print a histogram of hostcache hashbucket utilization");
208 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
210 "Immediately purge all entries");
215#define HOSTCACHE_HASH(inc) \
216 ((inc)->inc_flags & INC_ISIPV6) ? \
217 (jenkins_hash32((inc)->inc6_faddr.s6_addr32, 4, \
218 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask) \
220 (jenkins_hash32(&(inc)->inc_faddr.s_addr, 1, \
221 V_tcp_hostcache.hashsalt) & V_tcp_hostcache.hashmask)
223#define THC_LOCK(h) mtx_lock(&(h)->hch_mtx)
224#define THC_UNLOCK(h) mtx_unlock(&(h)->hch_mtx)
242 TUNABLE_INT_FETCH(
"net.inet.tcp.hostcache.hashsize",
245 printf(
"WARNING: hostcache hash size is not a power of 2.\n");
250 TUNABLE_INT_FETCH(
"net.inet.tcp.hostcache.bucketlimit",
255 TUNABLE_INT_FETCH(
"net.inet.tcp.hostcache.cachelimit",
265 M_HOSTCACHE, M_WAITOK | M_ZERO);
281 uma_zcreate(
"hostcache",
sizeof(
struct hc_metrics),
282 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_SMR);
323 if (memcmp(&inc->inc6_faddr, &hc_entry->ip6,
324 sizeof(inc->inc6_faddr)) == 0)
327 if (memcmp(&inc->inc_faddr, &hc_entry->ip4,
328 sizeof(inc->inc_faddr)) == 0)
345 KASSERT(inc != NULL, (
"%s: NULL in_conninfo", __func__));
353 CK_SLIST_FOREACH(hc_entry, &
hc_head->hch_bucket, rmx_q)
357 if (hc_entry != NULL) {
358 if (atomic_load_int(&hc_entry->rmx_expire) !=
360 atomic_store_int(&hc_entry->rmx_expire,
362#ifdef TCP_HC_COUNTERS
363 hc_entry->rmx_hits++;
394 if (hc_entry == NULL) {
425 if (hc_entry == NULL) {
429 mtu = atomic_load_32(&hc_entry->rmx_mtu);
466 CK_SLIST_FOREACH(hc_entry, &
hc_head->hch_bucket, rmx_q) {
469 if (CK_SLIST_NEXT(hc_entry, rmx_q) != NULL)
473 if (hc_entry != NULL) {
474 if (atomic_load_int(&hc_entry->rmx_expire) !=
476 atomic_store_int(&hc_entry->rmx_expire,
478#ifdef TCP_HC_COUNTERS
479 hc_entry->rmx_updates++;
496 if (hc_prev != NULL) {
497 hc_entry = CK_SLIST_NEXT(hc_prev, rmx_q);
498 KASSERT(CK_SLIST_NEXT(hc_entry, rmx_q) == NULL,
499 (
"%s: %p is not one to last",
501 CK_SLIST_REMOVE_AFTER(hc_prev, rmx_q);
502 }
else if ((hc_entry =
503 CK_SLIST_FIRST(&
hc_head->hch_bucket)) != NULL) {
504 KASSERT(CK_SLIST_NEXT(hc_entry, rmx_q) == NULL,
505 (
"%s: %p is not the only element",
506 __func__, hc_entry));
507 CK_SLIST_REMOVE_HEAD(&
hc_head->hch_bucket,
513 KASSERT(
hc_head->hch_length > 0 &&
515 (
"tcp_hostcache: bucket length violated at %p",
527 if (hc_entry == NULL) {
535 bzero(hc_entry,
sizeof(*hc_entry));
537 hc_entry->ip6 = inc->inc6_faddr;
538 hc_entry->ip6_zoneid = inc->inc6_zoneid;
540 hc_entry->ip4 = inc->inc_faddr;
550 atomic_store_32(&hc_entry->rmx_mtu, hcml->
rmx_mtu);
553 if (hc_entry->rmx_rtt == 0)
556 v = ((uint64_t)hc_entry->rmx_rtt +
558 atomic_store_32(&hc_entry->rmx_rtt, v);
562 if (hc_entry->rmx_rttvar == 0)
565 v = ((uint64_t)hc_entry->rmx_rttvar +
567 atomic_store_32(&hc_entry->rmx_rttvar, v);
571 if (hc_entry->rmx_ssthresh == 0)
575 atomic_store_32(&hc_entry->rmx_ssthresh, v);
579 if (hc_entry->rmx_cwnd == 0)
582 v = ((uint64_t)hc_entry->rmx_cwnd +
584 atomic_store_32(&hc_entry->rmx_cwnd, v);
588 if (hc_entry->rmx_sendpipe == 0)
591 v = ((uint64_t)hc_entry->rmx_sendpipe +
593 atomic_store_32(&hc_entry->rmx_sendpipe, v);
597 if (hc_entry->rmx_recvpipe == 0)
600 v = ((uint64_t)hc_entry->rmx_recvpipe +
602 atomic_store_32(&hc_entry->rmx_recvpipe, v);
610 CK_SLIST_INSERT_HEAD(&
hc_head->hch_bucket, hc_entry, rmx_q);
613 (
"tcp_hostcache: bucket length too high at %p",
hc_head));
616 }
else if (hc_entry != CK_SLIST_FIRST(&
hc_head->hch_bucket)) {
617 KASSERT(CK_SLIST_NEXT(hc_prev, rmx_q) == hc_entry,
618 (
"%s: %p next is not %p", __func__, hc_prev, hc_entry));
619 CK_SLIST_REMOVE_AFTER(hc_prev, rmx_q);
620 CK_SLIST_INSERT_HEAD(&
hc_head->hch_bucket, hc_entry, rmx_q);
632 const int linesize = 128;
636 char ip4buf[INET_ADDRSTRLEN];
638 char ip6buf[INET6_ADDRSTRLEN];
641 if (jailed_without_vnet(curthread->td_ucred) != 0)
645 if (req->oldptr == NULL) {
648 return (SYSCTL_OUT(req, NULL, len));
651 error = sysctl_wire_old_buffer(req, 0);
661 "\nIP address MTU SSTRESH RTT RTTVAR "
662 " CWND SENDPIPE RECVPIPE "
663#ifdef TCP_HC_COUNTERS
669#define msec(u) (((u) + 500) / 1000)
672 CK_SLIST_FOREACH(hc_entry,
675 "%-15s %5u %8u %6lums %6lums %8u %8u %8u "
676#ifdef TCP_HC_COUNTERS
680 hc_entry->ip4.s_addr ?
683 ip6_sprintf(ip6buf, &hc_entry->ip6),
688 hc_entry->rmx_ssthresh,
689 msec((u_long)hc_entry->rmx_rtt *
691 msec((u_long)hc_entry->rmx_rttvar *
694 hc_entry->rmx_sendpipe,
695 hc_entry->rmx_recvpipe,
696#ifdef TCP_HC_COUNTERS
698 hc_entry->rmx_updates,
700 hc_entry->rmx_expire);
706 error = sbuf_finish(&sb);
718 const int linesize = 50;
724 if (jailed_without_vnet(curthread->td_ucred) != 0)
727 histo = (
int *)malloc(
sizeof(
int) * (
V_tcp_hostcache.bucket_limit + 1),
728 M_TEMP, M_NOWAIT|M_ZERO);
735 (
"tcp_hostcache: bucket limit exceeded at %u: %u",
741 sbuf_new_for_sysctl(&sb, NULL, 16 * linesize, req);
743 sbuf_printf(&sb,
"\nLength\tCount\n");
745 sbuf_printf(&sb,
"%u\t%u\n", i, histo[i]);
747 error = sbuf_finish(&sb);
760 struct hc_metrics *hc_entry, *hc_next, *hc_prev;
767 CK_SLIST_FOREACH_SAFE(hc_entry, &head->hch_bucket, rmx_q,
769 KASSERT(head->hch_length > 0 && head->hch_length <=
771 "bucket length out of range at %u: %u", i,
774 atomic_load_int(&hc_entry->rmx_expire) <= 0) {
775 if (hc_prev != NULL) {
777 CK_SLIST_NEXT(hc_prev, rmx_q),
778 (
"%s: %p is not next to %p",
779 __func__, hc_entry, hc_prev));
780 CK_SLIST_REMOVE_AFTER(hc_prev, rmx_q);
783 CK_SLIST_FIRST(&head->hch_bucket),
784 (
"%s: %p is not first",
785 __func__, hc_entry));
786 CK_SLIST_REMOVE_HEAD(&head->hch_bucket,
793 atomic_subtract_int(&hc_entry->rmx_expire,
809 CURVNET_SET((
struct vnet *) arg);
835 error = sysctl_handle_int(oidp, &val, 0, req);
836 if (error || !req->newptr)
char * inet_ntoa_r(struct in_addr ina, char *buf)
struct hc_head * hashbase
SYSCTL_UINT(_net_inet_tcp_hostcache, OID_AUTO, cachelimit, CTLFLAG_VNET|CTLFLAG_RDTUN, &VNET_NAME(tcp_hostcache.cache_limit), 0, "Overall entry limit for hostcache")
uint32_t tcp_hc_getmtu(struct in_conninfo *inc)
static void tcp_hc_purge(void *)
static void tcp_hc_purge_internal(int)
static int sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS)
static SYSCTL_NODE(_net_inet_tcp, OID_AUTO, hostcache, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "TCP Host cache")
void tcp_hc_updatemtu(struct in_conninfo *inc, uint32_t mtu)
void tcp_hc_get(struct in_conninfo *inc, struct hc_metrics_lite *hc_metrics_lite)
#define HOSTCACHE_HASH(inc)
static bool tcp_hc_cmp(struct hc_metrics *hc_entry, struct in_conninfo *inc)
VNET_DEFINE_STATIC(struct tcp_hostcache, tcp_hostcache)
#define TCP_HOSTCACHE_BUCKETLIMIT
#define V_tcp_use_hostcache
#define TCP_HOSTCACHE_HASHSIZE
SYSCTL_PROC(_net_inet_tcp_hostcache, OID_AUTO, list, CTLTYPE_STRING|CTLFLAG_RD|CTLFLAG_SKIP|CTLFLAG_MPSAFE, 0, 0, sysctl_tcp_hc_list, "A", "List of all hostcache entries")
VNET_DEFINE(int, tcp_use_hostcache)
static int sysctl_tcp_hc_histo(SYSCTL_HANDLER_ARGS)
static struct hc_metrics * tcp_hc_lookup(struct in_conninfo *)
#define TCP_HOSTCACHE_PRUNE
#define TCP_HOSTCACHE_EXPIRE
static int sysctl_tcp_hc_purgenow(SYSCTL_HANDLER_ARGS)
SYSCTL_INT(_net_inet_tcp_hostcache, OID_AUTO, enable, CTLFLAG_VNET|CTLFLAG_RW, &VNET_NAME(tcp_use_hostcache), 0, "Enable the TCP hostcache")
static MALLOC_DEFINE(M_HOSTCACHE, "hostcache", "TCP hostcache")
void tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
#define TCPSTAT_INC(name)