47#include "opt_ratelimit.h"
54#include <sys/libkern.h>
56#include <sys/malloc.h>
58#include <sys/callout.h>
59#include <sys/eventhandler.h>
60#include <sys/domain.h>
61#include <sys/protosw.h>
63#include <sys/socket.h>
64#include <sys/socketvar.h>
65#include <sys/sockio.h>
68#include <sys/refcount.h>
70#include <sys/kernel.h>
71#include <sys/sysctl.h>
81#include <net/if_var.h>
82#include <net/if_types.h>
83#include <net/if_llatbl.h>
85#include <net/rss_config.h>
88#if defined(INET) || defined(INET6)
105#include <netinet6/in6_pcb.h>
106#include <netinet6/in6_var.h>
107#include <netinet6/ip6_var.h>
109#include <net/route/nhop.h>
112#include <netipsec/ipsec_support.h>
114#include <security/mac/mac_framework.h>
116#define INPCBLBGROUP_SIZMIN 8
117#define INPCBLBGROUP_SIZMAX 256
118#define INP_FREED 0x00000200
149#define V_ipport_tcplastcount VNET(ipport_tcplastcount)
152static struct inpcb *in_pcblookup_hash_locked(
struct inpcbinfo *pcbinfo,
153 struct in_addr faddr, u_int fport_arg,
154 struct in_addr laddr, u_int lport_arg,
155 int lookupflags,
struct ifnet *ifp,
158#define RANGECHK(var, min, max) \
159 if ((var) < (min)) { (var) = (min); } \
160 else if ((var) > (max)) { (var) = (max); }
163sysctl_net_ipport_check(SYSCTL_HANDLER_ARGS)
167 error = sysctl_handle_int(oidp, arg1, arg2, req);
182 CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
185SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowfirst,
186 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
187 &VNET_NAME(ipport_lowfirstauto), 0, &sysctl_net_ipport_check,
"I",
189SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, lowlast,
190 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
191 &VNET_NAME(ipport_lowlastauto), 0, &sysctl_net_ipport_check,
"I",
193SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, first,
194 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
195 &VNET_NAME(ipport_firstauto), 0, &sysctl_net_ipport_check,
"I",
198 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
199 &VNET_NAME(ipport_lastauto), 0, &sysctl_net_ipport_check,
"I",
201SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hifirst,
202 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
203 &VNET_NAME(ipport_hifirstauto), 0, &sysctl_net_ipport_check,
"I",
205SYSCTL_PROC(_net_inet_ip_portrange, OID_AUTO, hilast,
206 CTLFLAG_VNET | CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
207 &VNET_NAME(ipport_hilastauto), 0, &sysctl_net_ipport_check,
"I",
209SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedhigh,
210 CTLFLAG_VNET | CTLFLAG_RW | CTLFLAG_SECURE,
211 &VNET_NAME(ipport_reservedhigh), 0,
"");
212SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, reservedlow,
213 CTLFLAG_RW|CTLFLAG_SECURE, &VNET_NAME(ipport_reservedlow), 0,
"");
214SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomized,
215 CTLFLAG_VNET | CTLFLAG_RW,
216 &VNET_NAME(ipport_randomized), 0,
"Enable random port allocation");
217SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomcps,
218 CTLFLAG_VNET | CTLFLAG_RW,
219 &VNET_NAME(ipport_randomcps), 0,
"Maximum number of random port "
220 "allocations before switching to a sequential one");
221SYSCTL_INT(_net_inet_ip_portrange, OID_AUTO, randomtime,
222 CTLFLAG_VNET | CTLFLAG_RW,
223 &VNET_NAME(ipport_randomtime), 0,
224 "Minimum time to keep sequential port "
225 "allocation before switching to a random one");
228counter_u64_t rate_limit_new;
229counter_u64_t rate_limit_chg;
230counter_u64_t rate_limit_active;
231counter_u64_t rate_limit_alloc_fail;
232counter_u64_t rate_limit_set_ok;
234static SYSCTL_NODE(_net_inet_ip, OID_AUTO, rl, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
237 &rate_limit_active,
"Active rate limited connections");
239 &rate_limit_alloc_fail,
"Rate limited connection failures");
241 &rate_limit_set_ok,
"Rate limited setting succeeded");
243 &rate_limit_new,
"Total Rate limit new attempts");
245 &rate_limit_chg,
"Total Rate limited change attempts");
278 grp = malloc(bytes, M_PCB, M_ZERO | M_NOWAIT);
286 CK_LIST_INSERT_HEAD(hdr, grp, il_list);
303 CK_LIST_REMOVE(grp, il_list);
321 (
"invalid new local group size %d and old local group count %d",
361 const static struct timeval interval = { 60, 0 };
362 static struct timeval lastprint;
364 struct inpcblbgrouphead *hdr;
392 CK_LIST_FOREACH(grp, hdr, il_list) {
410 if (ratecheck(&lastprint, &interval))
411 printf(
"lb group port %d, limit reached\n",
423 (
"invalid local group size %d and count %d", grp->
il_inpsiz,
438 struct inpcblbgrouphead *hdr;
449 CK_LIST_FOREACH(grp, hdr, il_list) {
451 if (grp->
il_inp[i] != inp)
470 struct inpcblbgrouphead *hdr;
477 numa_domain = M_NODOM;
480 numa_domain = PCPU_GET(domain);
483 if (arg < 0 || arg >= vm_ndomains)
494 CK_LIST_FOREACH(grp, hdr, il_list) {
496 if (grp->
il_inp[i] != inp)
500 goto abort_with_hash_wlock;
508 goto abort_with_hash_wlock;
512abort_with_hash_wlock:
525 u_int hash_nelements, u_int porthash_nelements)
538 porthash_nelements = imin(porthash_nelements, IPPORT_MAX + 1);
556 (
"%s: ipi_count = %u", __func__, pcbinfo->
ipi_count));
581 sizeof(
struct inpcbport), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
583 uma_zone_get_smr(pcbstor->
ips_zone));
609 inp = uma_zalloc_smr(pcbinfo->
ipi_zone, M_NOWAIT);
618 inp->
inp_cred = crhold(so->so_cred);
621 error = mac_inpcb_init(inp, M_NOWAIT);
624 mac_inpcb_create(so, inp);
626#if defined(IPSEC) || defined(IPSEC_SUPPORT)
627 error = ipsec_init_pcbpolicy(inp);
630 mac_inpcb_destroy(inp);
641 if (V_ip6_auto_flowlabel)
661 CK_LIST_INSERT_HEAD(&pcbinfo->
ipi_listhead, inp, inp_list);
667#if defined(IPSEC) || defined(IPSEC_SUPPORT) || defined(MAC)
669 uma_zfree_smr(pcbinfo->
ipi_zone, inp);
680 KASSERT(nam == NULL || nam->sa_family == AF_INET,
681 (
"%s: invalid address family for %p", __func__, nam));
682 KASSERT(nam == NULL || nam->sa_len ==
sizeof(
struct sockaddr_in),
683 (
"%s: invalid address length for %p", __func__, nam));
687 if (inp->inp_lport != 0 || inp->inp_laddr.s_addr !=
INADDR_ANY)
689 anonport = nam == NULL || ((
struct sockaddr_in *)nam)->sin_port == 0;
691 &inp->inp_lport, cred);
705#if defined(INET) || defined(INET6)
714 struct sockaddr *fsa, u_short fport,
struct ucred *cred,
int lookupflags)
717 struct inpcb *tmpinp;
718 unsigned short *lastport;
719 int count, dorandom, error;
720 u_short aux, first, last, lport;
725 struct in6_addr *laddr6, *faddr6;
742 error = priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT);
798 laddr6 = &((
struct sockaddr_in6 *)lsa)->sin6_addr;
800 faddr6 = &((
struct sockaddr_in6 *)fsa)->sin6_addr;
808 *lastport = first + (arc4random() % (last - first));
810 count = last - first;
814 return (EADDRNOTAVAIL);
816 if (*lastport < first || *lastport > last)
818 lport = htons(*lastport);
822 if (lsa->sa_family == AF_INET) {
823 tmpinp = in_pcblookup_hash_locked(pcbinfo,
824 faddr, fport, laddr, lport, lookupflags,
829 if (lsa->sa_family == AF_INET6) {
830 tmpinp = in6_pcblookup_hash_locked(pcbinfo,
831 faddr6, fport, laddr6, lport, lookupflags,
838 tmpinp = in6_pcblookup_local(pcbinfo,
839 &inp->in6p_laddr, lport, lookupflags, cred);
841#if defined(INET) && defined(INET6)
846 lport, lookupflags, cred);
849 }
while (tmpinp != NULL);
861 struct ucred *cred,
int lookupflags)
866 bzero(&laddr,
sizeof(laddr));
867 laddr.sin_family = AF_INET;
868 laddr.sin_addr = *laddrp;
871 NULL, lportp, NULL, 0, cred, lookupflags));
885 so_options |= SO_REUSEPORT_LB;
887 so_options |= SO_REUSEPORT;
889 so_options |= SO_REUSEADDR;
936 u_short *lportp,
struct ucred *cred)
943 int lookupflags = 0, reuseport = (so->so_options & SO_REUSEPORT);
950 int reuseport_lb = (so->so_options & SO_REUSEPORT_LB);
958 laddr.s_addr = *laddrp;
959 if (nam != NULL && laddr.s_addr !=
INADDR_ANY)
961 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT|SO_REUSEPORT_LB)) == 0)
969 (
"%s: invalid family for address %p", __func__, sin));
970 KASSERT(sin->
sin_len ==
sizeof(*sin),
971 (
"%s: invalid length for address %p", __func__, sin));
991 if ((so->so_options & (SO_REUSEADDR|SO_REUSEPORT)) != 0)
992 reuseport = SO_REUSEADDR|SO_REUSEPORT;
997 if ((so->so_options &
998 (SO_REUSEADDR|SO_REUSEPORT_LB)) != 0)
999 reuseport_lb = SO_REUSEADDR|SO_REUSEPORT_LB;
1009 ifa_ifwithaddr_check((
struct sockaddr *)sin) == 0)
1010 return (EADDRNOTAVAIL);
1012 laddr = sin->sin_addr;
1020 priv_check_cred(cred, PRIV_NETINET_RESERVEDPORT))
1022 if (!IN_MULTICAST(ntohl(sin->sin_addr.s_addr)) &&
1023 priv_check_cred(inp->
inp_cred, PRIV_NETINET_REUSEPORT) != 0) {
1033 (so->so_type != SOCK_STREAM ||
1035 (ntohl(sin->sin_addr.s_addr) !=
INADDR_ANY ||
1041 return (EADDRINUSE);
1050 return (EADDRINUSE);
1053 lport, lookupflags, cred);
1066 return (EADDRINUSE);
1073 if (ntohl(sin->sin_addr.s_addr) !=
1075 ntohl(t->inp_laddr.s_addr) !=
1080 return (EADDRINUSE);
1082 return (EADDRINUSE);
1089 error =
in_pcb_lport(inp, &laddr, &lport, cred, lookupflags);
1093 *laddrp = laddr.s_addr;
1108 u_short lport, fport;
1110 int anonport, error;
1115 lport = inp->inp_lport;
1116 laddr = inp->inp_laddr.s_addr;
1117 anonport = (lport == 0);
1124 if (inp->inp_laddr.s_addr ==
INADDR_ANY && inp->inp_lport == 0) {
1125 KASSERT(rehash ==
true,
1126 (
"Rehashing required for unbound inps"));
1127 inp->inp_lport = lport;
1128 inp->inp_laddr.s_addr = laddr;
1137 inp->inp_lport = lport;
1138 inp->inp_laddr.s_addr = laddr;
1139 inp->inp_faddr.s_addr = faddr;
1140 inp->inp_fport = fport;
1161 struct sockaddr *sa;
1163 struct nhop_object *nh;
1167 KASSERT(laddr != NULL, (
"%s: laddr NULL", __func__));
1178 bzero(&dst,
sizeof(dst));
1190 if ((inp->
inp_socket->so_options & SO_DONTROUTE) == 0)
1202 if (nh == NULL || nh->nh_ifp == NULL) {
1206 ia =
ifatoia(ifa_ifwithdstaddr((
struct sockaddr *)sin,
1209 ia =
ifatoia(ifa_ifwithnet((
struct sockaddr *)sin, 0,
1213 error = ENETUNREACH;
1217 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1224 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1226 if (sa->sa_family != AF_INET)
1253 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) == 0) {
1258 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1279 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1281 if (sa->sa_family != AF_INET)
1308 if ((nh->nh_ifp->if_flags & IFF_LOOPBACK) != 0) {
1319 if (cred == NULL || !prison_flag(cred, PR_IP4)) {
1321 error = ENETUNREACH;
1334 CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
1336 if (sa->sa_family != AF_INET)
1378 struct inpcb **oinpp,
struct ucred *cred)
1384 u_short lport, fport;
1388 (
"%s: invalid address family for %p", __func__, sin));
1389 KASSERT(sin->
sin_len ==
sizeof(*sin),
1390 (
"%s: invalid address length for %p", __func__, sin));
1403 return (EADDRNOTAVAIL);
1409 if (CALC_FLOWID_OUTBOUND) {
1413 inp->
inp_socket->so_proto->pr_protocol, &hash_type);
1436 faddr =
satosin(&CK_STAILQ_FIRST(
1447 if (IN_MULTICAST(ntohl(faddr.
s_addr)) &&
1449 struct ip_moptions *imo;
1453 if (imo->imo_multicast_ifp != NULL) {
1454 ifp = imo->imo_multicast_ifp;
1456 if ((ia->ia_ifp == ifp) &&
1463 error = EADDRNOTAVAIL;
1475 oinp = in_pcblookup_hash_locked(inp->
inp_pcbinfo, faddr,
1476 fport, laddr, lport, 0, NULL, M_NODOM);
1480 return (EADDRINUSE);
1485 bzero(&lsin,
sizeof(lsin));
1486 bzero(&fsin,
sizeof(fsin));
1487 lsin.sin_family = AF_INET;
1488 lsin.sin_addr = laddr;
1489 fsin.sin_family = AF_INET;
1490 fsin.sin_addr = faddr;
1492 &lport, (
struct sockaddr *)& fsin, fport, cred,
1527 KASSERT(inp->
inp_socket != NULL, (
"%s: inp_socket == NULL", __func__));
1531 in_pcbdetach_txrtlmt(inp);
1594 if (__predict_true(refcount_acquire_if_not_zero(&inp->
inp_refcount))) {
1643#define II_LIST_FIRST(ipi, hash) \
1644 (((hash) == INP_ALL_LIST) ? \
1645 CK_LIST_FIRST(&(ipi)->ipi_listhead) : \
1646 CK_LIST_FIRST(&(ipi)->ipi_hashbase[(hash)]))
1647#define II_LIST_NEXT(inp, hash) \
1648 (((hash) == INP_ALL_LIST) ? \
1649 CK_LIST_NEXT((inp), inp_list) : \
1650 CK_LIST_NEXT((inp), inp_hash))
1651#define II_LOCK_ASSERT(inp, lock) \
1652 rw_assert(&(inp)->inp_lock, \
1653 (lock) == INPLOOKUP_RLOCKPCB ? RA_RLOCKED : RA_WLOCKED )
1659 void *ctx = ii->
ctx;
1661 int hash = ii->
hash;
1664 if (ii->
inp == NULL) {
1670 if (match != NULL && (match)(inp, ctx) ==
false)
1703 if (match != NULL && (match)(inp, ctx) ==
false)
1729 if (__predict_true(refcount_acquire_if_not_zero(&inp->
inp_refcount))) {
1732 if (__predict_false(
in_pcbrele(inp, lock))) {
1764 u_int old __diagused;
1767 KASSERT(old > 0, (
"%s: refcount 0", __func__));
1822 struct ip_moptions *imo;
1825 struct ip6_moptions *im6o;
1829 KASSERT(inp->
inp_socket == NULL, (
"%s: inp_socket != NULL", __func__));
1831 (
"%s: called twice for pcb %p", __func__, inp));
1837 CK_LIST_REMOVE(inp, inp_list);
1847 CK_LIST_REMOVE(inp, inp_hash);
1848 CK_LIST_REMOVE(inp, inp_portlist);
1850 CK_LIST_REMOVE(phd, phd_hash);
1859 mac_inpcb_destroy(inp);
1861#if defined(IPSEC) || defined(IPSEC_SUPPORT)
1863 ipsec_delete_pcbpolicy(inp);
1882 ip6_freemoptions(im6o);
1893 struct inpcb *inp = mem;
1908 struct inpcb *inp = mem;
1947 CK_LIST_REMOVE(inp, inp_hash);
1948 CK_LIST_REMOVE(inp, inp_portlist);
1950 CK_LIST_REMOVE(phd, phd_hash);
1967 sin = malloc(
sizeof *sin, M_SONAME,
1974 return (
struct sockaddr *)sin;
1985 KASSERT(inp != NULL, (
"in_getsockaddr: inp == NULL"));
1988 port = inp->inp_lport;
1989 addr = inp->inp_laddr;
2004 KASSERT(inp != NULL, (
"in_getpeeraddr: inp == NULL"));
2007 port = inp->inp_fport;
2008 addr = inp->inp_faddr;
2017 struct inpcb *(*notify)(
struct inpcb *,
int))
2019 struct inpcb *inp, *inp_temp;
2022 CK_LIST_FOREACH_SAFE(inp, &pcbinfo->
ipi_listhead, inp_list, inp_temp) {
2030 if (inp->inp_faddr.s_addr != faddr.
s_addr ||
2035 if ((*notify)(inp, errno))
2042inp_v4_multi_match(
const struct inpcb *inp,
void *v __unused)
2055 inp_v4_multi_match, NULL);
2059 struct ip_moptions *imo;
2063 while ((inp =
inp_next(&inpi)) != NULL) {
2071 if (imo->imo_multicast_ifp == ifp)
2072 imo->imo_multicast_ifp = NULL;
2082 if ((inm = imf->
imf_inm) == NULL)
2098#define INP_LOOKUP_MAPPED_PCB_COST 3
2101 u_short lport,
int lookupflags,
struct ucred *cred)
2105 int matchwild = 3 + INP_LOOKUP_MAPPED_PCB_COST;
2112 (
"%s: invalid lookup flags %d", __func__, lookupflags));
2116 struct inpcbhead *head;
2123 CK_LIST_FOREACH(inp, head, inp_hash) {
2130 inp->inp_laddr.s_addr == laddr.
s_addr &&
2131 inp->inp_lport == lport) {
2146 struct inpcbporthead *porthash;
2148 struct inpcb *match = NULL;
2157 CK_LIST_FOREACH(phd, porthash, phd_hash) {
2166 CK_LIST_FOREACH(inp, &phd->
phd_pcblist, inp_portlist) {
2190 wildcard += INP_LOOKUP_MAPPED_PCB_COST;
2197 else if (inp->inp_laddr.s_addr != laddr.
s_addr)
2203 if (wildcard < matchwild) {
2205 matchwild = wildcard;
2214#undef INP_LOOKUP_MAPPED_PCB_COST
2216static struct inpcb *
2217in_pcblookup_lbgroup(
const struct inpcbinfo *pcbinfo,
2219 uint16_t fport,
int lookupflags,
int numa_domain)
2221 struct inpcb *local_wild, *numa_wild;
2222 const struct inpcblbgrouphead *hdr;
2242 CK_LIST_FOREACH(grp, hdr, il_list) {
2252 if (grp->il_laddr.s_addr == laddr->
s_addr) {
2253 if (numa_domain == M_NODOM ||
2255 return (grp->
il_inp[idx]);
2257 numa_wild = grp->
il_inp[idx];
2262 (local_wild == NULL || numa_domain == M_NODOM ||
2264 local_wild = grp->
il_inp[idx];
2267 if (numa_wild != NULL)
2270 return (local_wild);
2279static struct inpcb *
2281 u_int fport_arg,
struct in_addr laddr, u_int lport_arg,
int lookupflags,
2282 struct ifnet *ifp,
uint8_t numa_domain)
2284 struct inpcbhead *head;
2285 struct inpcb *inp, *tmpinp;
2286 u_short fport = fport_arg, lport = lport_arg;
2289 (
"%s: invalid lookup flags %d", __func__, lookupflags));
2298 CK_LIST_FOREACH(inp, head, inp_hash) {
2304 if (inp->inp_faddr.s_addr == faddr.
s_addr &&
2305 inp->inp_laddr.s_addr == laddr.
s_addr &&
2306 inp->inp_fport == fport &&
2307 inp->inp_lport == lport) {
2313 if (prison_flag(inp->
inp_cred, PR_IP4))
2326 inp = in_pcblookup_lbgroup(pcbinfo, &laddr, lport, &faddr,
2327 fport, lookupflags, numa_domain);
2336 struct inpcb *local_wild = NULL, *local_exact = NULL;
2338 struct inpcb *local_wild_mapped = NULL;
2340 struct inpcb *jail_wild = NULL;
2353 CK_LIST_FOREACH(inp, head, inp_hash) {
2360 inp->inp_lport != lport)
2363 injail = prison_flag(inp->
inp_cred, PR_IP4);
2366 inp->
inp_cred->cr_prison, &laddr) != 0)
2369 if (local_exact != NULL)
2373 if (inp->inp_laddr.s_addr == laddr.
s_addr) {
2378 }
else if (inp->inp_laddr.s_addr ==
INADDR_ANY) {
2382 local_wild_mapped = inp;
2391 if (jail_wild != NULL)
2393 if (local_exact != NULL)
2394 return (local_exact);
2395 if (local_wild != NULL)
2396 return (local_wild);
2398 if (local_wild_mapped != NULL)
2399 return (local_wild_mapped);
2411static struct inpcb *
2413 u_int fport,
struct in_addr laddr, u_int lport,
int lookupflags,
2414 struct ifnet *ifp,
uint8_t numa_domain)
2419 inp = in_pcblookup_hash_locked(pcbinfo, faddr, fport, laddr, lport,
2437 struct in_addr laddr, u_int lport,
int lookupflags,
struct ifnet *ifp)
2441 (
"%s: invalid lookup flags %d", __func__, lookupflags));
2443 (
"%s: LOCKPCB not set", __func__));
2445 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2446 lookupflags, ifp, M_NODOM));
2451 u_int fport,
struct in_addr laddr, u_int lport,
int lookupflags,
2452 struct ifnet *ifp,
struct mbuf *m)
2456 (
"%s: invalid lookup flags %d", __func__, lookupflags));
2458 (
"%s: LOCKPCB not set", __func__));
2460 return (in_pcblookup_hash(pcbinfo, faddr, fport, laddr, lport,
2461 lookupflags, ifp, m->m_pkthdr.numa_domain));
2471 struct inpcbhead *pcbhash;
2472 struct inpcbporthead *pcbporthash;
2481 (
"in_pcbinshash: INP_INHASHLIST"));
2486 inp->inp_lport, inp->inp_fport, pcbinfo->
ipi_hashmask)];
2490 inp->inp_lport, inp->inp_fport, pcbinfo->
ipi_hashmask)];
2500 if (so_options & SO_REUSEPORT_LB) {
2511 CK_LIST_FOREACH(phd, pcbporthash, phd_hash) {
2512 if (phd->
phd_port == inp->inp_lport)
2525 CK_LIST_INSERT_HEAD(pcbporthash, phd, phd_hash);
2528 CK_LIST_INSERT_HEAD(&phd->
phd_pcblist, inp, inp_portlist);
2529 CK_LIST_INSERT_HEAD(pcbhash, inp, inp_hash);
2549 struct inpcbhead *head;
2555 (
"in_pcbrehash: !INP_INHASHLIST"));
2560 inp->inp_lport, inp->inp_fport, pcbinfo->
ipi_hashmask)];
2564 inp->inp_lport, inp->inp_fport, pcbinfo->
ipi_hashmask)];
2566 CK_LIST_REMOVE(inp, inp_hash);
2567 CK_LIST_INSERT_HEAD(head, inp, inp_hash);
2595 KASSERT(inp != NULL, (
"in_pcbsosetlabel: so->so_pcb == NULL"));
2599 mac_inpcb_sosetlabel(so, inp);
2615 VNET_ITERATOR_DECL(vnet_iter);
2617 VNET_LIST_RLOCK_NOSLEEP();
2618 VNET_FOREACH(vnet_iter) {
2619 CURVNET_SET(vnet_iter);
2629 VNET_LIST_RUNLOCK_NOSLEEP();
2651 EVENTHANDLER_REGISTER(shutdown_pre_sync,
ip_fini, NULL,
2652 SHUTDOWN_PRI_DEFAULT);
2685#ifdef INVARIANT_SUPPORT
2708 while ((inp =
inp_next(&inpi)) != NULL)
2748 *laddr = inp->inp_laddr.s_addr;
2749 *faddr = inp->inp_faddr.s_addr;
2750 *lp = inp->inp_lport;
2751 *fp = inp->inp_fport;
2780 bzero(xi,
sizeof(*xi));
2781 xi->xi_len =
sizeof(
struct xinpcb);
2783 sotoxsocket(inp->
inp_socket, &xi->xi_socket);
2786 xi->inp_ppcb = (uintptr_t)inp->
inp_ppcb;
2804 int (*ctloutput_set)(
struct inpcb *,
struct sockopt *))
2806 struct sockopt sopt;
2810 struct sockopt_parameters *params;
2815 if (req->oldptr != NULL || req->oldlen != 0)
2817 if (req->newptr == NULL)
2819 if (req->newlen >
sizeof(buf))
2821 error = SYSCTL_IN(req, buf, req->newlen);
2824 if (req->newlen <
sizeof(
struct sockopt_parameters))
2826 params = (
struct sockopt_parameters *)buf;
2827 sopt.sopt_level = params->sop_level;
2828 sopt.sopt_name = params->sop_optname;
2829 sopt.sopt_dir = SOPT_SET;
2830 sopt.sopt_val = params->sop_optval;
2831 sopt.sopt_valsize = req->newlen -
sizeof(
struct sockopt_parameters);
2832 sopt.sopt_td = NULL;
2834 if (params->sop_inc.inc_flags &
INC_ISIPV6) {
2835 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_laddr))
2836 params->sop_inc.inc6_laddr.s6_addr16[1] =
2837 htons(params->sop_inc.inc6_zoneid & 0xffff);
2838 if (IN6_IS_SCOPE_LINKLOCAL(¶ms->sop_inc.inc6_faddr))
2839 params->sop_inc.inc6_faddr.s6_addr16[1] =
2840 htons(params->sop_inc.inc6_zoneid & 0xffff);
2843 if (params->sop_inc.inc_lport != htons(0)) {
2844 if (params->sop_inc.inc_fport == htons(0))
2851 ¶ms->sop_inc.inc6_faddr,
2852 params->sop_inc.inc_lport,
2853 params->sop_inc.inc_fport,
2858 ¶ms->sop_inc.inc_faddr,
2859 params->sop_inc.inc_lport,
2860 params->sop_inc.inc_fport,
2863 while ((inp =
inp_next(&inpi)) != NULL)
2867 return (ECONNRESET);
2870 KASSERT(so != NULL, (
"inp_socket == NULL"));
2872 error = (*ctloutput_set)(inp, &sopt);
2883db_print_indent(
int indent)
2887 for (i = 0; i < indent; i++)
2892db_print_inconninfo(
struct in_conninfo *inc,
const char *name,
int indent)
2894 char faddr_str[48], laddr_str[48];
2896 db_print_indent(indent);
2897 db_printf(
"%s at %p\n", name, inc);
2904 ip6_sprintf(laddr_str, &inc->inc6_laddr);
2905 ip6_sprintf(faddr_str, &inc->inc6_faddr);
2913 db_print_indent(indent);
2914 db_printf(
"inc_laddr %s inc_lport %u\n", laddr_str,
2915 ntohs(inc->inc_lport));
2916 db_print_indent(indent);
2917 db_printf(
"inc_faddr %s inc_fport %u\n", faddr_str,
2918 ntohs(inc->inc_fport));
2922db_print_inpflags(
int inp_flags)
2928 db_printf(
"%sINP_RECVOPTS", comma ?
", " :
"");
2932 db_printf(
"%sINP_RECVRETOPTS", comma ?
", " :
"");
2936 db_printf(
"%sINP_RECVDSTADDR", comma ?
", " :
"");
2940 db_printf(
"%sINP_ORIGDSTADDR", comma ?
", " :
"");
2944 db_printf(
"%sINP_HDRINCL", comma ?
", " :
"");
2948 db_printf(
"%sINP_HIGHPORT", comma ?
", " :
"");
2952 db_printf(
"%sINP_LOWPORT", comma ?
", " :
"");
2956 db_printf(
"%sINP_ANONPORT", comma ?
", " :
"");
2960 db_printf(
"%sINP_RECVIF", comma ?
", " :
"");
2964 db_printf(
"%sINP_MTUDISC", comma ?
", " :
"");
2968 db_printf(
"%sINP_RECVTTL", comma ?
", " :
"");
2972 db_printf(
"%sINP_DONTFRAG", comma ?
", " :
"");
2976 db_printf(
"%sINP_RECVTOS", comma ?
", " :
"");
2980 db_printf(
"%sIN6P_IPV6_V6ONLY", comma ?
", " :
"");
2984 db_printf(
"%sIN6P_PKTINFO", comma ?
", " :
"");
2988 db_printf(
"%sIN6P_HOPLIMIT", comma ?
", " :
"");
2992 db_printf(
"%sIN6P_HOPOPTS", comma ?
", " :
"");
2996 db_printf(
"%sIN6P_DSTOPTS", comma ?
", " :
"");
3000 db_printf(
"%sIN6P_RTHDR", comma ?
", " :
"");
3004 db_printf(
"%sIN6P_RTHDRDSTOPTS", comma ?
", " :
"");
3008 db_printf(
"%sIN6P_TCLASS", comma ?
", " :
"");
3012 db_printf(
"%sIN6P_AUTOFLOWLABEL", comma ?
", " :
"");
3016 db_printf(
"%sINP_TIMEWAIT", comma ?
", " :
"");
3020 db_printf(
"%sINP_ONESBCAST", comma ?
", " :
"");
3024 db_printf(
"%sINP_DROPPED", comma ?
", " :
"");
3028 db_printf(
"%sINP_SOCKREF", comma ?
", " :
"");
3032 db_printf(
"%sIN6P_RFC2292", comma ?
", " :
"");
3036 db_printf(
"IN6P_MTU%s", comma ?
", " :
"");
3042db_print_inpvflag(u_char inp_vflag)
3048 db_printf(
"%sINP_IPV4", comma ?
", " :
"");
3052 db_printf(
"%sINP_IPV6", comma ?
", " :
"");
3056 db_printf(
"%sINP_IPV6PROTO", comma ?
", " :
"");
3062db_print_inpcb(
struct inpcb *inp,
const char *name,
int indent)
3065 db_print_indent(indent);
3066 db_printf(
"%s at %p\n", name, inp);
3070 db_print_indent(indent);
3071 db_printf(
"inp_flow: 0x%x\n", inp->
inp_flow);
3073 db_print_inconninfo(&inp->
inp_inc,
"inp_conninfo", indent);
3075 db_print_indent(indent);
3076 db_printf(
"inp_ppcb: %p inp_pcbinfo: %p inp_socket: %p\n",
3079 db_print_indent(indent);
3080 db_printf(
"inp_label: %p inp_flags: 0x%x (",
3085 db_print_indent(indent);
3086 db_printf(
"inp_sp: %p inp_vflag: 0x%x (", inp->
inp_sp,
3091 db_print_indent(indent);
3092 db_printf(
"inp_ip_ttl: %d inp_ip_p: %d inp_ip_minttl: %d\n",
3095 db_print_indent(indent);
3098 db_printf(
"in6p_options: %p in6p_outputopts: %p "
3101 db_printf(
"in6p_icmp6filt: %p in6p_cksum %d "
3107 db_printf(
"inp_ip_tos: %d inp_ip_options: %p "
3112 db_print_indent(indent);
3113 db_printf(
"inp_phd: %p inp_gencnt: %ju\n", inp->
inp_phd,
3117DB_SHOW_COMMAND(
inpcb, db_show_inpcb)
3122 db_printf(
"usage: show inpcb <addr>\n");
3125 inp = (
struct inpcb *)addr;
3127 db_print_inpcb(inp,
"inpcb", 0);
3137in_pcbmodify_txrtlmt(
struct inpcb *inp,
uint32_t max_pacing_rate)
3139 union if_snd_tag_modify_params params = {
3140 .rate_limit.max_rate = max_pacing_rate,
3141 .rate_limit.flags = M_NOWAIT,
3143 struct m_snd_tag *mst;
3150 if (mst->sw->snd_tag_modify == NULL) {
3153 error = mst->sw->snd_tag_modify(mst, ¶ms);
3163in_pcbquery_txrtlmt(
struct inpcb *inp,
uint32_t *p_max_pacing_rate)
3165 union if_snd_tag_query_params params = { };
3166 struct m_snd_tag *mst;
3173 if (mst->sw->snd_tag_query == NULL) {
3176 error = mst->sw->snd_tag_query(mst, ¶ms);
3177 if (error == 0 && p_max_pacing_rate != NULL)
3178 *p_max_pacing_rate = params.rate_limit.max_rate;
3188in_pcbquery_txrlevel(
struct inpcb *inp,
uint32_t *p_txqueue_level)
3190 union if_snd_tag_query_params params = { };
3191 struct m_snd_tag *mst;
3198 if (mst->sw->snd_tag_query == NULL)
3199 return (EOPNOTSUPP);
3201 error = mst->sw->snd_tag_query(mst, ¶ms);
3202 if (error == 0 && p_txqueue_level != NULL)
3203 *p_txqueue_level = params.rate_limit.queue_level;
3212in_pcbattach_txrtlmt(
struct inpcb *inp,
struct ifnet *ifp,
3216 union if_snd_tag_alloc_params params = {
3217 .rate_limit.hdr.type = (max_pacing_rate == -1U) ?
3218 IF_SND_TAG_TYPE_UNLIMITED : IF_SND_TAG_TYPE_RATE_LIMIT,
3219 .rate_limit.hdr.flowid = flowid,
3220 .rate_limit.hdr.flowtype = flowtype,
3222 .rate_limit.max_rate = max_pacing_rate,
3223 .rate_limit.flags = M_NOWAIT,
3237 error = m_snd_tag_alloc(ifp, ¶ms, st);
3240 counter_u64_add(rate_limit_set_ok, 1);
3241 counter_u64_add(rate_limit_active, 1);
3242 }
else if (error != EOPNOTSUPP)
3243 counter_u64_add(rate_limit_alloc_fail, 1);
3249in_pcbdetach_tag(
struct m_snd_tag *mst)
3252 m_snd_tag_rele(mst);
3254 counter_u64_add(rate_limit_active, -1);
3263in_pcbdetach_txrtlmt(
struct inpcb *inp)
3265 struct m_snd_tag *mst;
3275 m_snd_tag_rele(mst);
3277 counter_u64_add(rate_limit_active, -1);
3282in_pcboutput_txrtlmt_locked(
struct inpcb *inp,
struct ifnet *ifp,
struct mbuf *mb,
uint32_t max_pacing_rate)
3293 in_pcbdetach_txrtlmt(inp);
3304 if (max_pacing_rate == 0 && inp->
inp_snd_tag == NULL) {
3306 }
else if (!(ifp->if_capenable & IFCAP_TXRTLMT)) {
3308 in_pcbdetach_txrtlmt(inp);
3316 if (M_HASHTYPE_GET(mb) == M_HASHTYPE_NONE) {
3319 error = in_pcbattach_txrtlmt(inp, ifp, M_HASHTYPE_GET(mb),
3320 mb->m_pkthdr.flowid, max_pacing_rate, &inp->
inp_snd_tag);
3323 error = in_pcbmodify_txrtlmt(inp, max_pacing_rate);
3325 if (error == 0 || error == EOPNOTSUPP)
3337in_pcboutput_txrtlmt(
struct inpcb *inp,
struct ifnet *ifp,
struct mbuf *mb)
3339 struct socket *socket;
3371 max_pacing_rate = socket->so_max_pacing_rate;
3373 error = in_pcboutput_txrtlmt_locked(inp, ifp, mb, max_pacing_rate);
3383in_pcboutput_eagain(
struct inpcb *inp)
3408 in_pcbdetach_txrtlmt(inp);
3421 rate_limit_new = counter_u64_alloc(M_WAITOK);
3422 rate_limit_chg = counter_u64_alloc(M_WAITOK);
3423 rate_limit_active = counter_u64_alloc(M_WAITOK);
3424 rate_limit_alloc_fail = counter_u64_alloc(M_WAITOK);
3425 rate_limit_set_ok = counter_u64_alloc(M_WAITOK);
3428SYSINIT(rl, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, rl_init, NULL);
SYSCTL_INT(_net_inet_accf_http, OID_AUTO, parsehttpversion, CTLFLAG_RW, &parse_http_version, 1, "Parse http version so that non 1.x requests work")
static SYSCTL_NODE(_net_inet_accf, OID_AUTO, http, CTLFLAG_RW|CTLFLAG_MPSAFE, 0, "HTTP accept filter")
SYSCTL_PROC(_net_inet_tcp_cc, OID_AUTO, algorithm, CTLFLAG_VNET|CTLTYPE_STRING|CTLFLAG_RW|CTLFLAG_MPSAFE, NULL, 0, cc_default_algo, "A", "Default congestion control algorithm")
char * inet_ntoa_r(struct in_addr ina, char *buf)
struct nhop_object * fib4_lookup(uint32_t fibnum, struct in_addr dst, uint32_t scopeid, uint32_t flags, uint32_t flowid)
uint32_t fib4_calc_software_hash(struct in_addr src, struct in_addr dst, unsigned short src_port, unsigned short dst_port, char proto, uint32_t *phashtype)
int prison_local_ip4(struct ucred *cred, struct in_addr *ia)
int prison_check_ip4(const struct ucred *cred, const struct in_addr *ia)
int prison_equal_ip4(struct prison *pr1, struct prison *pr2)
int prison_saddrsel_ip4(struct ucred *cred, struct in_addr *ia)
int prison_get_ip4(struct ucred *cred, struct in_addr *ia)
int prison_check_ip4_locked(const struct prison *pr, const struct in_addr *ia)
int in_leavegroup_locked(struct in_multi *inm, struct in_mfilter *imf)
void inp_freemoptions(struct ip_moptions *imo)
void ip_mfilter_free(struct in_mfilter *imf)
static void ipport_tick(void *xtp)
bool inp_smr_lock(struct inpcb *inp, const inp_lookup_t lock)
struct inpcb * inp_next(struct inpcb_iterator *ii)
static void in_pcblbgroup_free_deferred(epoch_context_t ctx)
bool in_pcbrele_rlocked(struct inpcb *inp)
static void in_pcbhashseed_init(void)
void inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp, uint32_t *faddr, uint16_t *fp)
int in_pcballoc(struct socket *so, struct inpcbinfo *pcbinfo)
void in_pcbrehash(struct inpcb *inp)
VNET_SYSINIT(in_pcbhashseed_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FIRST, in_pcbhashseed_init, 0)
struct tcpcb * inp_inpcbtotcpcb(struct inpcb *inp)
struct inpcb * so_sotoinpcb(struct socket *so)
static void inpcb_fini(void *, int)
SYSINIT(ipport_tick_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE, ipport_tick_init, NULL)
static void inp_unlock(struct inpcb *inp, const inp_lookup_t lock)
static void in_pcbremlbgrouphash(struct inpcb *inp)
void inp_ip_tos_set(struct inpcb *inp, int val)
VNET_DEFINE_STATIC(int, ipport_tcplastcount)
static void in_pcblbgroup_reorder(struct inpcblbgrouphead *hdr, struct inpcblbgroup **grpp, int i)
static int inp_trylock(struct inpcb *inp, const inp_lookup_t lock)
CTASSERT(sizeof(struct inpcbhead)==sizeof(LIST_HEAD(, inpcb)))
static bool in_pcbrele(struct inpcb *inp, const inp_lookup_t lock)
static void inp_lock(struct inpcb *inp, const inp_lookup_t lock)
static void ip_fini(void *xtp)
void in_pcbinfo_destroy(struct inpcbinfo *pcbinfo)
struct socket * inp_inpcbtosocket(struct inpcb *inp)
static struct callout ipport_tick_callout
void inp_wunlock(struct inpcb *inp)
VNET_DEFINE(int, ipport_lowfirstauto)
void in_pcbtoxinpcb(const struct inpcb *inp, struct xinpcb *xi)
void inp_apply_all(void(*func)(struct inpcb *, void *), void *arg)
#define II_LIST_FIRST(ipi, hash)
void in_pcbsosetlabel(struct socket *so)
void in_pcbstorage_init(void *arg)
static int in_pcbinslbgrouphash(struct inpcb *inp, uint8_t numa_domain)
#define V_ipport_tcplastcount
#define INPCBLBGROUP_SIZMAX
void in_pcbstorage_destroy(void *arg)
static void in_pcblbgroup_free(struct inpcblbgroup *grp)
#define II_LOCK_ASSERT(inp, lock)
static struct inpcblbgroup * in_pcblbgroup_alloc(struct inpcblbgrouphead *hdr, u_char vflag, uint16_t port, const union in_dependaddr *addr, int size, uint8_t numa_domain)
static void inpcb_dtor(void *, int, void *)
void inp_wlock(struct inpcb *inp)
static struct inpcblbgroup * in_pcblbgroup_resize(struct inpcblbgrouphead *hdr, struct inpcblbgroup *old_grp, int size)
struct tcpcb * so_sototcpcb(struct socket *so)
#define II_LIST_NEXT(inp, hash)
void inp_runlock(struct inpcb *inp)
void in_pcbdrop(struct inpcb *inp)
void in_pcbref(struct inpcb *inp)
int in_pcblbgroup_numa(struct inpcb *inp, int arg)
int in_pcbinshash(struct inpcb *inp)
int inp_ip_tos_get(const struct inpcb *inp)
void in_pcbdetach(struct inpcb *inp)
static void ipport_tick_init(const void *unused __unused)
void in_losing(struct inpcb *inp)
bool in_pcbrele_wlocked(struct inpcb *inp)
void in_pcbinfo_init(struct inpcbinfo *pcbinfo, struct inpcbstorage *pcbstor, u_int hash_nelements, u_int porthash_nelements)
void in_pcbfree(struct inpcb *inp)
int in_pcbbind_check_bindmulti(const struct inpcb *ni, const struct inpcb *oi)
#define INPCBLBGROUP_SIZMIN
void inp_rlock(struct inpcb *inp)
int sysctl_setsockopt(SYSCTL_HANDLER_ARGS, struct inpcbinfo *pcbinfo, int(*ctloutput_set)(struct inpcb *, struct sockopt *))
#define V_ipport_hilastauto
#define INP_TRY_UPGRADE(inp)
#define INP_PCBHASH_WILD(lport, mask)
#define INP_LOCK_ASSERT(inp)
#define V_ipport_randomized
#define INP_HASH_WLOCK_ASSERT(ipi)
#define V_ipport_lowlastauto
#define INP_HASH_WUNLOCK(ipi)
#define INP_PCBLBGROUP_PKTHASH(faddr, lport, fport)
#define V_ipport_lastauto
#define V_ipport_lowfirstauto
#define V_ipport_reservedlow
void in_pcbdisconnect(struct inpcb *)
struct inpcb * in_pcblookup(struct inpcbinfo *, struct in_addr, u_int, struct in_addr, u_int, int, struct ifnet *)
#define INP_HASH_LOCK_ASSERT(ipi)
#define INP_DOWNGRADE(inp)
#define INP_ALL_ITERATOR(_ipi, _lock)
#define INP_PCBPORTHASH(lport, mask)
int in_getsockaddr(struct socket *so, struct sockaddr **nam)
#define V_ipport_reservedhigh
#define V_ipport_randomcps
int in_pcbladdr(struct inpcb *, struct in_addr *, struct in_addr *, struct ucred *)
int in_pcbconnect_setup(struct inpcb *, struct sockaddr *, in_addr_t *, u_short *, in_addr_t *, u_short *, struct inpcb **, struct ucred *)
#define V_ipport_stoprandom
#define inp_lock_assert(inp)
int in_pcbbind(struct inpcb *, struct sockaddr *, struct ucred *)
#define inp_unlock_assert(inp)
#define INP_WLOCK_ASSERT(inp)
#define INP6_PCBHASH(faddr, lport, fport, mask)
#define INP_INFO_WUNLOCK(ipi)
int in_pcbbind_setup(struct inpcb *, struct sockaddr *, in_addr_t *, u_short *, struct ucred *)
int in_getpeeraddr(struct socket *so, struct sockaddr **nam)
#define V_ipport_tcpallocs
bool inp_match_t(const struct inpcb *, void *)
int in_pcbconnect(struct inpcb *, struct sockaddr *, struct ucred *, bool)
#define INP_HASH_WLOCK(ipi)
#define V_ipport_randomtime
void in_pcbpurgeif0(struct inpcbinfo *, struct ifnet *)
#define INP_RATE_LIMIT_CHANGED
#define V_ipport_hifirstauto
#define INPLOOKUP_LOCKMASK
#define V_ipport_firstauto
struct inpcb * in_pcblookup_mbuf(struct inpcbinfo *, struct in_addr, u_int, struct in_addr, u_int, int, struct ifnet *, struct mbuf *)
#define INP_UNLOCK_ASSERT(inp)
#define IN6P_AUTOFLOWLABEL
int inp_so_options(const struct inpcb *inp)
#define INP_INFO_WLOCK(ipi)
#define INP_CHECK_SOCKAF(so, af)
#define IN6P_RTHDRDSTOPTS
#define INP_LOCK_DESTROY(inp)
#define INP_RLOCK_ASSERT(inp)
struct sockaddr * in_sockaddr(in_port_t port, struct in_addr *addr)
void in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr, int, struct inpcb *(*)(struct inpcb *, int))
#define INP_ITERATOR(_ipi, _lock, _match, _ctx)
#define INP_PCBHASH(faddr, lport, fport, mask)
int in_pcb_lport(struct inpcb *, struct in_addr *, u_short *, struct ucred *, int)
struct inpcb * in_pcblookup_local(struct inpcbinfo *, struct in_addr, u_short, int, struct ucred *)
int in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa, u_short *lportp, struct sockaddr *fsa, u_short fport, struct ucred *cred, int lookupflags)
#define IN_MULTI_LOCK_ASSERT()
#define IP_MFILTER_FOREACH(imf, head)
static void ip_mfilter_remove(struct ip_mfilter_head *head, struct in_mfilter *imf)
static LIST_HEAD(carp_softc)
struct in_endpoints inc_ie
union in_dependaddr ie_dependladdr
struct sockaddr_in ia_addr
struct in_multi * imf_inm
const struct inpcbinfo * ipi
uint32_t inp_rss_listen_bucket
struct socket * inp_socket
struct inpcbport * inp_phd
struct icmp6_filter * in6p_icmp6filt
struct ip6_pktopts * in6p_outputopts
volatile uint16_t inp_hpts_cpu
struct inpcbpolicy * inp_sp
struct mbuf * inp_options
struct m_snd_tag * inp_snd_tag
struct ip6_moptions * in6p_moptions
struct ip_moptions * inp_moptions
struct inpcbinfo * inp_pcbinfo
struct mbuf * in6p_options
struct in_conninfo inp_inc
struct inpcbhead * ipi_hashbase
struct inpcblbgrouphead * ipi_lbgrouphashbase
u_long ipi_lbgrouphashmask
struct inpcbporthead * ipi_porthashbase
struct inpcbhead ipi_listhead
union in_dependaddr il_dependladdr
struct epoch_context il_epoch_ctx
struct inpcbhead phd_pcblist
const char * ips_zone_name
const char * ips_portzone_name
const char * ips_hashlock_name
const char * ips_infolock_name
#define TCP_REUSPORT_LB_NUMA_CURDOM
#define TCP_REUSPORT_LB_NUMA_NODOM
SYSCTL_COUNTER_U64(_net_inet_tcp_hpts_stats, OID_AUTO, hopeless, CTLFLAG_RD, &hpts_hopelessly_behind, "Number of times hpts could not catch up and was behind hopelessly")
uint16_t hpts_random_cpu(struct inpcb *inp)